summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorØyvind Grønnesby <oyving@verizonmedia.com>2021-11-23 16:19:20 +0100
committerØyvind Grønnesby <oyving@verizonmedia.com>2021-11-23 16:19:20 +0100
commitcd91614db66b12e84293b8ba938cf2103aa95b28 (patch)
tree0ce87ab3b7bd57e9f13aec9fdae2a56baeb6d22b
parentb342a8edb3c9ce28d53abbf89f70c1c3900a9db0 (diff)
parentbaa10a7a7762766d4b32d484bc21283cac543993 (diff)
Merge remote-tracking branch 'origin/master' into ogronnesby/supported-plans
-rw-r--r--abi-check-plugin/pom.xml2
-rw-r--r--client/go/auth0/auth0.go49
-rw-r--r--client/go/cmd/api_key.go10
-rw-r--r--client/go/cmd/config.go6
-rw-r--r--client/go/cmd/helpers.go23
-rw-r--r--client/go/cmd/login.go12
-rw-r--r--client/go/cmd/logout.go2
-rw-r--r--client/go/cmd/root.go1
-rw-r--r--client/go/vespa/deploy.go6
-rw-r--r--client/go/vespa/target.go84
-rw-r--r--client/go/vespa/target_test.go4
-rw-r--r--client/pom.xml2
-rw-r--r--cloud-tenant-base-dependencies-enforcer/pom.xml2
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java1
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java13
-rw-r--r--config-model/src/main/javacc/SDParser.jj14
-rw-r--r--config-model/src/test/derived/advanced/summary.cfg1
-rw-r--r--config-model/src/test/derived/array_of_struct_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/attributeprefetch/summary.cfg1
-rw-r--r--config-model/src/test/derived/complex/summary.cfg1
-rw-r--r--config-model/src/test/derived/emptychild/summary.cfg1
-rw-r--r--config-model/src/test/derived/emptydefault/summary.cfg1
-rw-r--r--config-model/src/test/derived/id/summary.cfg1
-rw-r--r--config-model/src/test/derived/imported_position_field/summary.cfg1
-rw-r--r--config-model/src/test/derived/imported_position_field_summary/summary.cfg1
-rw-r--r--config-model/src/test/derived/imported_struct_fields/summary.cfg1
-rw-r--r--config-model/src/test/derived/importedfields/summary.cfg1
-rw-r--r--config-model/src/test/derived/indexswitches/summary.cfg1
-rw-r--r--config-model/src/test/derived/inheritance/summary.cfg1
-rw-r--r--config-model/src/test/derived/integerattributetostringindex/summary.cfg1
-rw-r--r--config-model/src/test/derived/mail/summary.cfg1
-rw-r--r--config-model/src/test/derived/map_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/mlr/summary.cfg1
-rw-r--r--config-model/src/test/derived/multiplesummaries/summary.cfg1
-rw-r--r--config-model/src/test/derived/music/summary.cfg1
-rw-r--r--config-model/src/test/derived/newrank/summary.cfg1
-rw-r--r--config-model/src/test/derived/position_nosummary/summary.cfg1
-rw-r--r--config-model/src/test/derived/position_summary/summary.cfg1
-rw-r--r--config-model/src/test/derived/predicate_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/rankexpression/summary.cfg1
-rw-r--r--config-model/src/test/derived/ranktypes/summary.cfg1
-rw-r--r--config-model/src/test/derived/reference_fields/summary.cfg1
-rw-r--r--config-model/src/test/derived/schemainheritance/summary.cfg1
-rw-r--r--config-model/src/test/derived/streamingstruct/summary.cfg1
-rw-r--r--config-model/src/test/derived/streamingstructdefault/summary.cfg1
-rw-r--r--config-model/src/test/derived/tensor/summary.cfg1
-rw-r--r--config-model/src/test/derived/twostreamingstructs/summary.cfg1
-rw-r--r--config-model/src/test/derived/types/summary.cfg1
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java16
-rw-r--r--configdefinitions/src/vespa/summary.def3
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java4
-rw-r--r--container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java8
-rw-r--r--container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java32
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java77
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java38
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java50
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java51
-rw-r--r--container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java12
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java26
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java132
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java59
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java6
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java77
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java117
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java14
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java154
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java41
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java23
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java48
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java)2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java)2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java)2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java)2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java)2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java)2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java51
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java12
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java14
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java (renamed from controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java)2
-rw-r--r--controller-server/src/test/resources/test_runner_services.xml-cd (renamed from controller-server/src/test/resources/test_runner_services.xml-cd-osgi)0
-rw-r--r--controller-server/src/test/resources/test_runner_services.xml-cd-legacy22
-rw-r--r--default_build_settings.cmake2
-rw-r--r--dist/vespa.spec6
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/Flags.java14
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java2
-rw-r--r--integration/intellij/.gitignore (renamed from sd-plugin/.gitignore)7
-rw-r--r--integration/intellij/BACKLOG.md41
-rw-r--r--integration/intellij/README.md52
-rw-r--r--integration/intellij/build.gradle (renamed from sd-plugin/build.gradle)25
-rw-r--r--integration/intellij/pom.xml90
-rw-r--r--integration/intellij/settings.gradle5
-rw-r--r--integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf)117
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java)7
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettings.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java)5
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdIcons.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java)5
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguage.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java)5
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java)7
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java)7
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java)11
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java)44
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java)17
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java)10
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java)4
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java)7
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java)18
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java)15
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java)13
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java)13
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/lexer/SdLexerAdapter.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/lexer/SdLexerAdapter.java)6
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java)12
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclaration.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java)5
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclarationType.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java)5
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementFactory.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java)6
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java)8
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java)10
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java)10
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdIdentifier.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java)2
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdNamedElement.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java)8
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java)12
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java)49
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java)9
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java)20
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java)3
-rw-r--r--integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java)4
-rw-r--r--integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex (renamed from sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex)16
-rw-r--r--integration/intellij/src/main/resources/META-INF/plugin.xml41
-rw-r--r--integration/intellij/src/main/resources/META-INF/pluginIcon.svg19
-rw-r--r--integration/intellij/src/main/resources/icons/document_icon.png (renamed from sd-plugin/src/main/resources/icons/document_icon.png)bin436 -> 436 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/document_summary_icon.png (renamed from sd-plugin/src/main/resources/icons/document_summary_icon.png)bin426 -> 426 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/first_phase_icon.png (renamed from sd-plugin/src/main/resources/icons/first_phase_icon.png)bin424 -> 424 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/imported_field_icon.png (renamed from sd-plugin/src/main/resources/icons/imported_field_icon.png)bin538 -> 538 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/macro_icon.png (renamed from sd-plugin/src/main/resources/icons/macro_icon.png)bin579 -> 579 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/override_macro_icon.png (renamed from sd-plugin/src/main/resources/icons/override_macro_icon.png)bin703 -> 703 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/sd_icon.png (renamed from sd-plugin/src/main/resources/icons/sd_icon.png)bin682 -> 682 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/struct_field_icon.png (renamed from sd-plugin/src/main/resources/icons/struct_field_icon.png)bin627 -> 627 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/struct_icon.png (renamed from sd-plugin/src/main/resources/icons/struct_icon.png)bin279 -> 279 bytes
-rw-r--r--integration/intellij/src/main/resources/icons/summary_def_icon.png (renamed from sd-plugin/src/main/resources/icons/summary_def_icon.png)bin686 -> 686 bytes
-rw-r--r--parent/pom.xml5
-rw-r--r--persistence/src/vespa/persistence/conformancetest/conformancetest.cpp37
-rw-r--r--persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp39
-rw-r--r--persistence/src/vespa/persistence/dummyimpl/dummypersistence.h2
-rw-r--r--persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp9
-rw-r--r--persistence/src/vespa/persistence/spi/persistenceprovider.cpp5
-rw-r--r--persistence/src/vespa/persistence/spi/persistenceprovider.h3
-rw-r--r--persistence/src/vespa/persistence/spi/result.h1
-rw-r--r--pom.xml1
-rw-r--r--sd-plugin/BACKLOG.md34
-rw-r--r--sd-plugin/README.md42
-rw-r--r--sd-plugin/gradle/wrapper/gradle-wrapper.jarbin59536 -> 0 bytes
-rw-r--r--sd-plugin/gradle/wrapper/gradle-wrapper.properties5
-rwxr-xr-xsd-plugin/gradlew185
-rw-r--r--sd-plugin/gradlew.bat89
-rw-r--r--sd-plugin/settings.gradle3
-rw-r--r--sd-plugin/src/main/resources/META-INF/plugin.xml53
-rw-r--r--sd-plugin/src/main/resources/META-INF/pluginIcon.svg10
-rw-r--r--sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg10
-rw-r--r--sd-plugin/src/main/resources/flex/idea-flex.skeleton248
-rw-r--r--sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jarbin1301624 -> 0 bytes
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_test.cpp19
-rw-r--r--searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp8
-rw-r--r--searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp96
-rw-r--r--searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp38
-rw-r--r--searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp80
-rw-r--r--searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp48
-rw-r--r--searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentmetastore/CMakeLists.txt3
-rw-r--r--searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp17
-rwxr-xr-xsearchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh6
-rw-r--r--searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp18
-rw-r--r--searchcore/src/tests/proton/index/indexmanager_test.cpp26
-rw-r--r--searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp19
-rw-r--r--searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp31
-rw-r--r--searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp18
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h3
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp57
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp7
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/initializer/task_runner.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp50
-rw-r--r--searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h5
-rw-r--r--searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp17
-rw-r--r--searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h16
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp27
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt1
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.cpp75
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.h12
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp41
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h12
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp34
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp16
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/ifeedview.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp10
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp7
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h12
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h10
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp7
-rw-r--r--searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/test/thread_utils.h23
-rw-r--r--searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp26
-rw-r--r--searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp18
-rw-r--r--searchlib/src/vespa/searchlib/attribute/reference_attribute.h3
-rw-r--r--searchsummary/src/tests/docsummary/positionsdfw_test.cpp21
-rw-r--r--searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp4
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp71
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultclass.h1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp5
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h8
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp1
-rw-r--r--storage/src/tests/distributor/distributor_stripe_test_util.h2
-rw-r--r--storage/src/tests/persistence/common/persistenceproviderwrapper.cpp8
-rw-r--r--storage/src/tests/persistence/common/persistenceproviderwrapper.h2
-rw-r--r--storage/src/tests/persistence/persistencetestutils.cpp13
-rw-r--r--storage/src/tests/persistence/persistencetestutils.h4
-rw-r--r--storage/src/tests/persistence/processalltest.cpp10
-rw-r--r--storage/src/vespa/storage/persistence/asynchandler.cpp22
-rw-r--r--storage/src/vespa/storage/persistence/mergehandler.cpp7
-rw-r--r--storage/src/vespa/storage/persistence/provider_error_wrapper.cpp4
-rw-r--r--storage/src/vespa/storage/persistence/provider_error_wrapper.h2
-rw-r--r--vespa-feed-client/abi-spec.json1
-rw-r--r--vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java2
-rw-r--r--vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java28
-rw-r--r--vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java7
-rw-r--r--vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java8
-rw-r--r--vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java15
-rw-r--r--vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java63
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java29
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java1
-rw-r--r--vespalib/src/tests/datastore/datastore/datastore_test.cpp52
-rw-r--r--vespalib/src/vespa/vespalib/datastore/datastore.h4
-rw-r--r--vespalib/src/vespa/vespalib/datastore/datastore.hpp17
-rw-r--r--vsm/src/vespa/vsm/vsm/docsumfilter.cpp2
289 files changed, 2331 insertions, 2187 deletions
diff --git a/abi-check-plugin/pom.xml b/abi-check-plugin/pom.xml
index 7317205816c..059e593c0e9 100644
--- a/abi-check-plugin/pom.xml
+++ b/abi-check-plugin/pom.xml
@@ -40,7 +40,7 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>2.8.5</version>
+ <version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
diff --git a/client/go/auth0/auth0.go b/client/go/auth0/auth0.go
index 0cb613f13b7..377d56a1637 100644
--- a/client/go/auth0/auth0.go
+++ b/client/go/auth0/auth0.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
+ "net/url"
"os"
"os/signal"
"path/filepath"
@@ -16,7 +17,6 @@ import (
"sync"
"time"
- "github.com/joeshaw/envdecode"
"github.com/lestrrat-go/jwx/jwt"
"github.com/pkg/browser"
"github.com/vespa-engine/vespa/client/go/auth"
@@ -41,21 +41,19 @@ type System struct {
type Auth0 struct {
Authenticator *auth.Authenticator
system string
+ systemApiUrl string
initOnce sync.Once
errOnce error
Path string
config config
}
-// default to vespa-cd.auth0.com
-var (
- authCfg struct {
- Audience string `env:"AUTH0_AUDIENCE,default=https://vespa-cd.auth0.com/api/v2/"`
- ClientID string `env:"AUTH0_CLIENT_ID,default=4wYWA496zBP28SLiz0PuvCt8ltL11DZX"`
- DeviceCodeEndpoint string `env:"AUTH0_DEVICE_CODE_ENDPOINT,default=https://vespa-cd.auth0.com/oauth/device/code"`
- OauthTokenEndpoint string `env:"AUTH0_OAUTH_TOKEN_ENDPOINT,default=https://vespa-cd.auth0.com/oauth/token"`
- }
-)
+type authCfg struct {
+ Audience string `json:"audience"`
+ ClientID string `json:"client-id"`
+ DeviceCodeEndpoint string `json:"device-code-endpoint"`
+ OauthTokenEndpoint string `json:"oauth-token-endpoint"`
+}
func ContextWithCancel() context.Context {
ctx, cancel := context.WithCancel(context.Background())
@@ -71,22 +69,39 @@ func ContextWithCancel() context.Context {
// GetAuth0 will try to initialize the config context, as well as figure out if
// there's a readily available system.
-func GetAuth0(configPath string, systemName string) (*Auth0, error) {
+func GetAuth0(configPath string, systemName string, systemApiUrl string) (*Auth0, error) {
a := Auth0{}
a.Path = configPath
a.system = systemName
- if err := envdecode.StrictDecode(&authCfg); err != nil {
- return nil, fmt.Errorf("could not decode env: %w", err)
+ a.systemApiUrl = systemApiUrl
+ c, err := a.getDeviceFlowConfig()
+ if err != nil {
+ return nil, fmt.Errorf("cannot get auth config: %w", err)
}
a.Authenticator = &auth.Authenticator{
- Audience: authCfg.Audience,
- ClientID: authCfg.ClientID,
- DeviceCodeEndpoint: authCfg.DeviceCodeEndpoint,
- OauthTokenEndpoint: authCfg.OauthTokenEndpoint,
+ Audience: c.Audience,
+ ClientID: c.ClientID,
+ DeviceCodeEndpoint: c.DeviceCodeEndpoint,
+ OauthTokenEndpoint: c.OauthTokenEndpoint,
}
return &a, nil
}
+func (a *Auth0) getDeviceFlowConfig() (authCfg, error) {
+ systemApiUrl, _ := url.Parse(a.systemApiUrl + "/auth0/v1/device-flow-config")
+ r, err := http.Get(systemApiUrl.String())
+ if err != nil {
+ return authCfg{}, fmt.Errorf("cannot get auth config: %w", err)
+ }
+ defer r.Body.Close()
+ var res authCfg
+ err = json.NewDecoder(r.Body).Decode(&res)
+ if err != nil {
+ return authCfg{}, fmt.Errorf("cannot decode response: %w", err)
+ }
+ return res, nil
+}
+
// IsLoggedIn encodes the domain logic for determining whether we're
// logged in. This might check our config storage, or just in memory.
func (a *Auth0) IsLoggedIn() bool {
diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go
index ba2df8c40dc..9832f04e3f0 100644
--- a/client/go/cmd/api_key.go
+++ b/client/go/cmd/api_key.go
@@ -48,6 +48,16 @@ var apiKeyCmd = &cobra.Command{
if err := ioutil.WriteFile(apiKeyFile, apiKey, 0600); err == nil {
printSuccess("API private key written to ", apiKeyFile)
printPublicKey(apiKeyFile, app.Tenant)
+ if vespa.Auth0AccessTokenEnabled() {
+ if err == nil {
+ if err := cfg.Set(cloudAuthFlag, "api-key"); err != nil {
+ fatalErr(err, "Could not write config")
+ }
+ if err := cfg.Write(); err != nil {
+ fatalErr(err)
+ }
+ }
+ }
} else {
fatalErr(err, "Failed to write ", apiKeyFile)
}
diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go
index 62a97d9749d..750664e51b1 100644
--- a/client/go/cmd/config.go
+++ b/client/go/cmd/config.go
@@ -235,6 +235,12 @@ func (c *Config) Set(option, value string) error {
viper.Set(option, value)
return nil
}
+ case cloudAuthFlag:
+ switch value {
+ case "access-token", "api-key":
+ viper.Set(option, value)
+ return nil
+ }
}
return fmt.Errorf("invalid option or value: %q: %q", option, value)
}
diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go
index 084148f9bdc..79ba1fcef26 100644
--- a/client/go/cmd/helpers.go
+++ b/client/go/cmd/helpers.go
@@ -182,8 +182,8 @@ func getTarget() vespa.Target {
return nil
}
var apiKey []byte = nil
+ apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant))
if !vespa.Auth0AccessTokenEnabled() {
- apiKey, err = ioutil.ReadFile(cfg.APIKeyPath(deployment.Application.Tenant))
if err != nil {
fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
}
@@ -202,6 +202,20 @@ func getTarget() vespa.Target {
if err != nil {
fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'")
}
+ var cloudAuth string
+ if vespa.Auth0AccessTokenEnabled() {
+ cloudAuth, err = cfg.Get(cloudAuthFlag)
+ if err != nil {
+ if apiKey != nil {
+ cloudAuth = "api-key"
+ } else {
+ cloudAuth = "access-token"
+ }
+ }
+ } else {
+ cloudAuth = ""
+ }
+
return vespa.CloudTarget(getApiURL(), deployment, apiKey,
vespa.TLSOptions{
KeyPair: kp,
@@ -213,7 +227,8 @@ func getTarget() vespa.Target {
Level: vespa.LogLevel(logLevelArg),
},
cfg.AuthConfigPath(),
- getSystemName())
+ getSystemName(),
+ cloudAuth)
}
fatalErrHint(fmt.Errorf("Invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL")
return nil
@@ -244,9 +259,9 @@ func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.T
fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'")
return opts
}
+ var err error
+ opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
if !vespa.Auth0AccessTokenEnabled() {
- var err error
- opts.APIKey, err = cfg.ReadAPIKey(deployment.Application.Tenant)
if err != nil {
fatalErrHint(err, "Deployment to cloud requires an API key. Try 'vespa api-key'")
return opts
diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go
index 0e09a6d6244..f7b412a4613 100644
--- a/client/go/cmd/login.go
+++ b/client/go/cmd/login.go
@@ -24,11 +24,21 @@ var loginCmd = &cobra.Command{
if err != nil {
return err
}
- a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName())
+ a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL())
if err != nil {
return err
}
_, err = auth0.RunLogin(ctx, a, false)
+ if vespa.Auth0AccessTokenEnabled() {
+ if err == nil {
+ if err := cfg.Set(cloudAuthFlag, "access-token"); err != nil {
+ return err
+ }
+ if err := cfg.Write(); err != nil {
+ fatalErr(err)
+ }
+ }
+ }
return err
},
}
diff --git a/client/go/cmd/logout.go b/client/go/cmd/logout.go
index 18b7a2166ad..e3cfe6733eb 100644
--- a/client/go/cmd/logout.go
+++ b/client/go/cmd/logout.go
@@ -24,7 +24,7 @@ var logoutCmd = &cobra.Command{
if err != nil {
return err
}
- a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName())
+ a, err := auth0.GetAuth0(cfg.AuthConfigPath(), getSystemName(), getApiURL())
if err != nil {
return err
}
diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go
index 5aae55ab6e4..087be7c352d 100644
--- a/client/go/cmd/root.go
+++ b/client/go/cmd/root.go
@@ -51,6 +51,7 @@ const (
waitFlag = "wait"
colorFlag = "color"
quietFlag = "quiet"
+ cloudAuthFlag = "cloudAuth"
)
func isTerminal() bool {
diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go
index 9c5fb3a12ac..252a646bcfc 100644
--- a/client/go/vespa/deploy.go
+++ b/client/go/vespa/deploy.go
@@ -344,8 +344,10 @@ func checkDeploymentOpts(opts DeploymentOpts) error {
if !opts.ApplicationPackage.HasCertificate() {
return fmt.Errorf("%s: missing certificate in package", opts)
}
- if !Auth0AccessTokenEnabled() && opts.APIKey == nil {
- return fmt.Errorf("%s: missing api key", opts.String())
+ if !Auth0AccessTokenEnabled() {
+ if opts.APIKey == nil {
+ return fmt.Errorf("%s: missing api key", opts.String())
+ }
}
return nil
}
diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go
index e4779e14c0d..093cb2b5cad 100644
--- a/client/go/vespa/target.go
+++ b/client/go/vespa/target.go
@@ -209,10 +209,34 @@ type cloudTarget struct {
tlsOptions TLSOptions
logOptions LogOptions
- queryURL string
- documentURL string
+ urlsByCluster map[string]string
authConfigPath string
systemName string
+ cloudAuth string
+}
+
+func (t *cloudTarget) resolveEndpoint(cluster string) (string, error) {
+ if cluster == "" {
+ for _, u := range t.urlsByCluster {
+ if len(t.urlsByCluster) == 1 {
+ return u, nil
+ } else {
+ return "", fmt.Errorf("multiple clusters, none chosen: %v", t.urlsByCluster)
+ }
+ }
+ } else {
+ u := t.urlsByCluster[cluster]
+ if u == "" {
+ clusters := make([]string, len(t.urlsByCluster))
+ for c := range t.urlsByCluster {
+ clusters = append(clusters, c)
+ }
+ return "", fmt.Errorf("unknown cluster '%s': must be one of %v", cluster, clusters)
+ }
+ return u, nil
+ }
+
+ return "", fmt.Errorf("no endpoints")
}
func (t *cloudTarget) Type() string { return t.targetType }
@@ -227,25 +251,37 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64) (
case deployService:
return &Service{Name: name, BaseURL: t.apiURL}, nil
case queryService:
- if t.queryURL == "" {
- return nil, fmt.Errorf("service %s is not discovered", name)
+ queryURL, err := t.resolveEndpoint("")
+ if err != nil {
+ return nil, err
}
- return &Service{Name: name, BaseURL: t.queryURL, TLSOptions: t.tlsOptions}, nil
+ return &Service{Name: name, BaseURL: queryURL, TLSOptions: t.tlsOptions}, nil
case documentService:
- if t.documentURL == "" {
- return nil, fmt.Errorf("service %s is not discovered", name)
+ documentURL, err := t.resolveEndpoint("")
+ if err != nil {
+ return nil, err
}
- return &Service{Name: name, BaseURL: t.documentURL, TLSOptions: t.tlsOptions}, nil
+ return &Service{Name: name, BaseURL: documentURL, TLSOptions: t.tlsOptions}, nil
}
return nil, fmt.Errorf("unknown service: %s", name)
}
func (t *cloudTarget) PrepareApiRequest(req *http.Request, sigKeyId string) error {
if Auth0AccessTokenEnabled() {
- if err := t.addAuth0AccessToken(req); err != nil {
- return err
+ if t.cloudAuth == "access-token" {
+ if err := t.addAuth0AccessToken(req); err != nil {
+ return err
+ }
+ } else {
+ if t.apiKey == nil {
+ return fmt.Errorf("Deployment to cloud requires an API key. Try 'vespa api-key'")
+ }
+ signer := NewRequestSigner(sigKeyId, t.apiKey)
+ if err := signer.SignRequest(req); err != nil {
+ return err
+ }
}
- } else if t.apiKey != nil {
+ } else {
signer := NewRequestSigner(sigKeyId, t.apiKey)
if err := signer.SignRequest(req); err != nil {
return err
@@ -255,7 +291,7 @@ func (t *cloudTarget) PrepareApiRequest(req *http.Request, sigKeyId string) erro
}
func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error {
- a, err := auth0.GetAuth0(t.authConfigPath, t.systemName)
+ a, err := auth0.GetAuth0(t.authConfigPath, t.systemName, t.apiURL)
system, err := a.PrepareSystem(auth0.ContextWithCancel())
if err != nil {
return err
@@ -404,7 +440,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error {
if err := t.PrepareApiRequest(req, t.deployment.Application.SerializedForm()); err != nil {
return err
}
- var endpointURL string
+ urlsByCluster := make(map[string]string)
endpointFunc := func(status int, response []byte) (bool, error) {
if ok, err := isOK(status); !ok {
return ok, err
@@ -416,17 +452,21 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error {
if len(resp.Endpoints) == 0 {
return false, nil
}
- endpointURL = resp.Endpoints[0].URL
+ for _, endpoint := range resp.Endpoints {
+ if endpoint.Scope != "zone" {
+ continue
+ }
+ urlsByCluster[endpoint.Cluster] = endpoint.URL
+ }
return true, nil
}
if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.tlsOptions.KeyPair, timeout); err != nil {
return err
}
- if endpointURL == "" {
- return fmt.Errorf("no endpoint discovered")
+ if len(urlsByCluster) == 0 {
+ return fmt.Errorf("no endpoints discovered")
}
- t.queryURL = endpointURL
- t.documentURL = endpointURL
+ t.urlsByCluster = urlsByCluster
return nil
}
@@ -448,7 +488,8 @@ func CustomTarget(baseURL string) Target {
}
// CloudTarget creates a Target for the Vespa Cloud platform.
-func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions, authConfigPath string, systemName string) Target {
+func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions,
+ authConfigPath string, systemName string, cloudAuth string) Target {
return &cloudTarget{
apiURL: apiURL,
targetType: cloudTargetType,
@@ -458,11 +499,14 @@ func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions
logOptions: logOptions,
authConfigPath: authConfigPath,
systemName: systemName,
+ cloudAuth: cloudAuth,
}
}
type deploymentEndpoint struct {
- URL string `json:"url"`
+ Cluster string `json:"cluster"`
+ URL string `json:"url"`
+ Scope string `json:"scope"`
}
type deploymentResponse struct {
diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go
index 62bde3044bf..9d2418897e3 100644
--- a/client/go/vespa/target_test.go
+++ b/client/go/vespa/target_test.go
@@ -25,7 +25,7 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request
case "/application/v4/tenant/t1/application/a1/instance/i1/environment/dev/region/us-north-1":
response := "{}"
if v.deploymentConverged {
- response = fmt.Sprintf(`{"endpoints": [{"url": "%s"}]}`, v.serverURL)
+ response = fmt.Sprintf(`{"endpoints": [{"url": "%s","scope": "zone","cluster": "cluster1"}]}`, v.serverURL)
}
w.Write([]byte(response))
case "/application/v4/tenant/t1/application/a1/instance/i1/job/dev-us-north-1/run/42":
@@ -152,7 +152,7 @@ func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target {
target := CloudTarget("https://example.com", Deployment{
Application: ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"},
Zone: ZoneID{Environment: "dev", Region: "us-north-1"},
- }, apiKey, TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: logWriter}, "", "")
+ }, apiKey, TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: logWriter}, "", "", "")
if ct, ok := target.(*cloudTarget); ok {
ct.apiURL = url
} else {
diff --git a/client/pom.xml b/client/pom.xml
index 4abcdf9ac6c..3dee909b932 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -20,7 +20,7 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>2.8.5</version>
+ <version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml
index 7113787a715..e2cc7085353 100644
--- a/cloud-tenant-base-dependencies-enforcer/pom.xml
+++ b/cloud-tenant-base-dependencies-enforcer/pom.xml
@@ -238,7 +238,7 @@
<include>net.java.dev.jna:jna:4.5.2:jar:test</include>
<include>org.abego.treelayout:org.abego.treelayout.core:1.0.1:jar:test</include>
<include>org.antlr:antlr-runtime:3.5.2:jar:test</include>
- <include>org.antlr:antlr4-runtime:4.9.3:jar:test</include>
+ <include>org.antlr:antlr4-runtime:4.5:jar:test</include>
<include>org.apache.commons:commons-exec:1.3:jar:test</include>
<include>org.apache.commons:commons-math3:3.6.1:jar:test</include>
<include>org.apache.httpcomponents:httpclient:4.5.12:jar:test</include>
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 42041f3a0b0..6ed2f3daa30 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -108,6 +108,7 @@ public interface ModelContext {
@ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return false; }
@ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; }
@ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return false; }
+ @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; }
}
/** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 6165cdd2dc1..b49fde9e5d7 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -91,7 +91,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer {
}
if ( ! schema.isDocumentsOnly()) {
attributeFields = new AttributeFields(schema);
- summaries = new Summaries(schema, deployLogger);
+ summaries = new Summaries(schema, deployLogger, deployProperties.featureFlags());
summaryMap = new SummaryMap(schema);
juniperrc = new Juniperrc(schema);
rankProfileList = new RankProfileList(schema, schema.rankingConstants(), schema.rankExpressionFiles(),
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
index 5fdc51e8f5f..1455fbc92e1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
@@ -2,6 +2,7 @@
package com.yahoo.searchdefinition.derived;
import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.ModelContext;
import com.yahoo.searchdefinition.Schema;
import com.yahoo.vespa.documentmodel.DocumentSummary;
import com.yahoo.vespa.config.search.SummaryConfig;
@@ -14,9 +15,11 @@ import java.util.List;
*/
public class Summaries extends Derived implements SummaryConfig.Producer {
+ private final boolean useV8GeoPositions;
private List<SummaryClass> summaries=new java.util.ArrayList<>(1);
- public Summaries(Schema schema, DeployLogger deployLogger) {
+ public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) {
+ this.useV8GeoPositions = featureFlags.useV8GeoPositions();
// Make sure the default is first
summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger));
for (DocumentSummary summary : schema.getSummaries().values()) {
@@ -31,6 +34,7 @@ public class Summaries extends Derived implements SummaryConfig.Producer {
@Override
public void getConfig(SummaryConfig.Builder builder) {
builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode());
+ builder.usev8geopositions(useV8GeoPositions);
for (SummaryClass summaryClass : summaries) {
builder.classes(summaryClass.getSummaryClassConfig());
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index a65e6fe16c0..f167a79fd05 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -165,6 +165,7 @@ public class VespaMetricSet {
metrics.add(new Metric("httpapi_succeeded.rate"));
metrics.add(new Metric("httpapi_failed.rate"));
metrics.add(new Metric("httpapi_parse_error.rate"));
+ addMetric(metrics, "httpapi_test_and_set_condition_not_met", List.of("rate"));
metrics.add(new Metric("mem.heap.total.average"));
metrics.add(new Metric("mem.heap.free.average"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 81dd458570b..f69b08ff300 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -114,8 +114,8 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler";
public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName();
public static final String LOG_HANDLER_CLASS = com.yahoo.container.handler.LogHandler.class.getName();
- public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1";
public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15";
+ public static final String PARALLEL_GC = "-XX:+UseParallelGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1";
public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler";
public static final BindingPattern STATE_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(StateHandler.STATE_API_ROOT);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index d65fbba6a5e..319f334ffc2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -8,7 +8,6 @@ import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.application.api.Endpoint;
import com.yahoo.config.model.ConfigModelContext;
import com.yahoo.config.model.ConfigModelContext.ApplicationType;
import com.yahoo.config.model.api.ApplicationClusterEndpoint;
@@ -661,15 +660,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
static boolean incompatibleGCOptions(String jvmargs) {
Pattern gcAlgorithm = Pattern.compile("-XX:[-+]Use.+GC");
Pattern cmsArgs = Pattern.compile("-XX:[-+]*CMS");
- return (gcAlgorithm.matcher(jvmargs).find() ||cmsArgs.matcher(jvmargs).find());
+ return (gcAlgorithm.matcher(jvmargs).find() || cmsArgs.matcher(jvmargs).find());
}
- private static String buildJvmGCOptions(DeployState deployState, String jvmGCOPtions) {
- String options = (jvmGCOPtions != null)
- ? jvmGCOPtions
+ private static String buildJvmGCOptions(DeployState deployState, String jvmGCOptions) {
+ String options = (jvmGCOptions != null)
+ ? jvmGCOptions
: deployState.getProperties().jvmGCOptions();
return (options == null || options.isEmpty())
- ? (deployState.isHosted() ? ContainerCluster.CMS : ContainerCluster.G1GC)
+ ? (deployState.isHosted() ? ContainerCluster.PARALLEL_GC : ContainerCluster.G1GC)
: options;
}
@@ -685,7 +684,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
} else {
jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME);
if (incompatibleGCOptions(jvmOptions)) {
- deployLogger.logApplicationPackage(WARNING, "You need to move out your GC related options from 'jvmargs' to 'jvm-gc-options'");
+ deployLogger.logApplicationPackage(WARNING, "You need to move out your GC-related options from deprecated 'jvmargs' to 'jvm-gc-options'");
cluster.setJvmGCOptions(ContainerCluster.G1GC);
}
}
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index c3851f2918a..d3d992c11f5 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -1,10 +1,9 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// --------------------------------------------------------------------------------
-//
-// JavaCC options. When this file is changed, run "mvn generate-sources" to rebuild
-// the parser classes.
+
+// Schema parser.
//
-// --------------------------------------------------------------------------------
+// NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
+
options {
UNICODE_INPUT = true;
CACHE_TOKENS = false;
@@ -15,11 +14,6 @@ options {
USER_CHAR_STREAM = true;
}
-// --------------------------------------------------------------------------------
-//
-// Parser body.
-//
-// --------------------------------------------------------------------------------
PARSER_BEGIN(SDParser)
package com.yahoo.searchdefinition.parser;
diff --git a/config-model/src/test/derived/advanced/summary.cfg b/config-model/src/test/derived/advanced/summary.cfg
index f497461b460..e4845c1994e 100644
--- a/config-model/src/test/derived/advanced/summary.cfg
+++ b/config-model/src/test/derived/advanced/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1271952241
+usev8geopositions false
classes[].id 1271952241
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
index 965c875d5ce..e750d1454e8 100644
--- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
+++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 252850086
+usev8geopositions false
classes[].id 252850086
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg
index f0189f9a3c7..f52952e2871 100644
--- a/config-model/src/test/derived/attributeprefetch/summary.cfg
+++ b/config-model/src/test/derived/attributeprefetch/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1151071433
+usev8geopositions false
classes[].id 1151071433
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg
index 2dac4736d23..1eeef44cd54 100644
--- a/config-model/src/test/derived/complex/summary.cfg
+++ b/config-model/src/test/derived/complex/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1506848752
+usev8geopositions false
classes[].id 1506848752
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/emptychild/summary.cfg b/config-model/src/test/derived/emptychild/summary.cfg
index 82bed7fd55e..733fa1fde54 100644
--- a/config-model/src/test/derived/emptychild/summary.cfg
+++ b/config-model/src/test/derived/emptychild/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1814603381
+usev8geopositions false
classes[].id 1814603381
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/emptydefault/summary.cfg b/config-model/src/test/derived/emptydefault/summary.cfg
index 61294d97b4c..5aacec1d0b6 100644
--- a/config-model/src/test/derived/emptydefault/summary.cfg
+++ b/config-model/src/test/derived/emptydefault/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1151071433
+usev8geopositions false
classes[].id 1151071433
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/id/summary.cfg b/config-model/src/test/derived/id/summary.cfg
index b50b970afe2..f7e9f6a239f 100644
--- a/config-model/src/test/derived/id/summary.cfg
+++ b/config-model/src/test/derived/id/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1814716401
+usev8geopositions false
classes[].id 1814716401
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg
index 3ab8e7e29e5..722443641cd 100644
--- a/config-model/src/test/derived/imported_position_field/summary.cfg
+++ b/config-model/src/test/derived/imported_position_field/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1570252291
+usev8geopositions false
classes[].id 1570252291
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/imported_position_field_summary/summary.cfg b/config-model/src/test/derived/imported_position_field_summary/summary.cfg
index 76faac23170..0642382aabe 100644
--- a/config-model/src/test/derived/imported_position_field_summary/summary.cfg
+++ b/config-model/src/test/derived/imported_position_field_summary/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1194448774
+usev8geopositions false
classes[].id 1194448774
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/imported_struct_fields/summary.cfg b/config-model/src/test/derived/imported_struct_fields/summary.cfg
index ab6c6853925..0a9b29524e1 100644
--- a/config-model/src/test/derived/imported_struct_fields/summary.cfg
+++ b/config-model/src/test/derived/imported_struct_fields/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1570252291
+usev8geopositions false
classes[].id 1570252291
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/importedfields/summary.cfg b/config-model/src/test/derived/importedfields/summary.cfg
index 74b5b44214e..975c601a97d 100644
--- a/config-model/src/test/derived/importedfields/summary.cfg
+++ b/config-model/src/test/derived/importedfields/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1294344677
+usev8geopositions false
classes[].id 1294344677
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/indexswitches/summary.cfg b/config-model/src/test/derived/indexswitches/summary.cfg
index d04bc4eb167..bcc050dad4f 100644
--- a/config-model/src/test/derived/indexswitches/summary.cfg
+++ b/config-model/src/test/derived/indexswitches/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1698765342
+usev8geopositions false
classes[].id 1698765342
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/inheritance/summary.cfg b/config-model/src/test/derived/inheritance/summary.cfg
index dde9f95ecbe..73c22f82a99 100644
--- a/config-model/src/test/derived/inheritance/summary.cfg
+++ b/config-model/src/test/derived/inheritance/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1797992819
+usev8geopositions false
classes[].id 1797992819
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/integerattributetostringindex/summary.cfg b/config-model/src/test/derived/integerattributetostringindex/summary.cfg
index d5eb316ff01..bfbb1eb01ca 100644
--- a/config-model/src/test/derived/integerattributetostringindex/summary.cfg
+++ b/config-model/src/test/derived/integerattributetostringindex/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1195656216
+usev8geopositions false
classes[].id 1195656216
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/mail/summary.cfg b/config-model/src/test/derived/mail/summary.cfg
index 3628e1acb88..2b886a4622b 100644
--- a/config-model/src/test/derived/mail/summary.cfg
+++ b/config-model/src/test/derived/mail/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1831052622
+usev8geopositions false
classes[].id 1831052622
classes[].name "default"
classes[].fields[].name "snippet"
diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg
index b465bdfa541..b50b40b9b72 100644
--- a/config-model/src/test/derived/map_attribute/summary.cfg
+++ b/config-model/src/test/derived/map_attribute/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1376056200
+usev8geopositions false
classes[].id 1376056200
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
index 67988dbf30e..cf875704fa0 100644
--- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
+++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1131098132
+usev8geopositions false
classes[].id 1131098132
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/mlr/summary.cfg b/config-model/src/test/derived/mlr/summary.cfg
index b6a53a9a1d9..5bea06a8472 100644
--- a/config-model/src/test/derived/mlr/summary.cfg
+++ b/config-model/src/test/derived/mlr/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1868876861
+usev8geopositions false
classes[].id 1868876861
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg
index 8d85890b156..16d0024155b 100644
--- a/config-model/src/test/derived/multiplesummaries/summary.cfg
+++ b/config-model/src/test/derived/multiplesummaries/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 235127765
+usev8geopositions false
classes[1156201411].id 1156201411
classes[1156201411].name "attributeprefetch"
classes[1156201411].fields[a].name "a"
diff --git a/config-model/src/test/derived/music/summary.cfg b/config-model/src/test/derived/music/summary.cfg
index bc55727b407..c92b5491ebc 100644
--- a/config-model/src/test/derived/music/summary.cfg
+++ b/config-model/src/test/derived/music/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 2086497905
+usev8geopositions false
classes[].id 2086497905
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/newrank/summary.cfg b/config-model/src/test/derived/newrank/summary.cfg
index 0b98b20c342..ddebcbe6ca6 100644
--- a/config-model/src/test/derived/newrank/summary.cfg
+++ b/config-model/src/test/derived/newrank/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 912980235
+usev8geopositions false
classes[].id 912980235
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg
index 4222e88cc2f..cd7c295ab11 100644
--- a/config-model/src/test/derived/position_nosummary/summary.cfg
+++ b/config-model/src/test/derived/position_nosummary/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1727020212
+usev8geopositions false
classes[].id 1727020212
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg
index f54066d865e..7b8bf16287f 100644
--- a/config-model/src/test/derived/position_summary/summary.cfg
+++ b/config-model/src/test/derived/position_summary/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 230670304
+usev8geopositions false
classes[].id 230670304
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/predicate_attribute/summary.cfg b/config-model/src/test/derived/predicate_attribute/summary.cfg
index 9cc613107e0..10040fd71c6 100644
--- a/config-model/src/test/derived/predicate_attribute/summary.cfg
+++ b/config-model/src/test/derived/predicate_attribute/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1391971216
+usev8geopositions false
classes[].id 1391971216
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg
index 22f4c3d4ca7..aec076aa8fe 100644
--- a/config-model/src/test/derived/rankexpression/summary.cfg
+++ b/config-model/src/test/derived/rankexpression/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1753207254
+usev8geopositions false
classes[].id 1753207254
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/ranktypes/summary.cfg b/config-model/src/test/derived/ranktypes/summary.cfg
index 49b668e9edf..7ab01302d8b 100644
--- a/config-model/src/test/derived/ranktypes/summary.cfg
+++ b/config-model/src/test/derived/ranktypes/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1567556360
+usev8geopositions false
classes[].id 1567556360
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/reference_fields/summary.cfg b/config-model/src/test/derived/reference_fields/summary.cfg
index 410bccff7b3..b23b6c5b5a8 100644
--- a/config-model/src/test/derived/reference_fields/summary.cfg
+++ b/config-model/src/test/derived/reference_fields/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1987541865
+usev8geopositions false
classes[].id 1987541865
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/schemainheritance/summary.cfg b/config-model/src/test/derived/schemainheritance/summary.cfg
index d3286961007..6fcf5b2aaa8 100644
--- a/config-model/src/test/derived/schemainheritance/summary.cfg
+++ b/config-model/src/test/derived/schemainheritance/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1313596701
+usev8geopositions false
classes[].id 1313596701
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/streamingstruct/summary.cfg b/config-model/src/test/derived/streamingstruct/summary.cfg
index 655499a88be..d7b7057bf1f 100644
--- a/config-model/src/test/derived/streamingstruct/summary.cfg
+++ b/config-model/src/test/derived/streamingstruct/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 569269436
+usev8geopositions false
classes[].id 569269436
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/streamingstructdefault/summary.cfg b/config-model/src/test/derived/streamingstructdefault/summary.cfg
index a52b34925dc..7274804a5de 100644
--- a/config-model/src/test/derived/streamingstructdefault/summary.cfg
+++ b/config-model/src/test/derived/streamingstructdefault/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 718801936
+usev8geopositions false
classes[].id 718801936
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg
index 355cba0e561..de6a2a6e386 100644
--- a/config-model/src/test/derived/tensor/summary.cfg
+++ b/config-model/src/test/derived/tensor/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 898020074
+usev8geopositions false
classes[].id 898020074
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/derived/twostreamingstructs/summary.cfg b/config-model/src/test/derived/twostreamingstructs/summary.cfg
index 28f19e6fe25..969e91d4b87 100644
--- a/config-model/src/test/derived/twostreamingstructs/summary.cfg
+++ b/config-model/src/test/derived/twostreamingstructs/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 569269436
+usev8geopositions false
classes[].id 569269436
classes[].name "default"
classes[].fields[].name "coupleof"
diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg
index e0e67a5669d..fc3b9e655ca 100644
--- a/config-model/src/test/derived/types/summary.cfg
+++ b/config-model/src/test/derived/types/summary.cfg
@@ -1,4 +1,5 @@
defaultsummaryid 1131946680
+usev8geopositions false
classes[].id 1131946680
classes[].name "default"
classes[].omitsummaryfeatures false
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java
index 9fa86d38142..9fda6016969 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java
@@ -132,14 +132,14 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase {
@Test
public void requireThatJvmGCOptionsIsHonoured() throws IOException, SAXException {
- verifyJvmGCOptions(false, null,null, ContainerCluster.G1GC);
- verifyJvmGCOptions(true, null,null, ContainerCluster.CMS);
- verifyJvmGCOptions(true, "",null, ContainerCluster.CMS);
- verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC");
- verifyJvmGCOptions(true, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC");
- verifyJvmGCOptions(false, null,"-XX:+UseG1GC", "-XX:+UseG1GC");
- verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC","-XX:+UseG1GC", "-XX:+UseG1GC");
- verifyJvmGCOptions(false, null,"-XX:+UseConcMarkSweepGC", "-XX:+UseConcMarkSweepGC");
+ verifyJvmGCOptions(false, null, null, ContainerCluster.G1GC);
+ verifyJvmGCOptions(true, null, null, ContainerCluster.PARALLEL_GC);
+ verifyJvmGCOptions(true, "", null, ContainerCluster.PARALLEL_GC);
+ verifyJvmGCOptions(false, "-XX:+UseG1GC", null, "-XX:+UseG1GC");
+ verifyJvmGCOptions(true, "-XX:+UseG1GC", null, "-XX:+UseG1GC");
+ verifyJvmGCOptions(false, null, "-XX:+UseG1GC", "-XX:+UseG1GC");
+ verifyJvmGCOptions(false, "-XX:+UseParallelGC", "-XX:+UseG1GC", "-XX:+UseG1GC");
+ verifyJvmGCOptions(false, null, "-XX:+UseParallelGC", "-XX:+UseParallelGC");
}
}
diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def
index 5169df7d72e..9b231d86c38 100644
--- a/configdefinitions/src/vespa/summary.def
+++ b/configdefinitions/src/vespa/summary.def
@@ -2,6 +2,9 @@
namespace=vespa.config.search
defaultsummaryid int default=-1
+
+usev8geopositions bool default=false
+
classes[].id int
classes[].name string
classes[].omitsummaryfeatures bool default=false
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index ea823e5befb..a7deac443a5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -200,6 +200,7 @@ public class ModelContextImpl implements ModelContext {
private final boolean asyncApplyBucketDiff;
private final boolean ignoreThreadStackSizes;
private final boolean unorderedMergeChaining;
+ private final boolean useV8GeoPositions;
public FeatureFlags(FlagSource source, ApplicationId appId) {
this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -238,6 +239,8 @@ public class ModelContextImpl implements ModelContext {
this.asyncApplyBucketDiff = flagValue(source, appId, Flags.ASYNC_APPLY_BUCKET_DIFF);
this.ignoreThreadStackSizes = flagValue(source, appId, Flags.IGNORE_THREAD_STACK_SIZES);
this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING);
+ this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS);
+
}
@Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
@@ -278,6 +281,7 @@ public class ModelContextImpl implements ModelContext {
@Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; }
@Override public boolean ignoreThreadStackSizes() { return ignoreThreadStackSizes; }
@Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; }
+ @Override public boolean useV8GeoPositions() { return useV8GeoPositions; }
private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
return flag.bindTo(source)
diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java
index 779a5f65673..aedbb3afb69 100644
--- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java
+++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java
@@ -100,6 +100,8 @@ class HttpRequestDispatch {
if (t != null) requestCompletion.completeExceptionally(t);
else requestCompletion.complete(null);
});
+ // Start the reader after wiring of "finished futures" are complete
+ servletRequestReader.start();
}
ContentChannel dispatchFilterRequest(Response response) {
@@ -217,11 +219,7 @@ class HttpRequestDispatch {
HttpRequestFactory.copyHeaders(jettyRequest, jdiscRequest);
requestContentChannel = requestHandler.handleRequest(jdiscRequest, servletResponseController.responseHandler());
}
- //TODO If the below method throws servletRequestReader will not complete and
- // requestContentChannel will not be closed and there is a reference leak
- // Ditto for the servletInputStream
- return new ServletRequestReader(
- jettyRequest.getInputStream(), requestContentChannel, jDiscContext.janitor, metricReporter);
+ return new ServletRequestReader(jettyRequest, requestContentChannel, jDiscContext.janitor, metricReporter);
}
private static RequestHandler newRequestHandler(JDiscContext context,
diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java
index 1def9ccaab1..43050a53f58 100644
--- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java
+++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java
@@ -6,6 +6,7 @@ import com.yahoo.jdisc.handler.ContentChannel;
import javax.servlet.ReadListener;
import javax.servlet.ServletInputStream;
+import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Objects;
@@ -33,7 +34,7 @@ import java.util.logging.Logger;
class ServletRequestReader {
private enum State {
- READING, ALL_DATA_READ, REQUEST_CONTENT_CLOSED
+ NOT_STARTED, READING, ALL_DATA_READ, REQUEST_CONTENT_CLOSED
}
private static final Logger log = Logger.getLogger(ServletRequestReader.class.getName());
@@ -42,11 +43,12 @@ class ServletRequestReader {
private final Object monitor = new Object();
- private final ServletInputStream in;
+ private final HttpServletRequest req;
private final ContentChannel requestContentChannel;
private final Janitor janitor;
private final RequestMetricReporter metricReporter;
+ private ServletInputStream in;
private Throwable errorDuringRead;
private int bytesRead;
@@ -63,7 +65,7 @@ class ServletRequestReader {
* (i.e. when being called from user code, don't call back into user code.)
*/
// GuardedBy("monitor")
- private State state = State.READING;
+ private State state = State.NOT_STARTED;
/**
* Number of calls that we're waiting for from user code.
@@ -94,15 +96,31 @@ class ServletRequestReader {
private final CompletableFuture<Void> finishedFuture = new CompletableFuture<>();
ServletRequestReader(
- ServletInputStream in,
+ HttpServletRequest req,
ContentChannel requestContentChannel,
Janitor janitor,
RequestMetricReporter metricReporter) {
- this.in = Objects.requireNonNull(in);
+ this.req = Objects.requireNonNull(req);
this.requestContentChannel = Objects.requireNonNull(requestContentChannel);
this.janitor = Objects.requireNonNull(janitor);
this.metricReporter = Objects.requireNonNull(metricReporter);
- in.setReadListener(new Listener());
+ }
+
+ /** Register read listener to start reading request data */
+ void start() {
+ try {
+ ServletInputStream in;
+ synchronized (monitor) {
+ if (state != State.NOT_STARTED) throw new IllegalStateException("State=" + state);
+ in = req.getInputStream(); // may throw
+ this.in = in;
+ state = State.READING;
+ }
+ // Not holding monitor in case listener is invoked from this thread
+ in.setReadListener(new Listener()); // may throw
+ } catch (Throwable t) {
+ fail(t);
+ }
}
CompletableFuture<Void> finishedFuture() { return finishedFuture; }
@@ -111,6 +129,8 @@ class ServletRequestReader {
@Override
public void onDataAvailable() throws IOException {
+ ServletInputStream in;
+ synchronized (monitor) { in = ServletRequestReader.this.in; }
while (in.isReady()) {
final byte[] buffer = new byte[BUFFER_SIZE_BYTES];
int numBytesRead;
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java
new file mode 100644
index 00000000000..1784aa77966
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveCharStream.java
@@ -0,0 +1,77 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.search.yql;
+
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.CharStream;
+
+/**
+ * This class supports case-insensitive lexing by wrapping an existing
+ * {@link CharStream} and forcing the lexer to see only lowercase characters.
+ * Grammar literals should then be only lower case such as 'begin'. The text of the character
+ * stream is unaffected. Example: input 'BeGiN' would match lexer rule
+ * 'begin', but getText() would return 'BeGiN'.
+ * It is based on https://github.com/antlr/antlr4/blob/master/doc/resources/CaseChangingCharStream.java
+ */
+class CaseInsensitiveCharStream implements CharStream {
+
+ final CharStream stream;
+
+ /**
+ * Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing
+ * all characters lower case.
+ * @param stream The stream to wrap.
+ */
+ CaseInsensitiveCharStream(CharStream stream) {
+ this.stream = stream;
+ }
+
+ @Override
+ public String getText(Interval interval) {
+ return stream.getText(interval);
+ }
+
+ @Override
+ public void consume() {
+ stream.consume();
+ }
+
+ @Override
+ public int LA(int i) {
+ int c = stream.LA(i);
+ if (c <= 0) {
+ return c;
+ }
+ return Character.toLowerCase(c);
+ }
+
+ @Override
+ public int mark() {
+ return stream.mark();
+ }
+
+ @Override
+ public void release(int marker) {
+ stream.release(marker);
+ }
+
+ @Override
+ public int index() {
+ return stream.index();
+ }
+
+ @Override
+ public void seek(int index) {
+ stream.seek(index);
+ }
+
+ @Override
+ public int size() {
+ return stream.size();
+ }
+
+ @Override
+ public String getSourceName() {
+ return stream.getSourceName();
+ }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java
deleted file mode 100644
index 5e21ce234d8..00000000000
--- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.yql;
-
-import org.antlr.v4.runtime.ANTLRFileStream;
-import org.antlr.v4.runtime.CharStream;
-
-import java.io.IOException;
-
-/**
- * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token.
- */
-
-class CaseInsensitiveFileStream extends ANTLRFileStream {
-
- public CaseInsensitiveFileStream(String fileName) throws IOException {
- super(fileName);
- }
-
- public CaseInsensitiveFileStream(String fileName, String encoding) throws IOException {
- super(fileName, encoding);
- }
-
- @Override
- public int LA(int i) {
- if (i == 0) {
- return 0;
- }
- if (i < 0) {
- i++; // e.g., translate LA(-1) to use offset 0
- }
-
- if ((p + i - 1) >= n) {
- return CharStream.EOF;
- }
- return Character.toLowerCase(data[p + i - 1]);
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java
deleted file mode 100644
index ed071370f4e..00000000000
--- a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.yql;
-
-import org.antlr.v4.runtime.ANTLRInputStream;
-import org.antlr.v4.runtime.CharStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token.
- */
-class CaseInsensitiveInputStream extends ANTLRInputStream {
-
- public CaseInsensitiveInputStream() {
- super();
- }
-
- public CaseInsensitiveInputStream(InputStream input) throws IOException {
- super(input);
- }
-
- public CaseInsensitiveInputStream(InputStream input, int size) throws IOException {
- super(input, size);
- }
-
- public CaseInsensitiveInputStream(char[] data, int numberOfActualCharsInArray) throws IOException {
- super(data, numberOfActualCharsInArray);
- }
-
- public CaseInsensitiveInputStream(String input) throws IOException {
- super(input);
- }
-
- @Override
- public int LA(int i) {
- if (i == 0) {
- return 0;
- }
- if (i < 0) {
- i++; // e.g., translate LA(-1) to use offset 0
- }
-
- if ((p + i - 1) >= n) {
- return CharStream.EOF;
- }
- return Character.toLowerCase(data[p + i - 1]);
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
index d32033249f1..c84ed0a0565 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
@@ -56,6 +56,7 @@ import com.yahoo.search.yql.yqlplusParser.TimeoutContext;
import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext;
import com.yahoo.search.yql.yqlplusParser.WhereContext;
+import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CommonTokenStream;
@@ -69,7 +70,6 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.RuleNode;
import org.antlr.v4.runtime.tree.TerminalNode;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
@@ -83,50 +83,39 @@ import java.util.Set;
final class ProgramParser {
public yqlplusParser prepareParser(String programName, InputStream input) throws IOException {
- return prepareParser(programName, new CaseInsensitiveInputStream(input));
+ //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading
+ return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input)));
}
public yqlplusParser prepareParser(String programName, String input) throws IOException {
- return prepareParser(programName, new CaseInsensitiveInputStream(input));
+ //TODO ANTLRInputStream goes away on 4.7, so must use CharStreams.fromXXX() when upgrading
+ return prepareParser(programName, new CaseInsensitiveCharStream(new ANTLRInputStream(input)));
}
- public yqlplusParser prepareParser(File file) throws IOException {
- return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath()));
+ private static class ErrorListener extends BaseErrorListener {
+ private final String programName;
+ ErrorListener(String programName) { this.programName = programName; }
+ @Override
+ public void syntaxError(Recognizer<?, ?> recognizer,
+ Object offendingSymbol,
+ int line,
+ int charPositionInLine,
+ String msg,
+ RecognitionException e) {
+ throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg);
+ }
}
private yqlplusParser prepareParser(String programName, CharStream input) {
+ ErrorListener errorListener = new ErrorListener(programName);
yqlplusLexer lexer = new yqlplusLexer(input);
lexer.removeErrorListeners();
- lexer.addErrorListener(new BaseErrorListener() {
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer,
- Object offendingSymbol,
- int line,
- int charPositionInLine,
- String msg,
- RecognitionException e) {
- throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg);
- }
-
- });
+ lexer.addErrorListener(errorListener);
TokenStream tokens = new CommonTokenStream(lexer);
yqlplusParser parser = new yqlplusParser(tokens);
parser.removeErrorListeners();
- parser.addErrorListener(new BaseErrorListener() {
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer,
- Object offendingSymbol,
- int line,
- int charPositionInLine,
- String msg,
- RecognitionException e) {
- throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg);
- }
-
- });
+ parser.addErrorListener(errorListener);
parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
return parser;
}
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index f9005387716..ab99118b179 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -4,15 +4,14 @@ package com.yahoo.search.yql;
import com.yahoo.component.chain.Chain;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.language.Language;
-import com.yahoo.language.simple.SimpleLinguistics;
import com.yahoo.prelude.Index;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.IndexModel;
import com.yahoo.prelude.SearchDefinition;
import com.yahoo.prelude.query.AndItem;
import com.yahoo.prelude.query.BoolItem;
-import com.yahoo.prelude.query.IndexedItem;
import com.yahoo.prelude.query.ExactStringItem;
+import com.yahoo.prelude.query.IndexedItem;
import com.yahoo.prelude.query.Item;
import com.yahoo.prelude.query.MarkerWordItem;
import com.yahoo.prelude.query.PhraseItem;
@@ -28,6 +27,7 @@ import com.yahoo.prelude.query.WeakAndItem;
import com.yahoo.prelude.query.WordAlternativesItem;
import com.yahoo.prelude.query.WordItem;
import com.yahoo.prelude.querytransform.QueryRewrite;
+import com.yahoo.processing.IllegalInputException;
import com.yahoo.search.Query;
import com.yahoo.search.Searcher;
import com.yahoo.search.config.IndexInfoConfig;
@@ -42,7 +42,6 @@ import com.yahoo.search.query.Sorting.Order;
import com.yahoo.search.query.Sorting.UcaSorter;
import com.yahoo.search.query.parser.Parsable;
import com.yahoo.search.query.parser.ParserEnvironment;
-
import com.yahoo.search.searchchain.Execution;
import org.junit.Test;
@@ -71,6 +70,13 @@ public class YqlParserTestCase {
private final YqlParser parser = new YqlParser(new ParserEnvironment());
@Test
+ public void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() {
+ assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10;",
+ new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 " +
+ "no viable alternative at input 'rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d''"));
+ }
+
+ @Test
public void testParserDefaults() {
assertTrue(parser.isQueryParser());
assertNull(parser.getDocTypes());
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java
index 55a7af45fd2..2f1b93158ab 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java
@@ -50,9 +50,10 @@ public class EndpointStatus {
}
/**
- * @return The epoch for when this status became active
+ * @return The epoch for when this status became active, in seconds
*/
public long getEpoch() {
return epoch;
}
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index f3368d0918b..9e7c614d4e8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -351,6 +351,7 @@ public class ApplicationController {
TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application());
ZoneId zone = job.type().zone(controller.system());
+ DeploymentId deployment = new DeploymentId(job.application(), zone);
try (Lock deploymentLock = lockForDeployment(job.application(), zone)) {
Set<ContainerEndpoint> containerEndpoints;
@@ -364,7 +365,7 @@ public class ApplicationController {
Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform());
ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication());
- ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(new DeploymentId(job.application(), zone), revision));
+ ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision));
try (Lock lock = lock(applicationId)) {
LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
@@ -376,8 +377,7 @@ public class ApplicationController {
applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get());
endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone, applicationPackage.deploymentSpec());
-
- containerEndpoints = controller.routing().containerEndpointsOf(application, job.application().instance(), zone);
+ containerEndpoints = controller.routing().of(deployment).prepare(application);
} // Release application lock while doing the deployment, which is a lengthy task.
@@ -391,7 +391,7 @@ public class ApplicationController {
// For direct deployments use the full deployment ID, but otherwise use just the tenant and application as
// the source since it's the same application, so it should have the same warnings
NotificationSource source = zone.environment().isManuallyDeployed() ?
- NotificationSource.from(new DeploymentId(job.application(), zone)) : NotificationSource.from(applicationId);
+ NotificationSource.from(deployment) : NotificationSource.from(applicationId);
List<String> warnings = Optional.ofNullable(result.prepareResponse().log)
.map(logs -> logs.stream()
.filter(log -> log.applicationPackage)
@@ -476,6 +476,7 @@ public class ApplicationController {
ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints,
Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
boolean dryRun) {
+ DeploymentId deployment = new DeploymentId(application, zone);
try {
Optional<DockerImage> dockerImageRepo = Optional.ofNullable(
dockerImageRepoFlag
@@ -490,7 +491,7 @@ public class ApplicationController {
.map(tenant -> ((AthenzTenant)tenant).domain());
if (zone.environment().isManuallyDeployed())
- controller.applications().applicationStore().putMeta(new DeploymentId(application, zone),
+ controller.applications().applicationStore().putMeta(deployment,
clock.instant(),
applicationPackage.metaDataZip());
@@ -502,9 +503,9 @@ public class ApplicationController {
.filter(tenant-> tenant instanceof CloudTenant)
.map(tenant -> ((CloudTenant) tenant).tenantSecretStores())
.orElse(List.of());
- List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(new DeploymentId(application, zone)).stream()
- .map(SupportAccessGrant::certificate)
- .collect(toList());
+ List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream()
+ .map(SupportAccessGrant::certificate)
+ .collect(toList());
ConfigServer.PreparedApplication preparedApplication =
configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform,
@@ -515,9 +516,10 @@ public class ApplicationController {
return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(),
applicationPackage.zippedContent().length);
} finally {
- // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that
- // any DNS updates can be propagated as early as possible.
- controller.routing().policies().refresh(application, applicationPackage.deploymentSpec(), zone);
+ // Even if prepare fails, routing configuration may need to be updated
+ if ( ! application.instance().isTester()) {
+ controller.routing().of(deployment).configure(applicationPackage.deploymentSpec());
+ }
}
}
@@ -701,7 +703,7 @@ public class ApplicationController {
try {
configServer.deactivate(id);
} finally {
- controller.routing().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone);
+ controller.routing().of(id).configure(application.get().deploymentSpec());
if (zone.environment().isManuallyDeployed())
applicationStore.putMetaTombstone(id, clock.instant());
if (!zone.environment().isTest())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
index ea2bcfcac4b..6e31c93dbdd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
@@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus;
import java.time.Instant;
import java.util.Collection;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 3794b69c023..4772dbeaab1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -18,7 +18,6 @@ import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.flags.BooleanFlag;
import com.yahoo.vespa.flags.FetchVector;
import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
@@ -31,10 +30,16 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
-import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
-import com.yahoo.vespa.hosted.controller.rotation.RotationRepository;
import com.yahoo.vespa.hosted.controller.routing.RoutingId;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.ExclusiveDeploymentRoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.SharedDeploymentRoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.context.ExclusiveRoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.context.SharedRoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository;
import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
import java.nio.charset.StandardCharsets;
@@ -44,7 +49,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -77,6 +81,25 @@ public class RoutingController {
this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource());
}
+ /** Create a routing context for given deployment */
+ public DeploymentRoutingContext of(DeploymentId deployment) {
+ if (usesSharedRouting(deployment.zoneId())) {
+ return new SharedDeploymentRoutingContext(deployment,
+ this,
+ controller.serviceRegistry().configServer(),
+ controller.clock());
+ }
+ return new ExclusiveDeploymentRoutingContext(deployment, this);
+ }
+
+ /** Create a routing context for given zone */
+ public RoutingContext of(ZoneId zone) {
+ if (usesSharedRouting(zone)) {
+ return new SharedRoutingContext(zone, controller.serviceRegistry().configServer());
+ }
+ return new ExclusiveRoutingContext(zone, routingPolicies);
+ }
+
public RoutingPolicies policies() {
return routingPolicies;
}
@@ -145,16 +168,17 @@ public class RoutingController {
.collect(Collectors.toMap(t -> new DeploymentId(application.id().instance(t.instance()),
ZoneId.from(Environment.prod, t.region())),
t -> t.weight()));
- List<RoutingMethod> availableRoutingMethods = routingMethodsOfAll(deployments.keySet(), deploymentSpec);
- for (var routingMethod : availableRoutingMethods) {
- endpoints.add(Endpoint.of(application.id())
- .targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
- ClusterSpec.Id.from(declaredEndpoint.containerId()),
- deployments)
- .routingMethod(routingMethod)
- .on(Port.fromRoutingMethod(routingMethod))
- .in(controller.system()));
- }
+ // An application endpoint can only target a single zone, so we just pick the zone of any deployment target
+ ZoneId zone = deployments.keySet().iterator().next().zoneId();
+ // Application endpoints are only supported when using direct routing methods
+ RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive;
+ endpoints.add(Endpoint.of(application.id())
+ .targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
+ ClusterSpec.Id.from(declaredEndpoint.containerId()),
+ deployments)
+ .routingMethod(routingMethod)
+ .on(Port.fromRoutingMethod(routingMethod))
+ .in(controller.system()));
}
return EndpointList.copyOf(endpoints);
}
@@ -216,43 +240,6 @@ public class RoutingController {
return Collections.unmodifiableList(endpointDnsNames);
}
- /** Change status of all global endpoints for given deployment */
- public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) {
- readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> {
- try {
- controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), status);
- } catch (Exception e) {
- throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e);
- }
- });
- }
-
- /** Get global endpoint status for given deployment */
- public Map<Endpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) {
- var routingEndpoints = new LinkedHashMap<Endpoint, EndpointStatus>();
- readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> {
- var upstreamName = endpoint.upstreamIdOf(deployment);
- var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, upstreamName);
- routingEndpoints.put(endpoint, status);
- });
- return Collections.unmodifiableMap(routingEndpoints);
- }
-
- /**
- * Assigns one or more global rotations to given application, if eligible. The given application is implicitly
- * stored, ensuring that the assigned rotation(s) are persisted when this returns.
- */
- private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) {
- try (RotationLock rotationLock = rotationRepository.lock()) {
- var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(),
- application.get().require(instanceName),
- rotationLock);
- application = application.with(instanceName, instance -> instance.with(rotations));
- controller.applications().store(application); // store assigned rotation even if deployment fails
- }
- return application;
- }
-
/** Returns the global and application-level endpoints for given deployment, as container endpoints */
public Set<ContainerEndpoint> containerEndpointsOf(LockedApplication application, InstanceName instanceName, ZoneId zone) {
// Assign rotations to application
@@ -354,6 +341,36 @@ public class RoutingController {
Priority.normal));
}
+ /** Returns direct routing endpoints if any exist and feature flag is set for given application */
+ // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant
+ public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) {
+ boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value();
+ EndpointList directEndpoints = endpoints.direct();
+ if (hideSharedEndpoint && !directEndpoints.isEmpty()) {
+ return directEndpoints;
+ }
+ return endpoints;
+ }
+
+ /**
+ * Assigns one or more global rotations to given application, if eligible. The given application is implicitly
+ * stored, ensuring that the assigned rotation(s) are persisted when this returns.
+ */
+ private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) {
+ try (RotationLock rotationLock = rotationRepository.lock()) {
+ var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(),
+ application.get().require(instanceName),
+ rotationLock);
+ application = application.with(instanceName, instance -> instance.with(rotations));
+ controller.applications().store(application); // store assigned rotation even if deployment fails
+ }
+ return application;
+ }
+
+ private boolean usesSharedRouting(ZoneId zone) {
+ return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared);
+ }
+
/** Returns the routing methods that are available across all given deployments */
private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) {
var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>();
@@ -437,23 +454,12 @@ public class RoutingController {
}
/** Create a common name based on a hash of given application. This must be less than 64 characters long. */
- private String commonNameHashOf(ApplicationId application, SystemName system) {
+ private static String commonNameHashOf(ApplicationId application, SystemName system) {
HashCode sha1 = Hashing.sha1().hashString(application.serializedForm(), StandardCharsets.UTF_8);
String base32 = BaseEncoding.base32().omitPadding().lowerCase().encode(sha1.asBytes());
return 'v' + base32 + Endpoint.internalDnsSuffix(system);
}
- /** Returns direct routing endpoints if any exist and feature flag is set for given application */
- // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant
- public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) {
- boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value();
- EndpointList directEndpoints = endpoints.direct();
- if (hideSharedEndpoint && !directEndpoints.isEmpty()) {
- return directEndpoints;
- }
- return endpoints;
- }
-
private static String asString(Endpoint.Scope scope) {
switch (scope) {
case application: return "application";
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java
index 1596456b7cc..ab9304e75f3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java
@@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.application;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.RegionName;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
import java.util.Collection;
import java.util.Objects;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index c736863a57e..aee7c1052be 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -60,7 +60,7 @@ public class Endpoint {
this.instance = requireInstance(instanceName, scope);
this.url = url;
this.targets = List.copyOf(requireTargets(targets, application, instanceName, scope, certificateName));
- this.scope = scope;
+ this.scope = requireScope(scope, routingMethod);
this.legacy = legacy;
this.routingMethod = routingMethod;
this.tls = port.tls;
@@ -329,6 +329,11 @@ public class Endpoint {
return instanceName;
}
+ private static Scope requireScope(Scope scope, RoutingMethod routingMethod) {
+ if (scope == Scope.application && !routingMethod.isDirect()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not support " + scope + "-scoped endpoints");
+ return scope;
+ }
+
private static List<Target> requireTargets(List<Target> targets, TenantAndApplicationId application, Optional<InstanceName> instanceName, Scope scope, boolean certificateName) {
if (!certificateName && targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints");
if (scope == Scope.zone && targets.size() != 1) throw new IllegalArgumentException("Exactly one target must be given for " + scope + " endpoints");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 4fcd6b10efa..9789bbd4da2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -41,15 +41,16 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails;
import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail;
import com.yahoo.vespa.hosted.controller.application.ActivateResult;
-import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.config.ControllerConfig;
import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
import com.yahoo.vespa.hosted.controller.notification.Notification;
import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
-import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
import com.yahoo.yolean.Exceptions;
import javax.security.auth.x500.X500Principal;
@@ -477,12 +478,12 @@ public class InternalStepRunner implements StepRunner {
}
private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) {
- var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zone)));
+ DeploymentId deployment = new DeploymentId(id, zone);
+ Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readZoneEndpointsOf(Set.of(deployment));
if ( ! endpoints.containsKey(zone)) {
logger.log("Endpoints not yet ready.");
return false;
}
- var policies = controller.routing().policies().get(new DeploymentId(id, zone));
for (var endpoint : endpoints.get(zone)) {
HostName endpointName = HostName.from(endpoint.dnsName());
var ipAddress = controller.jobController().cloud().resolveHostName(endpointName);
@@ -490,10 +491,10 @@ public class InternalStepRunner implements StepRunner {
logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'.");
return false;
}
- if (endpoint.routingMethod() == RoutingMethod.exclusive) {
- var policy = policies.get(new RoutingPolicyId(id, ClusterSpec.Id.from(endpoint.name()), zone));
- if (policy == null)
- throw new IllegalStateException(endpoint + " has no matching policy in " + policies);
+ DeploymentRoutingContext context = controller.routing().of(deployment);
+ if (context.routingMethod() == RoutingMethod.exclusive) {
+ RoutingPolicy policy = context.routingPolicy(ClusterSpec.Id.from(endpoint.name()))
+ .orElseThrow(() -> new IllegalStateException(endpoint + " has no matching policy"));
var cNameValue = controller.jobController().cloud().resolveCname(endpointName);
if ( ! cNameValue.map(policy.canonicalName()::equals).orElse(false)) {
@@ -848,11 +849,9 @@ public class InternalStepRunner implements StepRunner {
ZoneId zone = id.type().zone(controller.system());
boolean useTesterCertificate = useTesterCertificate(id);
- boolean useOsgiBasedTestRuntime = testerPlatformVersion(id).isAfter(new Version(7, 247, 11));
byte[] servicesXml = servicesXml( ! controller.system().isPublic(),
useTesterCertificate,
- useOsgiBasedTestRuntime,
testerResourcesFor(zone, spec.requireInstance(id.application().instance())),
controller.controllerConfig().steprunner().testerapp());
byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version);
@@ -904,9 +903,8 @@ public class InternalStepRunner implements StepRunner {
}
/** Returns the generated services.xml content for the tester application. */
- static byte[] servicesXml(
- boolean systemUsesAthenz, boolean useTesterCertificate, boolean useOsgiBasedTestRuntime,
- NodeResources resources, ControllerConfig.Steprunner.Testerapp config) {
+ static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate,
+ NodeResources resources, ControllerConfig.Steprunner.Testerapp config) {
int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?).
int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb());
@@ -920,24 +918,6 @@ public class InternalStepRunner implements StepRunner {
String runtimeProviderClass = config.runtimeProviderClass();
String tenantCdBundle = config.tenantCdBundle();
- String extraJUnitComponents =
- "\n" +
- " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" +
- "\n" +
- " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
- " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" +
- " <artifactsPath>artifacts</artifactsPath>\n" +
- " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
- " </config>\n" +
- " </component>\n" +
- "\n" +
- " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
- " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" +
- " <artifactsPath>artifacts</artifactsPath>\n" +
- " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
- " </config>\n" +
- " </component>\n";
-
String servicesXml =
"<?xml version='1.0' encoding='UTF-8'?>\n" +
"<services xmlns:deploy='vespa' version='1.0'>\n" +
@@ -955,7 +935,22 @@ public class InternalStepRunner implements StepRunner {
" <handler id=\"com.yahoo.vespa.testrunner.TestRunnerHandler\" bundle=\"vespa-osgi-testrunner\">\n" +
" <binding>http://*/tester/v1/*</binding>\n" +
" </handler>\n" +
- (useOsgiBasedTestRuntime ? extraJUnitComponents : "") +
+ "\n" +
+ " <component id=\"" + runtimeProviderClass + "\" bundle=\"" + tenantCdBundle + "\" />\n" +
+ "\n" +
+ " <component id=\"com.yahoo.vespa.testrunner.JunitRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
+ " <config name=\"com.yahoo.vespa.testrunner.junit-test-runner\">\n" +
+ " <artifactsPath>artifacts</artifactsPath>\n" +
+ " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+ " </config>\n" +
+ " </component>\n" +
+ "\n" +
+ " <component id=\"com.yahoo.vespa.testrunner.VespaCliTestRunner\" bundle=\"vespa-osgi-testrunner\">\n" +
+ " <config name=\"com.yahoo.vespa.testrunner.vespa-cli-test-runner\">\n" +
+ " <artifactsPath>artifacts</artifactsPath>\n" +
+ " <useAthenzCredentials>" + systemUsesAthenz + "</useAthenzCredentials>\n" +
+ " </config>\n" +
+ " </component>\n" +
"\n" +
" <nodes count=\"1\" allocated-memory=\"" + jdiscMemoryPct + "%\">\n" +
" " + resourceString + "\n" +
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 6ba8ad5bf36..ebda767d79e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -578,16 +578,8 @@ public class JobController {
});
}
- // TODO(mpolden): Eliminate duplication in this and ApplicationController#deactivate
public void deactivateTester(TesterId id, JobType type) {
- var zone = type.zone(controller.system());
- try {
- controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), zone));
- } finally {
- // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and
- // tester instances have none.
- controller.routing().policies().refresh(id.id(), DeploymentSpec.empty, zone);
- }
+ controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system())));
}
private void prunePackages(TenantAndApplicationId id) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 47df7a9da92..2939d10f99e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
import com.yahoo.vespa.hosted.controller.deployment.JobList;
-import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock;
import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java
index 1d5d444a32c..5acb21917eb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
@@ -25,7 +26,8 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer {
for (var zone : controller().zoneRegistry().zones().reachable().ids()) {
for (var application : SystemApplication.values()) {
if (!application.hasEndpoint()) continue;
- controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone);
+ DeploymentId deployment = new DeploymentId(application.id(), zone);
+ controller().routing().of(deployment).configure(DeploymentSpec.empty);
}
}
return 1.0;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index e8a7f7729fb..4b060846090 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -31,9 +31,9 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus;
import java.security.PublicKey;
import java.time.Instant;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 5cd5a70e4a4..1046737d860 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -44,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.Instance;
import com.yahoo.vespa.hosted.controller.LockedTenant;
import com.yahoo.vespa.hosted.controller.NotExistsException;
import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction;
@@ -97,10 +96,11 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer;
import com.yahoo.vespa.hosted.controller.notification.Notification;
import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
import com.yahoo.vespa.hosted.controller.security.AccessControlRequests;
import com.yahoo.vespa.hosted.controller.security.Credentials;
import com.yahoo.vespa.hosted.controller.support.access.SupportAccess;
@@ -1292,18 +1292,22 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
request.getUri()).toString());
}
}
- // Add dummy values for not-yet-existent prod deployments.
- status.jobSteps().keySet().stream()
- .filter(job -> job.application().instance().equals(instance.name()))
- .filter(job -> job.type().isProduction() && job.type().isDeployment())
+ // Add dummy values for not-yet-existent prod deployments, and running dev/perf deployments.
+ Stream.concat(status.jobSteps().keySet().stream()
+ .filter(job -> job.application().instance().equals(instance.name()))
+ .filter(job -> job.type().isProduction() && job.type().isDeployment()),
+ controller.jobController().active(instance.id()).stream()
+ .map(run -> run.id().job())
+ .filter(job -> job.type().environment().isManuallyDeployed()))
.map(job -> job.type().zone(controller.system()))
.filter(zone -> ! instance.deployments().containsKey(zone))
.forEach(zone -> {
- Cursor deploymentObject = instancesArray.addObject();
- deploymentObject.setString("environment", zone.environment().value());
- deploymentObject.setString("region", zone.region().value());
+ Cursor deploymentObject = instancesArray.addObject();
+ deploymentObject.setString("environment", zone.environment().value());
+ deploymentObject.setString("region", zone.region().value());
});
+
// TODO jonmv: Remove when clients are updated
application.deployKeys().stream().findFirst().ifPresent(key -> object.setString("pemDeployKey", KeyUtils.toPem(key)));
@@ -1532,49 +1536,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
if (deployment == null) {
throw new NotExistsException(instance + " has no deployment in " + zone);
}
-
- // The order here matters because setGlobalRotationStatus involves an external request that may fail.
- // TODO(mpolden): Set only one of these when only one kind of global endpoints are supported per zone.
- var deploymentId = new DeploymentId(instance.id(), zone);
- setGlobalRotationStatus(deploymentId, inService, request);
- setGlobalEndpointStatus(deploymentId, inService, request);
-
+ DeploymentId deploymentId = new DeploymentId(instance.id(), zone);
+ RoutingStatus.Agent agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant;
+ RoutingStatus.Value status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out;
+ controller.routing().of(deploymentId).setRoutingStatus(status, agent);
return new MessageResponse(Text.format("Successfully set %s in %s %s service",
instance.id().toShortString(), zone, inService ? "in" : "out of"));
}
- /** Set the global endpoint status for given deployment. This only applies to global endpoints backed by a cloud service */
- private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) {
- var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant;
- var status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out;
- controller.routing().policies().setRoutingStatus(deployment, status, agent);
- }
-
- /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */
- private void setGlobalRotationStatus(DeploymentId deployment, boolean inService, HttpRequest request) {
- var requestData = toSlime(request.getData()).get();
- var reason = mandatory("reason", requestData).asString();
- var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant;
- long timestamp = controller.clock().instant().getEpochSecond();
- var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out;
- var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp);
- controller.routing().setGlobalRotationStatus(deployment, endpointStatus);
- }
-
private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) {
DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
requireZone(environment, region));
Slime slime = new Slime();
Cursor array = slime.setObject().setArray("globalrotationoverride");
- controller.routing().globalRotationStatus(deploymentId)
- .forEach((endpoint, status) -> {
- array.addString(endpoint.upstreamIdOf(deploymentId));
- Cursor statusObject = array.addObject();
- statusObject.setString("status", status.getStatus().name());
- statusObject.setString("reason", status.getReason() == null ? "" : status.getReason());
- statusObject.setString("agent", status.getAgent() == null ? "" : status.getAgent());
- statusObject.setLong("timestamp", status.getEpoch());
- });
+ Optional<Endpoint> primaryEndpoint = controller.routing().readDeclaredEndpointsOf(deploymentId.applicationId())
+ .requiresRotation()
+ .primary();
+ if (primaryEndpoint.isPresent()) {
+ DeploymentRoutingContext context = controller.routing().of(deploymentId);
+ RoutingStatus status = context.routingStatus();
+ array.addString(primaryEndpoint.get().upstreamIdOf(deploymentId));
+ Cursor statusObject = array.addObject();
+ statusObject.setString("status", status.value().name());
+ statusObject.setString("reason", "");
+ statusObject.setString("agent", status.agent().name());
+ statusObject.setLong("timestamp", status.changedAt().getEpochSecond());
+ }
return new SlimeJsonResponse(slime);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index 45abf7f2946..226a7ca9561 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -19,26 +19,26 @@ import com.yahoo.slime.Slime;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.role.Role;
import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition;
import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.EndpointList;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
+import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext;
import com.yahoo.yolean.Exceptions;
import java.net.URI;
-import java.time.Instant;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.logging.Level;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
/**
* This implements the /routing/v1 API, which provides operators and tenants routing control at both zone- (operator
@@ -112,11 +112,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
var deploymentsStatus = deployments.stream()
.collect(Collectors.toMap(
deploymentId -> deploymentId,
- deploymentId -> Stream.concat(
- directGlobalRoutingStatus(deploymentId).stream(),
- sharedGlobalRoutingStatus(deploymentId).stream()
- ).collect(Collectors.toList())
- ));
+ deploymentId -> controller.routing().of(deploymentId).routingStatus())
+ );
var slime = new Slime();
var root = slime.setObject();
@@ -125,11 +122,11 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
var endpointRoot = endpointsRoot.addObject();
endpointToSlime(endpointRoot, endpoint);
var zonesRoot = endpointRoot.setArray("zones");
- endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value())).forEach(deployment -> {
- deploymentsStatus.getOrDefault(deployment, List.of()).forEach(status -> {
- deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod());
- });
- });
+ endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value()))
+ .forEach(deployment -> {
+ RoutingStatus status = deploymentsStatus.get(deployment);
+ deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod());
+ });
});
return new SlimeJsonResponse(slime);
@@ -211,13 +208,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
}
private HttpResponse setZoneStatus(Path path, boolean in) {
- var zone = zoneFrom(path);
- if (exclusiveRoutingIn(zone)) {
- var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out;
- controller.routing().policies().setRoutingStatus(zone, status);
- } else {
- controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in);
- }
+ ZoneId zone = zoneFrom(path);
+ RoutingContext context = controller.routing().of(zone);
+ RoutingStatus.Value newStatus = in ? RoutingStatus.Value.in : RoutingStatus.Value.out;
+ context.setRoutingStatus(newStatus, RoutingStatus.Agent.operator);
return new MessageResponse("Set global routing status for deployments in " + zone + " to " +
(in ? "IN" : "OUT"));
}
@@ -231,16 +225,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
}
private void toSlime(ZoneId zone, Cursor zoneObject) {
- if (exclusiveRoutingIn(zone)) {
- var zonePolicy = controller.routing().policies().get(zone);
- zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.routingStatus(), RoutingMethod.exclusive);
- } else {
- // Rotation status per zone only exposes in/out status, no agent or time of change.
- var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone);
- var globalRouting = new RoutingStatus(in ? RoutingStatus.Value.in : RoutingStatus.Value.out,
- RoutingStatus.Agent.operator, Instant.EPOCH);
- zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared);
- }
+ RoutingContext context = controller.routing().of(zone);
+ zoneStatusToSlime(zoneObject, zone, context.routingStatus(), context.routingMethod());
}
private HttpResponse setDeploymentStatus(Path path, boolean in, HttpRequest request) {
@@ -249,18 +235,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out;
var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant;
requireDeployment(deployment, instance);
-
- if (sharedRoutingIn(deployment.zoneId())) {
- // Set rotation status
- var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out,
- "",
- agent.name(),
- controller.clock().instant().getEpochSecond());
- controller.routing().setGlobalRotationStatus(deployment, endpointStatus);
- } else {
- // Set policy status
- controller.routing().policies().setRoutingStatus(deployment, status, agent);
- }
+ controller.routing().of(deployment).setRoutingStatus(status, agent);
return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT"));
}
@@ -279,66 +254,24 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
var instances = instanceId == null
? application.instances().values()
: List.of(application.instances().get(instanceId.instance()));
+ EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application);
for (var instance : instances) {
var zones = zoneId == null
? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value))
.collect(Collectors.toList())
: List.of(zoneId);
for (var zone : zones) {
- var deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance);
- // Include status from rotation
- sharedGlobalRoutingStatus(deploymentId).ifPresent(status -> {
- deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.shared);
- });
-
- // Include status from routing policies
- directGlobalRoutingStatus(deploymentId).forEach(status -> {
- deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.exclusive);
- });
- }
- }
- }
-
- }
-
- private Optional<RoutingStatus> sharedGlobalRoutingStatus(DeploymentId deploymentId) {
- if (sharedRoutingIn(deploymentId.zoneId())) {
- var rotationStatus = controller.routing().globalRotationStatus(deploymentId);
- // Status is equal across all global endpoints, as the status is per deployment, not per endpoint.
- var endpointStatus = rotationStatus.values().stream().findFirst();
- if (endpointStatus.isPresent()) {
- var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch());
- RoutingStatus.Agent agent;
- try {
- agent = RoutingStatus.Agent.valueOf(endpointStatus.get().getAgent());
- } catch (IllegalArgumentException e) {
- agent = RoutingStatus.Agent.unknown;
+ DeploymentId deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance);
+ DeploymentRoutingContext context = controller.routing().of(deploymentId);
+ if (declaredEndpoints.targets(deploymentId).isEmpty()) continue; // No declared endpoints point to this deployment
+ deploymentStatusToSlime(deploymentsArray.addObject(),
+ deploymentId,
+ context.routingStatus(),
+ context.routingMethod());
}
- var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in
- ? RoutingStatus.Value.in
- : RoutingStatus.Value.out;
- return Optional.of(new RoutingStatus(status, agent, changedAt));
}
}
- return Optional.empty();
- }
-
- private List<RoutingStatus> directGlobalRoutingStatus(DeploymentId deploymentId) {
- return controller.routing().policies().get(deploymentId).values().stream()
- .filter(p -> ! p.instanceEndpoints().isEmpty()) // This policy does not apply to a global endpoint
- .filter(p -> exclusiveRoutingIn(p.id().zone()))
- .map(p -> p.status().routingStatus())
- .collect(Collectors.toList());
- }
-
- /** Returns whether given zone uses exclusive routing */
- private boolean exclusiveRoutingIn(ZoneId zone) {
- return controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive);
- }
- /** Returns whether given zone uses shared routing */
- private boolean sharedRoutingIn(ZoneId zone) {
- return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared);
}
private static void zoneStatusToSlime(Cursor object, ZoneId zone, RoutingStatus routingStatus, RoutingMethod method) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index 2a39ed08014..d2dc2771160 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -215,7 +215,7 @@ public class RoutingPolicies {
Application application = controller.applications().requireApplication(routingTable.keySet().iterator().next().application());
Map<DeploymentId, Map<EndpointId, Integer>> targetWeights = targetWeights(application);
- Map<String, Set<AliasTarget>> targetsByEndpoint = new LinkedHashMap<>();
+ Map<Endpoint, Set<AliasTarget>> targetsByEndpoint = new LinkedHashMap<>();
for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
RoutingId routingId = routeEntry.getKey();
EndpointList endpoints = controller.routing().declaredEndpointsOf(application)
@@ -230,19 +230,27 @@ public class RoutingPolicies {
for (var policy : routeEntry.getValue()) {
for (var target : endpoint.targets()) {
if (!policy.appliesTo(target.deployment())) continue;
+ if (policy.dnsZone().isEmpty()) continue; // Does not support ALIAS records
int weight = target.weight();
if (isConfiguredOut(policy, inactiveZones) && removableFromApplicationEndpoint(policy, application, targetWeights)) {
weight = 0;
}
WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(),
target.deployment().zoneId(), weight);
- targetsByEndpoint.computeIfAbsent(endpoint.dnsName(), (k) -> new LinkedHashSet<>())
+ targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>())
.add(weightedAliasTarget);
}
}
}
targetsByEndpoint.forEach((applicationEndpoint, targets) -> {
- controller.nameServiceForwarder().createAlias(RecordName.from(applicationEndpoint), targets, Priority.normal);
+ ZoneId targetZone = applicationEndpoint.targets().stream()
+ .map(Endpoint.Target::deployment)
+ .map(DeploymentId::zoneId)
+ .findFirst()
+ .get();
+ nameServiceForwarderIn(targetZone).createAlias(RecordName.from(applicationEndpoint.dnsName()),
+ targets,
+ Priority.normal);
});
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
new file mode 100644
index 00000000000..28fbeee28f5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
@@ -0,0 +1,154 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing.context;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.vespa.hosted.controller.LockedApplication;
+import com.yahoo.vespa.hosted.controller.RoutingController;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment.
+ *
+ * @author mpolden
+ */
+public abstract class DeploymentRoutingContext implements RoutingContext {
+
+ final DeploymentId deployment;
+ final RoutingController controller;
+ final RoutingMethod method;
+
+ public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController controller) {
+ this.deployment = Objects.requireNonNull(deployment);
+ this.controller = Objects.requireNonNull(controller);
+ this.method = Objects.requireNonNull(method);
+ }
+
+ /**
+ * Prepare routing configuration for the deployment in this context
+ *
+ * @return the container endpoints relevant for this deployment, as declared in deployment spec
+ */
+ public final Set<ContainerEndpoint> prepare(LockedApplication application) {
+ return controller.containerEndpointsOf(application, deployment.applicationId().instance(), deployment.zoneId());
+ }
+
+ /** Configure routing for the deployment in this context, using given deployment spec */
+ public final void configure(DeploymentSpec deploymentSpec) {
+ controller.policies().refresh(deployment.applicationId(), deploymentSpec, deployment.zoneId());
+ }
+
+ /** Routing method of this context */
+ public final RoutingMethod routingMethod() {
+ return method;
+ }
+
+ /** Read the routing policy for given cluster in this deployment */
+ public final Optional<RoutingPolicy> routingPolicy(ClusterSpec.Id cluster) {
+ RoutingPolicyId id = new RoutingPolicyId(deployment.applicationId(), cluster, deployment.zoneId());
+ return Optional.ofNullable(controller.policies().get(deployment).get(id));
+ }
+
+ /**
+ * Extension of a {@link DeploymentRoutingContext} for deployments using either {@link RoutingMethod#shared} or
+ * {@link RoutingMethod#sharedLayer4} routing.
+ */
+ public static class SharedDeploymentRoutingContext extends DeploymentRoutingContext {
+
+ private final Clock clock;
+ private final ConfigServer configServer;
+
+ public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) {
+ super(deployment, RoutingMethod.shared, controller);
+ this.clock = Objects.requireNonNull(clock);
+ this.configServer = Objects.requireNonNull(configServer);
+ }
+
+ @Override
+ public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
+ EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in
+ ? EndpointStatus.Status.in
+ : EndpointStatus.Status.out,
+ "",
+ agent.name(),
+ clock.instant().getEpochSecond());
+ primaryEndpoint().ifPresent(endpoint -> {
+ try {
+ configServer.setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), newStatus);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e);
+ }
+ });
+ }
+
+ @Override
+ public RoutingStatus routingStatus() {
+ Optional<EndpointStatus> status = primaryEndpoint().map(endpoint -> {
+ var upstreamName = endpoint.upstreamIdOf(deployment);
+ return configServer.getGlobalRotationStatus(deployment, upstreamName);
+ });
+ if (status.isEmpty()) return RoutingStatus.DEFAULT;
+ RoutingStatus.Agent agent;
+ try {
+ agent = RoutingStatus.Agent.valueOf(status.get().getAgent().toLowerCase());
+ } catch (IllegalArgumentException e) {
+ agent = RoutingStatus.Agent.unknown;
+ }
+ return new RoutingStatus(status.get().getStatus() == EndpointStatus.Status.in
+ ? RoutingStatus.Value.in
+ : RoutingStatus.Value.out,
+ agent,
+ Instant.ofEpochSecond(status.get().getEpoch()));
+ }
+
+ private Optional<Endpoint> primaryEndpoint() {
+ return controller.readDeclaredEndpointsOf(deployment.applicationId())
+ .requiresRotation()
+ .primary();
+ }
+
+ }
+
+ /**
+ * Implementation of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#exclusive}
+ * routing.
+ */
+ public static class ExclusiveDeploymentRoutingContext extends DeploymentRoutingContext {
+
+ public ExclusiveDeploymentRoutingContext(DeploymentId deployment, RoutingController controller) {
+ super(deployment, RoutingMethod.exclusive, controller);
+ }
+
+ @Override
+ public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
+ controller.policies().setRoutingStatus(deployment, value, agent);
+ }
+
+ @Override
+ public RoutingStatus routingStatus() {
+ // Status for a deployment applies to all clusters within the deployment, so we use the status from the
+ // first matching policy here
+ return controller.policies().get(deployment).values().stream()
+ .findFirst()
+ .map(RoutingPolicy::status)
+ .map(RoutingPolicy.Status::routingStatus)
+ .orElse(RoutingStatus.DEFAULT);
+ }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java
new file mode 100644
index 00000000000..e949c45f2fd
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java
@@ -0,0 +1,41 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing.context;
+
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+
+import java.util.Objects;
+
+/**
+ * An implementation of {@link RoutingContext} for a zone using {@link RoutingMethod#exclusive} routing.
+ *
+ * @author mpolden
+ */
+public class ExclusiveRoutingContext implements RoutingContext {
+
+ private final RoutingPolicies policies;
+ private final ZoneId zone;
+
+ public ExclusiveRoutingContext(ZoneId zone, RoutingPolicies policies) {
+ this.policies = Objects.requireNonNull(policies);
+ this.zone = Objects.requireNonNull(zone);
+ }
+
+ @Override
+ public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
+ policies.setRoutingStatus(zone, value);
+ }
+
+ @Override
+ public RoutingStatus routingStatus() {
+ return policies.get(zone).routingStatus();
+ }
+
+ @Override
+ public RoutingMethod routingMethod() {
+ return RoutingMethod.exclusive;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java
new file mode 100644
index 00000000000..6f43416b9b5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java
@@ -0,0 +1,23 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing.context;
+
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+
+/**
+ * Top-level interface for a routing context, which provides control of routing status for a deployment or zone.
+ *
+ * @author mpolden
+ */
+public interface RoutingContext {
+
+ /** Change the routing status for the zone or deployment represented by this context */
+ void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent);
+
+ /** Get the current routing status for the zone or deployment represented by this context */
+ RoutingStatus routingStatus();
+
+ /** Routing method used in this context */
+ RoutingMethod routingMethod();
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java
new file mode 100644
index 00000000000..e38212d7f80
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java
@@ -0,0 +1,48 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing.context;
+
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * An implementation of {@link RoutingContext} for a zone, using either {@link RoutingMethod#shared} or
+ * {@link RoutingMethod#sharedLayer4} routing.
+ *
+ * @author mpolden
+ */
+public class SharedRoutingContext implements RoutingContext {
+
+ private final ConfigServer configServer;
+ private final ZoneId zone;
+
+ public SharedRoutingContext(ZoneId zone, ConfigServer configServer) {
+ this.configServer = Objects.requireNonNull(configServer);
+ this.zone = Objects.requireNonNull(zone);
+ }
+
+ @Override
+ public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
+ boolean in = value == RoutingStatus.Value.in;
+ configServer.setGlobalRotationStatus(zone, in);
+ }
+
+ @Override
+ public RoutingStatus routingStatus() {
+ boolean in = configServer.getGlobalRotationStatus(zone);
+ RoutingStatus.Value newValue = in ? RoutingStatus.Value.in : RoutingStatus.Value.out;
+ return new RoutingStatus(newValue,
+ RoutingStatus.Agent.operator,
+ Instant.EPOCH); // API does not support time of change
+ }
+
+ @Override
+ public RoutingMethod routingMethod() {
+ return RoutingMethod.shared;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java
index ca5d2d5915f..0cf7101cac0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
import com.yahoo.text.Text;
import java.util.Objects;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java
index 2b75777fbbd..4d97962a40a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
import java.util.Objects;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java
index fe9280b1193..36a43f80e9a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
import com.yahoo.vespa.curator.Lock;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java
index 5b24f39717b..961fdc6dd9c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java
index 032f01433b3..19e816a0b51 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
/**
* The possible states of a global rotation.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java
index 1ddbd640e53..6d95ad9a230 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.Deployment;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 9472801ef2c..30cdd1b8466 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -19,7 +19,6 @@ import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.path.Path;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
@@ -38,8 +37,10 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
-import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
import org.junit.Test;
@@ -214,22 +215,18 @@ public class ControllerTest {
// Check initial rotation status
var deployment1 = context.deploymentIdIn(zone1);
- var status1 = tester.controller().routing().globalRotationStatus(deployment1);
- assertEquals(1, status1.size());
- assertTrue("All upstreams are in", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in));
+ DeploymentRoutingContext routingContext = tester.controller().routing().of(deployment1);
+ RoutingStatus status1 = routingContext.routingStatus();
+ assertEquals(RoutingStatus.Value.in, status1.value());
// Set the deployment out of service in the global rotation
- var newStatus = new EndpointStatus(EndpointStatus.Status.out, "unit-test", ControllerTest.class.getSimpleName(), tester.clock().instant().getEpochSecond());
- tester.controller().routing().setGlobalRotationStatus(deployment1, newStatus);
- status1 = tester.controller().routing().globalRotationStatus(deployment1);
- assertEquals(1, status1.size());
- assertTrue("All upstreams are out", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out));
- assertTrue("Reason is set", status1.values().stream().allMatch(es -> es.getReason().equals("unit-test")));
+ routingContext.setRoutingStatus(RoutingStatus.Value.out, RoutingStatus.Agent.operator);
+ RoutingStatus status2 = routingContext.routingStatus();
+ assertEquals(RoutingStatus.Value.out, status2.value());
// Other deployment remains in
- var status2 = tester.controller().routing().globalRotationStatus(context.deploymentIdIn(zone2));
- assertEquals(1, status2.size());
- assertTrue("All upstreams are in", status2.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in));
+ RoutingStatus status3 = tester.controller().routing().of(context.deploymentIdIn(zone2)).routingStatus();
+ assertEquals(RoutingStatus.Value.in, status3.value());
}
@Test
@@ -625,13 +622,13 @@ public class ControllerTest {
.instances("beta,main")
.region("us-west-1")
.region("us-east-3")
- .applicationEndpoint("a", "qrs", "us-west-1",
+ .applicationEndpoint("a", "default", "us-west-1",
Map.of(InstanceName.from("beta"), 2,
InstanceName.from("main"), 8))
- .applicationEndpoint("b", "qrs", "us-west-1",
+ .applicationEndpoint("b", "default", "us-west-1",
Map.of(InstanceName.from("beta"), 1,
InstanceName.from("main"), 1))
- .applicationEndpoint("c", "qrs", "us-east-3",
+ .applicationEndpoint("c", "default", "us-east-3",
Map.of(InstanceName.from("beta"), 4,
InstanceName.from("main"), 6))
.build();
@@ -640,11 +637,11 @@ public class ControllerTest {
// Endpoint names are passed to each deployment
DeploymentId usWest = context.deploymentIdIn(ZoneId.from("prod", "us-west-1"));
DeploymentId usEast = context.deploymentIdIn(ZoneId.from("prod", "us-east-3"));
- Map<DeploymentId, List<String>> deploymentEndpoints = Map.of(usWest, List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", "b--app1--tenant1.us-west-1-r.vespa.oath.cloud"),
- usEast, List.of("c--app1--tenant1.us-east-3-r.vespa.oath.cloud"));
+ Map<DeploymentId, List<String>> deploymentEndpoints = Map.of(usWest, List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", "b.app1.tenant1.us-west-1-r.vespa.oath.cloud"),
+ usEast, List.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud"));
deploymentEndpoints.forEach((zone, endpointNames) -> {
assertEquals("Endpoint names are passed to config server in " + zone,
- Set.of(new ContainerEndpoint("qrs", "application",
+ Set.of(new ContainerEndpoint("default", "application",
endpointNames)),
tester.configServer().containerEndpoints().get(zone));
});
@@ -653,21 +650,21 @@ public class ControllerTest {
// DNS records are created for each endpoint
Set<Record> records = tester.controllerTester().nameService().records();
assertEquals(Set.of(new Record(Record.Type.CNAME,
- RecordName.from("a--app1--tenant1.us-west-1-r.vespa.oath.cloud"),
+ RecordName.from("a.app1.tenant1.us-west-1-r.vespa.oath.cloud"),
RecordData.from("vip.prod.us-west-1.")),
new Record(Record.Type.CNAME,
- RecordName.from("b--app1--tenant1.us-west-1-r.vespa.oath.cloud"),
+ RecordName.from("b.app1.tenant1.us-west-1-r.vespa.oath.cloud"),
RecordData.from("vip.prod.us-west-1.")),
new Record(Record.Type.CNAME,
- RecordName.from("c--app1--tenant1.us-east-3-r.vespa.oath.cloud"),
+ RecordName.from("c.app1.tenant1.us-east-3-r.vespa.oath.cloud"),
RecordData.from("vip.prod.us-east-3."))),
records);
List<String> endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application())
.scope(Endpoint.Scope.application)
.mapToList(Endpoint::dnsName);
- assertEquals(List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud",
- "b--app1--tenant1.us-west-1-r.vespa.oath.cloud",
- "c--app1--tenant1.us-east-3-r.vespa.oath.cloud"),
+ assertEquals(List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud",
+ "b.app1.tenant1.us-west-1-r.vespa.oath.cloud",
+ "c.app1.tenant1.us-east-3-r.vespa.oath.cloud"),
endpointDnsNames);
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 11086ff7663..5cf554f2c01 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -521,22 +521,16 @@ public class InternalStepRunnerTest {
}
@Test
- public void generates_correct_services_xml_using_osgi_based_runtime() {
- generates_correct_services_xml("test_runner_services.xml-cd-osgi", true);
+ public void generates_correct_services_xml() {
+ generates_correct_services_xml("test_runner_services.xml-cd");
}
- @Test
- public void generates_correct_services_xml_using_legacy_runtime() {
- generates_correct_services_xml("test_runner_services.xml-cd-legacy", false);
- }
-
- private void generates_correct_services_xml(String filenameExpectedOutput, boolean useOsgiBasedRuntime) {
+ private void generates_correct_services_xml(String filenameExpectedOutput) {
ControllerConfig.Steprunner.Testerapp config = new ControllerConfig.Steprunner.Testerapp.Builder().build();
assertFile(filenameExpectedOutput,
new String(InternalStepRunner.servicesXml(
true,
false,
- useOsgiBasedRuntime,
new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local),
config)));
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index f1421b5affd..b33f8f6f7e7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -23,9 +23,9 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus;
import org.junit.Test;
import java.nio.file.Files;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index ae6232ae419..afd67824ee8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -69,6 +69,7 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
+import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
import com.yahoo.vespa.hosted.controller.security.AthenzCredentials;
import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec;
import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant;
@@ -1868,13 +1869,12 @@ public class ApplicationApiTest extends ControllerContainerTest {
}
private void assertGlobalRouting(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) {
- var changedAt = tester.controller().clock().instant();
- var westPolicies = tester.controller().routing().policies().get(deployment);
- assertEquals(1, westPolicies.size());
- var westPolicy = westPolicies.values().iterator().next();
- assertEquals(value, westPolicy.status().routingStatus().value());
- assertEquals(agent, westPolicy.status().routingStatus().agent());
- assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().routingStatus().changedAt());
+ Instant changedAt = tester.controller().clock().instant();
+ DeploymentRoutingContext context = tester.controller().routing().of(deployment);
+ RoutingStatus status = context.routingStatus();
+ assertEquals(value, status.value());
+ assertEquals(agent, status.agent());
+ assertEquals(changedAt.truncatedTo(ChronoUnit.SECONDS), status.changedAt());
}
private static class RequestBuilder implements Supplier<Request> {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
index fb6088f54b8..ab2a3bf945c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
@@ -24,9 +24,9 @@
{
"cluster": "foo",
"tls": true,
- "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/",
+ "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/",
"scope": "application",
- "routingMethod": "shared",
+ "routingMethod": "sharedLayer4",
"legacy": false
}
],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
index 934e0cf43b9..de2266fd197 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
@@ -4,7 +4,7 @@
{
"status": "in",
"reason": "",
- "agent": "",
+ "agent": "unknown",
"timestamp": 1497618757
}
]
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
index 409e97b063c..62ad3a2db7e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
@@ -27,9 +27,9 @@
{
"cluster": "foo",
"tls": true,
- "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/",
+ "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/",
"scope": "application",
- "routingMethod": "shared",
+ "routingMethod": "sharedLayer4",
"legacy": false
}
],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java
index e7c2eacbd02..9a3ac8b547d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.rotation;
+package com.yahoo.vespa.hosted.controller.routing.rotation;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi b/controller-server/src/test/resources/test_runner_services.xml-cd
index 634137e3fb6..634137e3fb6 100644
--- a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi
+++ b/controller-server/src/test/resources/test_runner_services.xml-cd
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-legacy b/controller-server/src/test/resources/test_runner_services.xml-cd-legacy
deleted file mode 100644
index c6046479934..00000000000
--- a/controller-server/src/test/resources/test_runner_services.xml-cd-legacy
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<services xmlns:deploy='vespa' version='1.0'>
- <container version='1.0' id='tester'>
-
- <component id="com.yahoo.vespa.hosted.testrunner.TestRunner" bundle="vespa-testrunner-components">
- <config name="com.yahoo.vespa.hosted.testrunner.test-runner">
- <artifactsPath>artifacts</artifactsPath>
- <surefireMemoryMb>5120</surefireMemoryMb>
- <useAthenzCredentials>true</useAthenzCredentials>
- <useTesterCertificate>false</useTesterCertificate>
- </config>
- </component>
-
- <handler id="com.yahoo.vespa.testrunner.TestRunnerHandler" bundle="vespa-osgi-testrunner">
- <binding>http://*/tester/v1/*</binding>
- </handler>
-
- <nodes count="1" allocated-memory="17%">
- <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/>
- </nodes>
- </container>
-</services>
diff --git a/default_build_settings.cmake b/default_build_settings.cmake
index e482439dd7d..b0dfed2bfd5 100644
--- a/default_build_settings.cmake
+++ b/default_build_settings.cmake
@@ -34,7 +34,7 @@ function(setup_vespa_default_build_settings_centos_8)
if (VESPA_OS_DISTRO_NAME STREQUAL "CentOS Stream")
set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE)
else()
- set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE)
+ set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE)
endif()
endfunction()
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 031213ac693..bfb4c14bdeb 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -101,7 +101,7 @@ BuildRequires: libarchive
%if 0%{?_centos_stream}
BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13)
%else
-BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12)
+BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13)
%endif
%else
BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11)
@@ -246,7 +246,7 @@ Requires: vespa-gtest = 1.11.0
%if 0%{?_centos_stream}
%define _vespa_llvm_version 12
%else
-%define _vespa_llvm_version 11
+%define _vespa_llvm_version 12
%endif
%else
%define _vespa_llvm_version 10
@@ -375,7 +375,7 @@ Requires: openssl-libs
%if 0%{?_centos_stream}
Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13)
%else
-Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12)
+Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13)
%endif
%else
Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 6260ce07c3f..685d2a351c5 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -384,6 +384,20 @@ public class Flags {
"Takes effect on config server restart",
ZONE_ID);
+ public static final UnboundBooleanFlag USE_V8_GEO_POSITIONS = defineFeatureFlag(
+ "use-v8-geo-positions", false,
+ List.of("arnej"), "2021-11-15", "2022-12-31",
+ "Use Vespa 8 types and formats for geographical positions",
+ "Takes effect at redeployment",
+ ZONE_ID, APPLICATION_ID);
+
+ public static final UnboundBooleanFlag USE_LEGACY_LB_SERVICES = defineFeatureFlag(
+ "use-legacy-lb-services", true,
+ List.of("tokle"), "2021-11-22", "2021-12-31",
+ "Whether to generate routing table based on legacy lb-services config",
+ "Takes effect on container reboot",
+ ZONE_ID, HOSTNAME);
+
/** WARNING: public for testing: All flags should be defined in {@link Flags}. */
public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
String createdAt, String expiresAt, String description,
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
index 593c5e5f05b..34547d14616 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
@@ -114,7 +114,7 @@ public class PermanentFlags {
ZONE_ID, APPLICATION_ID);
public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag(
- "zookeeper-server-version", "3.6.3",
+ "zookeeper-server-version", "3.7.0",
"ZooKeeper server version, a jar file zookeeper-server-<ZOOKEEPER_SERVER_VERSION>-jar-with-dependencies.jar must exist",
"Takes effect on restart of Docker container",
NODE_TYPE, APPLICATION_ID, HOSTNAME);
diff --git a/sd-plugin/.gitignore b/integration/intellij/.gitignore
index fa33885331d..96a25be6725 100644
--- a/sd-plugin/.gitignore
+++ b/integration/intellij/.gitignore
@@ -12,4 +12,9 @@ gradle-app.setting
!gradle/wrapper/gradle-wrapper.jar
# Cache of project
-.gradletasknamecache \ No newline at end of file
+.gradletasknamecache
+
+# These are not needed, but IntelliJ automatically adds them
+gradlew
+gradlew.bat
+gradle/
diff --git a/integration/intellij/BACKLOG.md b/integration/intellij/BACKLOG.md
new file mode 100644
index 00000000000..aaef8ddd411
--- /dev/null
+++ b/integration/intellij/BACKLOG.md
@@ -0,0 +1,41 @@
+### Open Issues
+
+In some cases, the parser stops on bad syntax and can't build the PSI tree.
+That means no features will work in the file.
+
+To enable the grammar recognize some keywords as identifiers (e.g. "filter" as a field's name),
+the identifier rule (named "IdentifierVal") wraps the regex (ID_REG) and the KeywordOrIdentifier
+rule (which contains all the keywords in the language).
+
+The implementation of the GoTo Declaration feature is not exactly the same as IntelliJ.
+In IntelliJ if a reference has several declarations, after clicking "Goto Declaration"
+there is a little window with all the declarations to choose from.
+It can be done by changing the method "multiResolve" in SdReference.java to return
+more than one declaration. The problem with that is that it causes the "Find Usages"
+feature to not work. For now the plugin "Goto Declaration" feature shows only the
+most specific declaration by the right rank-profile scope.
+
+The "Find Usages" window can group usages only under rank-profiles and document-summaries.
+Other usages appear directly under the .sd file. To create another group type of usages' group,
+you'll need to create 2 classes: one for the extension "fileStructureGroupRuleProvider"
+(e.g. SdRankProfileGroupingRuleProvider.java), and one for the
+grouping rule itself (e.g. SdRankProfileGroupingRule.java).
+Another open problem is that the navigation isn't working in the current grouping rules.
+It means that when clicking on the group headline (e.g. some name of a rank-profile)
+the IDE doesn't "jump" to the matching declaration.
+
+Goto declaration doesn't work for document and schema inherits. E.g. if document A inherits from
+document B, B doesn't have a reference to its declaration.
+
+There aren't any tests for the plugin.
+
+Semicolons in indexing statements are marked with red background for some reason.
+They are not marked as errors.
+
+Type suggestions should include all primitive types, not just annotations
+
+Even if the parser continues, only the first error in a file is marked.
+
+Aliases with dot does not work.
+
+
diff --git a/integration/intellij/README.md b/integration/intellij/README.md
new file mode 100644
index 00000000000..fec547f8a06
--- /dev/null
+++ b/integration/intellij/README.md
@@ -0,0 +1,52 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+# SD Reader
+
+IntelliJ plugin for working with Vespa application packages.
+
+## Using the plugin
+
+Download it from JetBrains Marketplace.
+
+## Using a local build
+
+Build (see below) and load it in IntelliJ by choosing
+Preferences -> Plugins -> Press the gear icon -> Install Plugin from Disk.
+
+## Building the plugin
+
+ gradle
+
+This produces an installable plugin .zip in the directory build/distributions
+
+*Prerequisite*: gradle 7.
+
+Why gradle? Because it's what JetBrains supports for building plugins.
+However, gradle is configured with a maven directory layout.
+
+## Optional IntelliJ plugins for working with plugin development
+
+1. Plugin DevKit
+2. Grammar-Kit: For reading the .bnf file.
+3. PsiViewer: Helps testing the bnf grammar.
+
+With the first (?), you can run the gradle task "intellij/runIde" (or "./gradlew runIde" in the command line),
+open a project with some sd file and see how the plugin works on it.
+
+
+## Some useful links:
+
+1. JetBrains official tutorials: https://plugins.jetbrains.com/docs/intellij/custom-language-support.html and
+ https://plugins.jetbrains.com/docs/intellij/custom-language-support-tutorial.html
+
+2. Grammar-Kit HOWTO: Helps to understand the BNF syntax.
+ https://github.com/JetBrains/Grammar-Kit/blob/master/HOWTO.md
+
+3. How to deal with left-recursion in the grammar (in SD for example it happens in expressions). Last answer here:
+ https://intellij-support.jetbrains.com/hc/en-us/community/posts/360001258300-What-s-the-alternative-to-left-recursion-in-GrammarKit-
+
+4. Great tutorial for a custom-language-plugin, but only for the basics (mainly the parser and lexer):
+ https://medium.com/@shan1024/custom-language-plugin-development-for-intellij-idea-part-01-d6a41ab96bc9
+
+5. Code of Dart (some custom language) plugin for IntelliJ:
+ https://github.com/JetBrains/intellij-plugins/tree/0f07ca63355d5530b441ca566c98f17c560e77f8/Dart \ No newline at end of file
diff --git a/sd-plugin/build.gradle b/integration/intellij/build.gradle
index e5b39769b96..4801bc810b0 100644
--- a/sd-plugin/build.gradle
+++ b/integration/intellij/build.gradle
@@ -1,5 +1,4 @@
-// This "noinspection" comment below is here to fix a warning
-//noinspection GroovyAssignabilityCheck
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
plugins {
id 'org.jetbrains.intellij' version '1.1.4'
id 'java'
@@ -17,17 +16,17 @@ import org.jetbrains.grammarkit.tasks.GenerateLexer
import org.jetbrains.grammarkit.tasks.GenerateParser
task generateSdLexer(type: GenerateLexer) {
- source 'src/main/java/org/intellij/sdk/language/lexer/sd.flex'
- targetDir 'src/main/gen/org/intellij/sdk/language/lexer/'
+ source 'src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex'
+ targetDir 'target/generated-sources/jflex/ai/vespa/intellij/schema/lexer/'
targetClass 'SdLexer'
purgeOldFiles true
}
task generateSdParser(type: GenerateParser) {
- source 'src/main/java/org/intellij/sdk/language/parser/sd.bnf'
- targetRoot 'src/main/gen'
- pathToParser 'org/intellij/sdk/language/parser/SdParser.java'
- pathToPsiRoot 'org/intellij/sdk/language/psi/'
+ source 'src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf'
+ targetRoot 'target/generated-sources/bnf/'
+ pathToParser 'ai/vespa/intellij/schema/parser/SdParser.java'
+ pathToPsiRoot 'ai/vespa/intellij/schema/parser/psi/'
purgeOldFiles true
}
@@ -36,20 +35,18 @@ compileJava {
dependsOn generateSdParser
}
-
-group 'org.yahoo.native'
-version '1.0.2'
+group 'ai.vespa'
+version '1.0.0' // Also update pom.xml version if this is changed
sourceCompatibility = 11
// This "noinspection" comment below is here to fix a warning
-//noinspection GroovyAssignabilityCheck
+// noinspection GroovyAssignabilityCheck
repositories {
mavenCentral()
}
-
-sourceSets.main.java.srcDirs 'src/main/gen'
+sourceSets.main.java.srcDirs = ['src/main/java', 'target/generated-sources/bnf', 'target/generated-sources/jflex']
// See https://github.com/JetBrains/gradle-intellij-plugin/
intellij {
diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml
new file mode 100644
index 00000000000..84a24b788b8
--- /dev/null
+++ b/integration/intellij/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>7-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle -->
+ <version>1.0.0</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle -->
+ <description>
+ Maven wrapper for the gradle build of this IntelliJ plugin.
+ </description>
+
+ <dependencies>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- The Gradle plugin must be built by Gradle; therefore the compilation is skipped. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <skipMain>true</skipMain>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <!-- Gradle is also responsible for creating javadoc such that this task is skipped here. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <!-- Tie Maven executions into the Gradle life-cycle. -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>gradle-clean</id>
+ <phase>clean</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>gradle</executable>
+ <arguments>
+ <argument>clean</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>gradle-build</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>gradle</executable>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Copies the artifact created by Gradle back to the Maven target folder. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-zip</id>
+ <phase>install</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy file="build/distributions/vespa-${project.version}.zip" todir="target" overwrite="true" />
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/integration/intellij/settings.gradle b/integration/intellij/settings.gradle
new file mode 100644
index 00000000000..94afd49df08
--- /dev/null
+++ b/integration/intellij/settings.gradle
@@ -0,0 +1,5 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+// This sets the name of the generated zip file uploadable to Intellij Marketplace
+// (But not the actual *name* of the plugin, which is the name in plugin.xml)
+rootProject.name = 'vespa'
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
index 67bdca7d2dc..6be7302a28c 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/sd.bnf
+++ b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
@@ -1,11 +1,13 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
/**
-This file is the SD grammar.
-NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many)
-@author: Shahar Ariel
+ * Vespa schema file grammar.
+ * NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats it like zero-to-many)
+ *
+ * @author: Shahar Ariel
*/
{
- parserClass="org.intellij.sdk.language.parser.SdParser" // Name and the location of the parser which will be generated.
+ parserClass="ai.vespa.intellij.schema.parser.SdParser" // Name and the location of the parser which will be generated.
extends="com.intellij.extapi.psi.ASTWrapperPsiElement" // All nodes will extend this class. Wraps AST node to a PSI node.
@@ -13,13 +15,13 @@ NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats
psiClassPrefix="Sd"
psiImplClassSuffix="Impl"
- psiPackage="org.intellij.sdk.language.psi" // Location to be used when generating PSI classes.
- psiImplPackage="org.intellij.sdk.language.psi.impl" // Location to be used when generating PSI implementation classes.
+ psiPackage="ai.vespa.intellij.schema.psi" // Location to be used when generating PSI classes.
+ psiImplPackage="ai.vespa.intellij.schema.psi.impl" // Location to be used when generating PSI implementation classes.
- elementTypeHolderClass="org.intellij.sdk.language.psi.SdTypes" // Element type holder class name.
+ elementTypeHolderClass="ai.vespa.intellij.schema.psi.SdTypes" // Element type holder class name.
- elementTypeClass="org.intellij.sdk.language.psi.SdElementType" // Class which will be used to create internal nodes.
- tokenTypeClass="org.intellij.sdk.language.psi.SdTokenType" // Class which will be used to create leaf nodes.
+ elementTypeClass="ai.vespa.intellij.schema.psi.SdElementType" // Class which will be used to create internal nodes.
+ tokenTypeClass="ai.vespa.intellij.schema.psi.SdTokenType" // Class which will be used to create leaf nodes.
extends(".*Expr")=RankingExpression // Here to deal with left-recursion that happens in expressions
@@ -30,7 +32,7 @@ NOTE: This grammar does not enforce zero-or-one occurrences of elements (treats
COMMENT = 'regexp:#.*'
SYMBOL = 'regexp:[!$|:{}(),.\[\]]'
COMPARISON_OPERATOR = 'regexp:[<>]|(==)|(<=)|(>=)|(~=)'
- ARITHMETIC_OPERATOR = 'regexp:[\-+*/]'
+ ARITHMETIC_OPERATOR = 'regexp:[\-+*/%]'
INTEGER_REG = 'regexp:[0-9]+'
FLOAT_REG = 'regexp:[0-9]+[.][0-9]+[e]?'
STRING_REG = 'regexp:\"([^\"\\]*(\\.[^\"\\]*)*)\"'
@@ -42,14 +44,14 @@ SdFile ::= SchemaDefinition | DocumentDefinition
SchemaDefinition ::= (search | schema) IdentifierVal? (inherits IdentifierVal)? '{' SchemaBody '}'
SchemaBody ::= SchemaBodyOptions* DocumentDefinition SchemaBodyOptions* // Does not support zero-or-one occurrences
private SchemaBodyOptions ::= SchemaFieldDefinition | ImportFieldDefinition | DocumentSummaryDefinition |
- RankProfileDefinition | IndexDefinition |
+ RankProfileDefinition | IndexDefinition | DocumentStructDefinition |
FieldSetDefinition | ConstantDefinition | OnnxModelDefinition | StemmingDefinition |
raw-as-base64-in-summary | SchemaAnnotationDefinition
SchemaFieldDefinition ::= field IdentifierVal type FieldTypeName '{' SchemaFieldBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
FieldTypeName ::= ("array" '<' (FieldTypeName | IdentifierVal) '>') | ("weightedset" '<' SingleValueFieldTypeName '>') |
@@ -63,15 +65,15 @@ private TensorDimension ::= WordWrapper (('{' '}') | ('[' INTEGER_REG ']'))
SchemaFieldBody ::= DocumentFieldBodyOptions* // Fields of schemas and documents defined the same way here
DocumentSummaryDefinition ::= document-summary IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' DocumentSummaryBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
DocumentSummaryBody ::= DocumentSummaryBodyOptions* // Does not support zero-or-one occurrences
private DocumentSummaryBodyOptions ::= SummaryDefinition | omit-summary-features | from-disk
ImportFieldDefinition ::= import field IdentifierVal '.' IdentifierVal as IdentifierVal '{''}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
FieldSetDefinition ::= fieldset IdentifierVal '{' FieldSetBody '}'
@@ -91,14 +93,14 @@ private OnnxModelBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) |
((input | output) (IdentifierVal | STRING_REG) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG))
SchemaAnnotationDefinition ::= AnnotationDefinition
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
private AnnotationDefinition ::= annotation IdentifierVal (inherits IdentifierVal)? '{' AnnotationFieldDefinition* '}'
AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName '{' '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
//-------------------------
@@ -128,8 +130,8 @@ PrimitiveExpr ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | Ra
//-- Rank Profile rules ---
//-------------------------
RankProfileDefinition ::= (rank-profile | model) IdentifierWithDashVal (inherits IdentifierWithDashVal)? '{' RankProfileBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration"]
}
private RankProfileBody ::= RankProfileBodyOptions* // Does not support zero-or-one occurrences
private RankProfileBodyOptions ::= MatchPhaseDefinition | NumThreadsDefinition | FunctionDefinition | TermwiseLimitDefinition |
@@ -153,7 +155,7 @@ private TermwiseLimitDefinition ::= termwise-limit ':' ('-')? (FLOAT_REG | INTEG
private MinHitsDefinition ::= min-hits-per-thread ':' ('-')? INTEGER_REG
private NumSearchPartitionDefinition ::= num-search-partition ':' INTEGER_REG
FieldWeightDefinition ::= weight IdentifierVal ':' INTEGER_REG
-FirstPhaseDefinition ::= first-phase '{' FirstPhaseBody '}' { mixin="org.intellij.sdk.language.psi.impl.SdFirstPhaseDefinitionMixin" }
+FirstPhaseDefinition ::= first-phase '{' FirstPhaseBody '}' { mixin="ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin" }
FirstPhaseBody ::= FirstPhaseBodyOptions* // Does not support zero-or-one occurrences
private FirstPhaseBodyOptions ::= (keep-rank-count ':' INTEGER_REG) | (rank-score-drop-limit ':' ('-')? (FLOAT_REG | INTEGER_REG)) | ExpressionDefinition
@@ -168,19 +170,19 @@ RankPropertiesBody ::= (RankPropertiesKey ':' RankPropertiesValue)+
RankPropertiesKey ::= (IdentifierWithDashVal | STRING_REG | '(' | ')' | '.' | ',' | '$' | INTEGER_REG)+
RankPropertiesValue ::= (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | WORD_REG | IdentifierVal | STRING_REG
-FunctionDefinition ::= (function | macro) inline? IdentifierVal '(' (ArgumentDefinition (',' ArgumentDefinition)*)? ')'
+FunctionDefinition ::= (function | macro) inline? IdentifierVal ( '()' | '(' (ArgumentDefinition (',' ArgumentDefinition)*)? ')' )
'{' ExpressionDefinition '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdFunctionDefinitionInterface" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdFunctionDefinitionInterface" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
ArgumentDefinition ::= IdentifierVal
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
SummaryFeaturesDefinition ::= summary-features ((':' RankFeature+) | ((inherits IdentifierVal)? '{' RankFeature* '}'))
-MatchFeaturesDefinition ::= match-features ((':' RankFeature+) | ('{' RankFeature* '}'))
+MatchFeaturesDefinition ::= match-features ((':' RankFeature+) | ((inherits IdentifierVal)? '{' RankFeature* '}'))
RankFeaturesDefinition ::= rank-features ((':' RankFeature+) | ('{' RankFeature* '}'))
@@ -188,46 +190,46 @@ ConstantsDefinition ::= constants '{' (IdentifierVal ':' RankPropertiesValue)* '
RankFeature ::= QueryDefinition | ItemRawScoreDefinition | FunctionCallExpr | (IdentifierWithDashVal ('.' IdentifierWithDashVal)* )
QueryDefinition ::= "query" '(' IdentifierWithDashVal ')'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
ItemRawScoreDefinition ::= "itemRawScore" '(' IdentifierVal ')'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
//-------------------------
//---- Document rules -----
//-------------------------
DocumentDefinition ::= document (IdentifierVal (inherits IdentifierVal (',' IdentifierVal)*)?)? '{' DocumentBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
DocumentBody ::= DocumentBodyOptions*
DocumentBodyOptions ::= DocumentStructDefinition | DocumentFieldDefinition | DocumentAnnotationDefinition
DocumentAnnotationDefinition ::= AnnotationDefinition
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
DocumentStructDefinition ::= struct IdentifierVal '{' DocumentStructBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
DocumentStructBody ::= DocumentStructFieldDefinition*
DocumentStructFieldDefinition ::= field IdentifierVal type FieldTypeName '{' DocumentStructFieldBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
DocumentStructFieldBody ::= MatchDefinition?
DocumentFieldDefinition ::= field IdentifierVal type FieldTypeName '{' DocumentFieldBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
DocumentFieldBody ::= DocumentFieldBodyOptions* // Does not support zero-or-one occurrences
@@ -235,12 +237,12 @@ private DocumentFieldBodyOptions ::= StructFieldDefinition | MatchDefinition | I
AliasDefinition | RankDefinition | IndexingRewriteState | QueryCommandDefinition | SummaryDefinition |
BoldingDefinition | (id ':' INTEGER_REG) | IndexDefinition | (normalizing ':' IdentifierWithDashVal) |
SortingDefinition | StemmingDefinition | (weight ':' INTEGER_REG) | WeightedSetDefinition |
- RankTypeDefinition | DictionaryDefinition | SummaryToDefinition | body
+ RankTypeDefinition | DictionaryDefinition | SummaryToDefinition | header | body
//***** Field's body elements ******//
// Struct
StructFieldDefinition ::= struct-field IdentifierVal ('.' IdentifierVal)? '{' StructFieldBody '}'
- { mixin="org.intellij.sdk.language.psi.impl.SdNamedElementImpl"
- implements=["org.intellij.sdk.language.psi.SdDeclaration" "org.intellij.sdk.language.psi.SdNamedElement"]
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl"
+ implements=["ai.vespa.intellij.schema.psi.SdDeclaration" "ai.vespa.intellij.schema.psi.SdNamedElement"]
}
StructFieldBody ::= StructFieldBodyOptions* // Does not support zero-or-one occurrences
@@ -248,15 +250,16 @@ StructFieldBodyOptions ::= IndexingDefinition | AttributeDefinition | MatchDefin
StructFieldDefinition | SummaryDefinition
// Match
MatchDefinition ::= match ((':' MatchProperty) | ('{' MatchProperty+ '}'))
-MatchProperty ::= text | exact | (exact-terminator ':' STRING_REG) | word | prefix | cased | uncased | substring |
+MatchProperty ::= text | token | exact | (exact-terminator ':' STRING_REG) | word | prefix | cased | uncased | substring |
suffix | (max-length ':' INTEGER_REG) | gram | (gram-size ':' INTEGER_REG) | WordWrapper
// Indexing
IndexingDefinition ::= indexing ((':' IndexingStatement) | ('{' IndexingStatement+ '}'))
-IndexingStatement ::= IndexingStatementOptions ((('|' | ';') IndexingStatementOptions)*)
+IndexingStatement ::= IndexingStatementOptions (('|' IndexingStatementOptions)*) (';')?
// Does not support zero-or-one occurrences
-IndexingStatementOptions ::= summary | attribute | index | set_language | lowercase | (input (IdentifierVal | IndexingStuff)+) |
+IndexingStatementOptions ::= summary | attribute | index | set_language | lowercase |
+ (input (IdentifierVal | IndexingStuff)+) |
('{' IndexingStatementOptions '}') | IndexingStuff+
-private IndexingStuff ::= WordWrapper | INTEGER_REG | FLOAT_REG | STRING_REG | ('{' IndexingStatementOptions+ '}') |
+private IndexingStuff ::= WordWrapper | INTEGER_REG | FLOAT_REG | STRING_REG | ('{' IndexingStatement '}') |
':' | ('|' IndexingStatementOptions) | ';' | '.' | '(' | ')' | ARITHMETIC_OPERATOR | COMPARISON_OPERATOR
// Attribute
AttributeDefinition ::= attribute ((':' SimpleAttributeProperty) | ('{' (ComplexAttributeProperty | SimpleAttributeProperty)+ '}'))
@@ -264,7 +267,7 @@ SimpleAttributeProperty ::= fast-search | fast-access | paged | mutable | enable
ComplexAttributeProperty ::= AliasDefinition | SortingDefinition | DistanceMetricDef // Does not support zero-or-one occurrences
DistanceMetricDef ::= distance-metric ':' IdentifierWithDashVal
// Alias
-AliasDefinition ::= alias (IdentifierWithDashVal ('.' IdentifierWithDashVal)*)? ':' IdentifierWithDashVal ('.' IdentifierWithDashVal)*
+AliasDefinition ::= alias (IdentifierVal)? ':' IdentifierWithDashVal ('.' IdentifierWithDashVal)*
// Stemming
StemmingDefinition ::= stemming ':' IdentifierWithDashVal
// Rank
@@ -276,7 +279,7 @@ IndexingRewriteState ::= indexing-rewrite ':' none
QueryCommandDefinition ::= query-command ':' (IdentifierVal | STRING_REG | WordWrapper)
// Summary
SummaryDefinition ::= summary IdentifierWithDashVal? (type FieldTypeName)? ((':' SummaryBodyOptions) | ( '{' SummaryBody '}'))
- { mixin="org.intellij.sdk.language.psi.impl.SdSummaryDefinitionMixin" }
+ { mixin="ai.vespa.intellij.schema.psi.impl.SdSummaryDefinitionMixin" }
SummaryBody ::= SummaryBodyOptions* // Does not support zero-or-one occurrences
SummaryBodyOptions ::= full | static | dynamic | (source ':' (IdentifierVal ('.' IdentifierVal)?) (',' IdentifierVal ('.' IdentifierVal)?)*) |
(to ':' IdentifierVal (',' IdentifierVal)*) | matched-elements-only | BoldingDefinition
@@ -317,12 +320,12 @@ DictionarySetting ::= hash | btree | cased | uncased
private WordWrapper ::= KeywordOrIdentifier | KeywordNotIdentifier | ID_REG | ID_WITH_DASH_REG | WORD_REG
-IdentifierVal ::= KeywordOrIdentifier | ID_REG { mixin="org.intellij.sdk.language.psi.impl.SdIdentifierMixin"
- implements=["org.intellij.sdk.language.psi.SdIdentifier"]
+IdentifierVal ::= KeywordOrIdentifier | ID_REG { mixin="ai.vespa.intellij.schema.psi.impl.SdIdentifierMixin"
+ implements=["ai.vespa.intellij.schema.psi.SdIdentifier"]
}
-IdentifierWithDashVal ::= ID_WITH_DASH_REG | IdentifierVal { mixin="org.intellij.sdk.language.psi.impl.SdIdentifierMixin"
- implements=["org.intellij.sdk.language.psi.SdIdentifier"]
+IdentifierWithDashVal ::= ID_WITH_DASH_REG | IdentifierVal { mixin="ai.vespa.intellij.schema.psi.impl.SdIdentifierMixin"
+ implements=["ai.vespa.intellij.schema.psi.SdIdentifier"]
}
// Those lists of keywords (KeywordOrIdentifier and KeywordNotIdentifier) have to be synchronized with sd.flex file.
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java
index 99bebe21b87..01992ef4a5b 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdChooseByNameContributor.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdChooseByNameContributor.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.navigation.ChooseByNameContributor;
import com.intellij.navigation.NavigationItem;
@@ -8,8 +8,8 @@ import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiManager;
import com.intellij.psi.search.FileTypeIndex;
import com.intellij.psi.search.GlobalSearchScope;
-import org.intellij.sdk.language.psi.SdDeclaration;
-import org.intellij.sdk.language.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdDeclaration;
+import ai.vespa.intellij.schema.psi.SdFile;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
@@ -18,6 +18,7 @@ import java.util.List;
/**
* This class is used for the extension (in plugin.xml) to enable "Go To Symbol" feature.
+ *
* @author Shahar Ariel
*/
public class SdChooseByNameContributor implements ChooseByNameContributor {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettings.java
index 93c3e7ccc50..bdf1bf3eb7b 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettings.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettings.java
@@ -1,11 +1,12 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.psi.codeStyle.CodeStyleSettings;
import com.intellij.psi.codeStyle.CustomCodeStyleSettings;
/**
* This class represent a code style settings, and creates an option page in settings/preferences.
+ *
* @author Shahar Ariel
*/
public class SdCodeStyleSettings extends CustomCodeStyleSettings {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java
index 3b075b0f438..b02e8371b77 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCodeStyleSettingsProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCodeStyleSettingsProvider.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.application.options.CodeStyleAbstractConfigurable;
import com.intellij.application.options.CodeStyleAbstractPanel;
@@ -13,6 +13,7 @@ import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml) to the class SdCodeStyleSettings.
+ *
* @author Shahar Ariel
*/
public class SdCodeStyleSettingsProvider extends CodeStyleSettingsProvider {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java
index d0f28848cf0..a8872f71b63 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCommenter.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCommenter.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.lang.Commenter;
import org.jetbrains.annotations.Nullable;
@@ -7,6 +7,7 @@ import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml), to enable turning a line into a comment with
* "Code -> Comment with line comment".
+ *
* @author Shahar Ariel
*/
public class SdCommenter implements Commenter {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java
index 8bcda614ee7..3e5b53c87ea 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdCompletionContributor.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdCompletionContributor.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.codeInsight.completion.CompletionContributor;
import com.intellij.codeInsight.completion.CompletionParameters;
@@ -9,11 +9,12 @@ import com.intellij.codeInsight.completion.CompletionType;
import com.intellij.codeInsight.lookup.LookupElementBuilder;
import com.intellij.patterns.PlatformPatterns;
import com.intellij.util.ProcessingContext;
-import org.intellij.sdk.language.psi.SdTypes;
+import ai.vespa.intellij.schema.psi.SdTypes;
import org.jetbrains.annotations.NotNull;
/**
* This class is used for the extension (in plugin.xml) to enables Auto-Complete. Partially works for now.
+ *
* @author Shahar Ariel
*/
public class SdCompletionContributor extends CompletionContributor {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java
index ccc4c6b6348..b469d0c0ebb 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdFileType.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdFileType.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.openapi.fileTypes.LanguageFileType;
import org.jetbrains.annotations.NotNull;
@@ -9,6 +9,7 @@ import javax.swing.Icon;
/**
* This class is used for the extension (in plugin.xml), to define SD as a file's type.
+ *
* @author Shahar Ariel
*/
public class SdFileType extends LanguageFileType {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdIcons.java
index 806fcd6b6ba..96f8428b223 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdIcons.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdIcons.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.openapi.util.IconLoader;
@@ -7,9 +7,11 @@ import javax.swing.Icon;
/**
* This class is used for defining Icons for the IDE.
+ *
* @author Shahar Ariel
*/
public class SdIcons {
+
public static final Icon FILE = IconLoader.getIcon("icons/sd_icon.png", SdIcons.class);
public static final Icon STRUCT_FIELD = IconLoader.getIcon("icons/struct_field_icon.png", SdIcons.class);
public static final Icon IMPORTED_FIELD = IconLoader.getIcon("icons/imported_field_icon.png", SdIcons.class);
@@ -20,5 +22,6 @@ public class SdIcons {
public static final Icon MACRO = IconLoader.getIcon("icons/macro_icon.png", SdIcons.class);
public static final Icon OVERRIDE_MACRO = IconLoader.getIcon("icons/override_macro_icon.png", SdIcons.class);
public static final Icon FIRST_PHASE = IconLoader.getIcon("icons/first_phase_icon.png", SdIcons.class);
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguage.java
index fa253c4d605..c4d471ee737 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguage.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguage.java
@@ -1,13 +1,15 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.lang.Language;
/**
* This class defines SD as a language.
+ *
* @author Shahar Ariel
*/
public class SdLanguage extends Language {
+
public static final SdLanguage INSTANCE = new SdLanguage();
private SdLanguage() {
@@ -18,4 +20,5 @@ public class SdLanguage extends Language {
public boolean isCaseSensitive() {
return true;
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java
index 24fef3af40c..5f618a62d6e 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdLanguageCodeStyleSettingsProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdLanguageCodeStyleSettingsProvider.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.lang.Language;
import com.intellij.psi.codeStyle.CodeStyleSettingsCustomizable;
@@ -8,6 +8,7 @@ import org.jetbrains.annotations.NotNull;
/**
* This class is used for the extension (in plugin.xml), to make the IDE use our plugin's code for coding style.
+ *
* @author Shahar Ariel
*/
public class SdLanguageCodeStyleSettingsProvider extends LanguageCodeStyleSettingsProvider {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java
index 1b5cba028a7..24de7bb6253 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdRefactoringSupportProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdRefactoringSupportProvider.java
@@ -1,19 +1,22 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.lang.refactoring.RefactoringSupportProvider;
import com.intellij.psi.PsiElement;
-import org.intellij.sdk.language.psi.SdIdentifier;
+import ai.vespa.intellij.schema.psi.SdIdentifier;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml), to enable refactoring.
+ *
* @author Shahar Ariel
*/
public class SdRefactoringSupportProvider extends RefactoringSupportProvider {
+
@Override
public boolean isMemberInplaceRenameAvailable(@NotNull PsiElement elementToRename, @Nullable PsiElement context) {
return (elementToRename instanceof SdIdentifier);
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java
index bce51ccc2ff..6404a738e59 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdReference.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdReference.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.codeInsight.lookup.LookupElement;
import com.intellij.codeInsight.lookup.LookupElementBuilder;
@@ -11,7 +11,7 @@ import com.intellij.psi.PsiPolyVariantReference;
import com.intellij.psi.PsiReferenceBase;
import com.intellij.psi.ResolveResult;
import com.intellij.util.IncorrectOperationException;
-import org.intellij.sdk.language.psi.SdDeclaration;
+import ai.vespa.intellij.schema.psi.SdDeclaration;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -19,7 +19,8 @@ import java.util.ArrayList;
import java.util.List;
/**
- * This class represent a reference to a Psi Element.
+ * A reference to a Psi Element.
+ *
* @author Shahar Ariel
*/
public class SdReference extends PsiReferenceBase<PsiElement> implements PsiPolyVariantReference {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java
index e6452685e3c..df3da81e9fc 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighter.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighter.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import static com.intellij.openapi.editor.colors.TextAttributesKey.createTextAttributesKey;
@@ -10,14 +10,15 @@ import com.intellij.openapi.editor.colors.TextAttributesKey;
import com.intellij.openapi.fileTypes.SyntaxHighlighterBase;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IElementType;
-import org.intellij.sdk.language.lexer.SdLexerAdapter;
-import org.intellij.sdk.language.psi.SdTypes;
+import ai.vespa.intellij.schema.lexer.SdLexerAdapter;
+import ai.vespa.intellij.schema.psi.SdTypes;
import org.jetbrains.annotations.NotNull;
import java.util.HashSet;
/**
- * This class defines the syntax highlighting of an SD file.
+ * Defines the syntax highlighting of an SD file.
+ *
* @author Shahar Ariel
*/
public class SdSyntaxHighlighter extends SyntaxHighlighterBase {
@@ -204,8 +205,8 @@ public class SdSyntaxHighlighter extends SyntaxHighlighterBase {
constants.add(SdTypes.LOCALE);
constants.add(SdTypes.CREATE_IF_NONEXISTENT);
constants.add(SdTypes.REMOVE_IF_ZERO);
-
return constants;
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java
index 618e188f680..6cf30ecb54e 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdSyntaxHighlighterFactory.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdSyntaxHighlighterFactory.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.openapi.fileTypes.SyntaxHighlighter;
import com.intellij.openapi.fileTypes.SyntaxHighlighterFactory;
@@ -9,6 +9,7 @@ import org.jetbrains.annotations.NotNull;
/**
* This class is used for the extension (in plugin.xml) to the class SdSyntaxHighlighter.
+ *
* @author Shahar Ariel
*/
public class SdSyntaxHighlighterFactory extends SyntaxHighlighterFactory {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java
index 5d6cd85489f..f3f949d645f 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/SdUtil.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/SdUtil.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language;
+package ai.vespa.intellij.schema;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.project.Project;
@@ -11,25 +11,25 @@ import com.intellij.psi.PsiReference;
import com.intellij.psi.search.FileTypeIndex;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition;
-import org.intellij.sdk.language.psi.SdArgumentDefinition;
-import org.intellij.sdk.language.psi.SdDeclaration;
-import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition;
-import org.intellij.sdk.language.psi.SdDocumentDefinition;
-import org.intellij.sdk.language.psi.SdDocumentFieldDefinition;
-import org.intellij.sdk.language.psi.SdDocumentStructDefinition;
-import org.intellij.sdk.language.psi.SdDocumentStructFieldDefinition;
-import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition;
-import org.intellij.sdk.language.psi.SdFieldTypeName;
-import org.intellij.sdk.language.psi.SdFile;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
-import org.intellij.sdk.language.psi.SdIdentifier;
-import org.intellij.sdk.language.psi.SdImportFieldDefinition;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
-import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition;
-import org.intellij.sdk.language.psi.SdSchemaFieldDefinition;
-import org.intellij.sdk.language.psi.SdSummaryDefinition;
-import org.intellij.sdk.language.psi.SdTypes;
+import ai.vespa.intellij.schema.psi.SdAnnotationFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdArgumentDefinition;
+import ai.vespa.intellij.schema.psi.SdDeclaration;
+import ai.vespa.intellij.schema.psi.SdDocumentAnnotationDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentStructDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentStructFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition;
+import ai.vespa.intellij.schema.psi.SdFieldTypeName;
+import ai.vespa.intellij.schema.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdIdentifier;
+import ai.vespa.intellij.schema.psi.SdImportFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition;
+import ai.vespa.intellij.schema.psi.SdSchemaFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdSummaryDefinition;
+import ai.vespa.intellij.schema.psi.SdTypes;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
@@ -38,7 +38,8 @@ import java.util.HashMap;
import java.util.List;
/**
- * This is the util class for the plugin's code.
+ * Util class for the plugin's code.
+ *
* @author Shahar Ariel
*/
public class SdUtil {
@@ -217,5 +218,4 @@ public class SdUtil {
return new ArrayList<>(PsiTreeUtil.collectElementsOfType(element, SdSummaryDefinition.class));
}
-
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java
index ba081e9d415..ab463562355 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRule.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRule.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.openapi.project.DumbAware;
import com.intellij.psi.PsiElement;
@@ -8,13 +8,14 @@ import com.intellij.usages.UsageGroup;
import com.intellij.usages.UsageTarget;
import com.intellij.usages.rules.PsiElementUsage;
import com.intellij.usages.rules.SingleParentUsageGroupingRule;
-import org.intellij.sdk.language.SdLanguage;
-import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition;
+import ai.vespa.intellij.schema.SdLanguage;
+import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
- * This class represent a Document Summary that groups elements in the "Find Usages" window.
+ * A Document Summary that groups elements in the "Find Usages" window.
+ *
* @author Shahar Ariel
*/
public class SdDocumentSummaryGroupingRule extends SingleParentUsageGroupingRule implements DumbAware {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java
index 0e6f0358ca5..76b82d7c857 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdDocumentSummaryGroupingRuleProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdDocumentSummaryGroupingRuleProvider.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.openapi.project.Project;
import com.intellij.usages.impl.FileStructureGroupRuleProvider;
@@ -9,6 +9,7 @@ import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml) to the class SdDocumentSummaryGroupingRule.
+ *
* @author Shahar Ariel
*/
public class SdDocumentSummaryGroupingRuleProvider implements FileStructureGroupRuleProvider {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java
index b48c35e5c49..4609f2e7fbb 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandler.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandler.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.find.findUsages.FindUsagesHandler;
import com.intellij.find.findUsages.FindUsagesOptions;
@@ -13,9 +13,9 @@ import com.intellij.psi.search.SearchScope;
import com.intellij.psi.search.searches.ReferencesSearch;
import com.intellij.usageView.UsageInfo;
import com.intellij.util.Processor;
-import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.psi.SdFile;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.SdUtil;
+import ai.vespa.intellij.schema.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
import org.jetbrains.annotations.NotNull;
import java.util.HashMap;
@@ -23,6 +23,7 @@ import java.util.List;
/**
* This class handles creating the "Find Usages" window.
+ *
* @author Shahar Ariel
*/
public class SdFindUsagesHandler extends FindUsagesHandler {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java
index b96cad508f7..98be15778e7 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesHandlerFactory.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesHandlerFactory.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.find.findUsages.FindUsagesHandler;
import com.intellij.find.findUsages.FindUsagesHandlerFactory;
@@ -10,6 +10,7 @@ import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml) to the class SdFindUsagesHandler.
+ *
* @author Shahar Ariel
*/
public class SdFindUsagesHandlerFactory extends FindUsagesHandlerFactory {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java
index e3eb6b431c7..a86fb2b058b 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdFindUsagesProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdFindUsagesProvider.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.lang.cacheBuilder.DefaultWordsScanner;
import com.intellij.lang.cacheBuilder.WordsScanner;
@@ -7,20 +7,22 @@ import com.intellij.lang.findUsages.FindUsagesProvider;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiNamedElement;
import com.intellij.psi.tree.TokenSet;
-import org.intellij.sdk.language.lexer.SdLexerAdapter;
-import org.intellij.sdk.language.psi.SdDeclaration;
-import org.intellij.sdk.language.psi.SdIdentifierVal;
-import org.intellij.sdk.language.psi.SdIdentifierWithDashVal;
-import org.intellij.sdk.language.psi.SdTypes;
-import org.intellij.sdk.language.psi.impl.SdNamedElementImpl;
+import ai.vespa.intellij.schema.lexer.SdLexerAdapter;
+import ai.vespa.intellij.schema.psi.SdDeclaration;
+import ai.vespa.intellij.schema.psi.SdIdentifierVal;
+import ai.vespa.intellij.schema.psi.SdIdentifierWithDashVal;
+import ai.vespa.intellij.schema.psi.SdTypes;
+import ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml), to enable "find Usages" window using the plugin code.
+ *
* @author Shahar Ariel
*/
public class SdFindUsagesProvider implements FindUsagesProvider {
+
@Nullable
@Override
public WordsScanner getWordsScanner() {
@@ -71,4 +73,5 @@ public class SdFindUsagesProvider implements FindUsagesProvider {
return "";
}
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java
index 1fa848313e0..fd3073b6016 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRule.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRule.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.openapi.project.DumbAware;
import com.intellij.psi.PsiElement;
@@ -8,13 +8,14 @@ import com.intellij.usages.UsageGroup;
import com.intellij.usages.UsageTarget;
import com.intellij.usages.rules.PsiElementUsage;
import com.intellij.usages.rules.SingleParentUsageGroupingRule;
-import org.intellij.sdk.language.SdLanguage;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.SdLanguage;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
- * This class represent a Rank Profile that groups elements in the "Find Usages" window.
+ * A Rank Profile that groups elements in the "Find Usages" window.
+ *
* @author Shahar Ariel
*/
public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule implements DumbAware {
@@ -34,4 +35,5 @@ public class SdRankProfileGroupingRule extends SingleParentUsageGroupingRule imp
return null;
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java
index 0b973a56644..0726e103943 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdRankProfileGroupingRuleProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdRankProfileGroupingRuleProvider.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.openapi.project.Project;
import com.intellij.usages.impl.FileStructureGroupRuleProvider;
@@ -9,6 +9,7 @@ import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml) to the class SdRankProfileGroupingRule.
+ *
* @author Shahar Ariel
*/
public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRuleProvider {
@@ -17,4 +18,5 @@ public class SdRankProfileGroupingRuleProvider implements FileStructureGroupRule
public @Nullable UsageGroupingRule getUsageGroupingRule(@NotNull Project project) {
return new SdRankProfileGroupingRule();
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java
index b67c561e8d4..6f24ce27476 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/findUsages/SdUsageGroup.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/findUsages/SdUsageGroup.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.findUsages;
+package ai.vespa.intellij.schema.findUsages;
import com.intellij.navigation.ItemPresentation;
import com.intellij.navigation.NavigationItemFileStatus;
@@ -10,17 +10,19 @@ import com.intellij.psi.SmartPointerManager;
import com.intellij.psi.SmartPsiElementPointer;
import com.intellij.usages.UsageGroup;
import com.intellij.usages.UsageView;
-import org.intellij.sdk.language.psi.SdDeclaration;
+import ai.vespa.intellij.schema.psi.SdDeclaration;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import javax.swing.*;
/**
- * This class represent a group of elements in the "Find Usages" window.
+ * A group of elements in the "Find Usages" window.
+ *
* @author Shahar Ariel
*/
public class SdUsageGroup implements UsageGroup {
+
private final VirtualFile myFile;
private final SmartPsiElementPointer<SdDeclaration> myElementPointer;
private final String myText;
@@ -107,4 +109,5 @@ public class SdUsageGroup implements UsageGroup {
@Override
public void update() {}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java
index 46b4e738fcb..38817e15553 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyBrowser.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyBrowser.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.ide.hierarchy.CallHierarchyBrowserBase;
import com.intellij.ide.hierarchy.HierarchyNodeDescriptor;
@@ -11,7 +11,7 @@ import com.intellij.openapi.actionSystem.IdeActions;
import com.intellij.openapi.project.Project;
import com.intellij.psi.PsiElement;
import com.intellij.util.ObjectUtils;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -21,7 +21,8 @@ import java.util.Map;
import javax.swing.JTree;
/**
- * This class is a browser for the "Call Hierarchy" window.
+ * A browser for the "Call Hierarchy" window.
+ *
* @author Shahar Ariel
*/
public class SdCallHierarchyBrowser extends CallHierarchyBrowserBase {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java
index fdd8871201e..f99a82dc81a 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyNodeDescriptor.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyNodeDescriptor.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.ide.hierarchy.HierarchyNodeDescriptor;
import com.intellij.ide.util.treeView.NodeDescriptor;
@@ -10,18 +10,19 @@ import com.intellij.psi.NavigatablePsiElement;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
-import org.intellij.sdk.language.SdIcons;
-import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.psi.SdFirstPhaseDefinition;
-import org.intellij.sdk.language.psi.impl.SdFirstPhaseDefinitionMixin;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.SdIcons;
+import ai.vespa.intellij.schema.SdUtil;
+import ai.vespa.intellij.schema.psi.SdFirstPhaseDefinition;
+import ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
import org.jetbrains.annotations.NotNull;
import javax.swing.Icon;
/**
- * This class represents a node descriptor to a node in a tree in the "Call Hierarchy" window.
+ * A node descriptor to a node in a tree in the "Call Hierarchy" window.
+ *
* @author Shahar Ariel
*/
public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor {
@@ -68,10 +69,7 @@ public class SdCallHierarchyNodeDescriptor extends HierarchyNodeDescriptor {
if (!Comparing.equal(myHighlightedText, oldText)) {
changes = true;
}
-
-
return changes;
-
}
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java
index 4808c55f1ae..f7f243356b9 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallHierarchyProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallHierarchyProvider.java
@@ -1,6 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
-
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.ide.hierarchy.CallHierarchyBrowserBase;
import com.intellij.ide.hierarchy.HierarchyBrowser;
@@ -11,8 +10,8 @@ import com.intellij.psi.PsiDocumentManager;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.PsiReference;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
-import org.intellij.sdk.language.psi.SdIdentifierVal;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdIdentifierVal;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import com.intellij.openapi.actionSystem.CommonDataKeys;
@@ -20,6 +19,7 @@ import com.intellij.openapi.project.Project;
/**
* This class is used for the extension (in plugin.xml), to enable "Call Hierarchy" window using the plugin code.
+ *
* @author Shahar Ariel
*/
public class SdCallHierarchyProvider implements HierarchyProvider {
@@ -61,4 +61,5 @@ public class SdCallHierarchyProvider implements HierarchyProvider {
public void browserActivated(@NotNull HierarchyBrowser hierarchyBrowser) {
((SdCallHierarchyBrowser) hierarchyBrowser).changeView(CallHierarchyBrowserBase.getCallerType());
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java
index 0f843d9fe6a..053e7bd20f9 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallTreeStructure.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallTreeStructure.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.ide.hierarchy.HierarchyNodeDescriptor;
import com.intellij.ide.hierarchy.HierarchyTreeStructure;
@@ -8,10 +8,10 @@ import com.intellij.psi.PsiElement;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.ArrayUtilRt;
-import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.psi.SdFile;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.SdUtil;
+import ai.vespa.intellij.schema.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
@@ -20,10 +20,12 @@ import java.util.HashSet;
import java.util.List;
/**
- * This abstract class represents a general tree in the "Call Hierarchy" window.
+ * A general tree in the "Call Hierarchy" window.
+ *
* @author Shahar Ariel
*/
public abstract class SdCallTreeStructure extends HierarchyTreeStructure {
+
protected final String myScopeType;
protected final SdFile myFile;
protected HashMap<String, List<PsiElement>> macrosMap;
@@ -73,4 +75,5 @@ public abstract class SdCallTreeStructure extends HierarchyTreeStructure {
}
return ArrayUtil.toObjectArray(descriptors);
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java
index cce1ebf172a..37fd336b615 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCalleeTreeStructure.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCalleeTreeStructure.java
@@ -1,15 +1,15 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.openapi.project.Project;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiNamedElement;
import com.intellij.psi.PsiReference;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.psi.SdExpressionDefinition;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
-import org.intellij.sdk.language.psi.SdIdentifier;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.psi.SdExpressionDefinition;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdIdentifier;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
import org.jetbrains.annotations.NotNull;
import java.util.HashMap;
@@ -17,7 +17,8 @@ import java.util.HashSet;
import java.util.List;
/**
- * This class represents a Callee tree in the "Call Hierarchy" window.
+ * A Callee tree in the "Call Hierarchy" window.
+ *
* @author Shahar Ariel
*/
public class SdCalleeTreeStructure extends SdCallTreeStructure {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java
index 1ab2f6613d9..444e71978c9 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdCallerTreeStructure.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdCallerTreeStructure.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.openapi.progress.ProgressManager;
import com.intellij.openapi.project.Project;
@@ -9,8 +9,8 @@ import com.intellij.psi.search.SearchScope;
import com.intellij.psi.search.searches.ReferencesSearch;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.containers.ContainerUtil;
-import org.intellij.sdk.language.psi.SdFirstPhaseDefinition;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdFirstPhaseDefinition;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
import org.jetbrains.annotations.NotNull;
import java.util.HashMap;
@@ -19,7 +19,8 @@ import java.util.List;
import java.util.function.Consumer;
/**
- * This class represents a Caller tree in the "Call Hierarchy" window.
+ * A Caller tree in the "Call Hierarchy" window.
+ *
* @author Shahar Ariel
*/
public class SdCallerTreeStructure extends SdCallTreeStructure {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java
index 19f9d72f6f4..dc925f0f369 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/hierarchy/SdHierarchyUtil.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/hierarchy/SdHierarchyUtil.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.hierarchy;
+package ai.vespa.intellij.schema.hierarchy;
import com.intellij.ide.hierarchy.HierarchyBrowserManager;
import com.intellij.ide.util.treeView.AlphaComparator;
@@ -8,16 +8,17 @@ import com.intellij.openapi.project.Project;
import com.intellij.psi.*;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.psi.SdFile;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.SdUtil;
+import ai.vespa.intellij.schema.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
import java.util.Comparator;
import java.util.HashSet;
/**
- * This class is util class to the Call Hierarchy feature.
+ * Call Hierarchy feature utilities.
+ *
* @author Shahar Ariel
*/
public class SdHierarchyUtil {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/SdLexerAdapter.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/lexer/SdLexerAdapter.java
index c7463fc9823..8b474dcff69 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/SdLexerAdapter.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/lexer/SdLexerAdapter.java
@@ -1,10 +1,11 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.lexer;
+package ai.vespa.intellij.schema.lexer;
import com.intellij.lexer.FlexAdapter;
/**
- * This class adapts the JFlex lexer to the IntelliJ Platform Lexer API.
+ * Adapter of the JFlex lexer to the IntelliJ Platform Lexer API.
+ *
* @author Shahar Ariel
*/
public class SdLexerAdapter extends FlexAdapter {
@@ -12,4 +13,5 @@ public class SdLexerAdapter extends FlexAdapter {
public SdLexerAdapter() {
super(new SdLexer(null));
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java
index 0497f0b4960..d5b76c039d0 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/parser/SdParserDefinition.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/parser/SdParserDefinition.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.parser;
+package ai.vespa.intellij.schema.parser;
import com.intellij.lang.ASTNode;
import com.intellij.lang.ParserDefinition;
@@ -12,17 +12,19 @@ import com.intellij.psi.PsiFile;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IFileElementType;
import com.intellij.psi.tree.TokenSet;
-import org.intellij.sdk.language.SdLanguage;
-import org.intellij.sdk.language.lexer.SdLexerAdapter;
-import org.intellij.sdk.language.psi.SdFile;
-import org.intellij.sdk.language.psi.SdTypes;
+import ai.vespa.intellij.schema.SdLanguage;
+import ai.vespa.intellij.schema.lexer.SdLexerAdapter;
+import ai.vespa.intellij.schema.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdTypes;
import org.jetbrains.annotations.NotNull;
/**
* This class is used for the extension (in plugin.xml), to make the parsing process use the plugin code.
+ *
* @author Shahar Ariel
*/
public class SdParserDefinition implements ParserDefinition {
+
public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE);
public static final TokenSet COMMENTS = TokenSet.create(SdTypes.COMMENT);
public static final TokenSet STRINGS = TokenSet.create(SdTypes.STRING_REG);
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclaration.java
index 4f11a2b4b67..c2880181d66 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclaration.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclaration.java
@@ -1,11 +1,12 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.navigation.NavigationItem;
import com.intellij.psi.PsiElement;
/**
- * This interface represents a declaration in the SD language.
+ * A declaration in the SD language.
+ *
* @author Shahar Ariel
*/
public interface SdDeclaration extends PsiElement, NavigationItem, SdNamedElement {}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclarationType.java
index c19e21d0f19..129a32153f0 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdDeclarationType.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdDeclarationType.java
@@ -1,11 +1,13 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
/**
* This Enum describes the different declarations' types and their names.
+ *
* @author Shahar Ariel
*/
public enum SdDeclarationType {
+
DOCUMENT("Document"),
STRUCT("Struct"),
ANNOTATION("Annotation"),
@@ -31,4 +33,5 @@ public enum SdDeclarationType {
public String toString() {
return typeName;
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java
index 76c4454b3b2..7cd2d7f9331 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementDescriptionProvider.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementDescriptionProvider.java
@@ -1,22 +1,24 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.psi.ElementDescriptionLocation;
import com.intellij.psi.ElementDescriptionProvider;
import com.intellij.psi.PsiElement;
-import org.intellij.sdk.language.psi.impl.SdNamedElementImpl;
+import ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml), to enable "find Usages" window take the element description from
* here. Used only for the "target" element.
+ *
* @author Shahar Ariel
*/
public class SdElementDescriptionProvider implements ElementDescriptionProvider {
/**
- * Controls the headline of the element in the "Find Usages" window
+ * Controls the headline of the element in the "Find Usages" window.
+ *
* @param psiElement the element to describe
* @return a string with the description to write in the headline
*/
@@ -29,4 +31,5 @@ public class SdElementDescriptionProvider implements ElementDescriptionProvider
return "";
}
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementFactory.java
index 1f3b98b6012..d6cbcece6d5 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementFactory.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementFactory.java
@@ -1,14 +1,15 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.openapi.project.Project;
import com.intellij.psi.PsiFileFactory;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.SdFileType;
+import ai.vespa.intellij.schema.SdFileType;
/**
* This class is a factory of psi elements in the SD PSI tree.
+ *
* @author Shahar Ariel
*/
public class SdElementFactory {
@@ -32,4 +33,5 @@ public class SdElementFactory {
return (SdFile) PsiFileFactory.getInstance(project).
createFileFromText(name, SdFileType.INSTANCE, text);
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java
index b26cd66455d..e8154bce8ad 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdElementType.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdElementType.java
@@ -1,13 +1,14 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.psi.tree.IElementType;
-import org.intellij.sdk.language.SdLanguage;
+import ai.vespa.intellij.schema.SdLanguage;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
/**
- * This class represent a SdElementType.
+ * An SdElementType.
+ *
* @author Shahar Ariel
*/
public class SdElementType extends IElementType {
@@ -15,4 +16,5 @@ public class SdElementType extends IElementType {
public SdElementType(@NotNull @NonNls String debugName) {
super(debugName, SdLanguage.INSTANCE);
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java
index c2d9249bce1..1260883c96f 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFile.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFile.java
@@ -1,15 +1,16 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.extapi.psi.PsiFileBase;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.psi.FileViewProvider;
-import org.intellij.sdk.language.SdFileType;
-import org.intellij.sdk.language.SdLanguage;
+import ai.vespa.intellij.schema.SdFileType;
+import ai.vespa.intellij.schema.SdLanguage;
import org.jetbrains.annotations.NotNull;
/**
- * This class represent an SD file.
+ * An SD file.
+ *
* @author Shahar Ariel
*/
public class SdFile extends PsiFileBase {
@@ -28,4 +29,5 @@ public class SdFile extends PsiFileBase {
public String toString() {
return "Sd File";
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java
index 78d964b45c9..e7fa94cf81c 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdFunctionDefinitionInterface.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdFunctionDefinitionInterface.java
@@ -1,13 +1,15 @@
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.SdUtil;
+import ai.vespa.intellij.schema.SdUtil;
/**
- * This interface represents a function's declaration in the SD language.
+ * A function's declaration in the SD language.
+ *
* @author Shahar Ariel
*/
public interface SdFunctionDefinitionInterface extends SdDeclaration {
+
default boolean isOverride() {
String macroName = this.getName();
@@ -24,6 +26,6 @@ public interface SdFunctionDefinitionInterface extends SdDeclaration {
curRankProfile = (SdRankProfileDefinition) SdUtil.getRankProfileParent(curRankProfile);
}
return false;
-
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdIdentifier.java
index e20eb4cb6c4..1ac88cb3acd 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdIdentifier.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdIdentifier.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.psi.PsiElement;
import com.intellij.psi.util.PsiTreeUtil;
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdNamedElement.java
index 6e56f548410..0cd7cedc967 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdNamedElement.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdNamedElement.java
@@ -1,11 +1,12 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.psi.PsiNameIdentifierOwner;
/**
* This interface is used to wrap a Psi Element with SdNamedElement interface, which enables the element to be a
* "name owner" (like an identifier). It allows the element to take a part in references, find usages and more.
+ *
* @author Shahar Ariel
*/
public interface SdNamedElement extends PsiNameIdentifierOwner {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java
index 1024995832d..e42906530b0 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/SdTokenType.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/SdTokenType.java
@@ -1,13 +1,14 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi;
+package ai.vespa.intellij.schema.psi;
import com.intellij.psi.tree.IElementType;
-import org.intellij.sdk.language.SdLanguage;
+import ai.vespa.intellij.schema.SdLanguage;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
/**
- * This class represent a SdTokenType.
+ * An SdTokenType.
+ *
* @author Shahar Ariel
*/
public class SdTokenType extends IElementType {
@@ -20,4 +21,5 @@ public class SdTokenType extends IElementType {
public String toString() {
return "SdTokenType." + super.toString();
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java
index 1a13318a38f..43c02ea4fcf 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdFirstPhaseDefinitionMixin.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdFirstPhaseDefinitionMixin.java
@@ -1,11 +1,12 @@
-package org.intellij.sdk.language.psi.impl;
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.psi.impl;
import com.intellij.extapi.psi.ASTWrapperPsiElement;
import com.intellij.lang.ASTNode;
import com.intellij.navigation.ItemPresentation;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.SdIcons;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.SdIcons;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -14,6 +15,7 @@ import javax.swing.Icon;
/**
* This class is used for methods' implementations for SdFirstPhaseDefinition. Connected with "mixin" to
* FirstPhaseDefinition rule in sd.bnf
+ *
* @author Shahar Ariel
*/
public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement {
@@ -56,3 +58,4 @@ public class SdFirstPhaseDefinitionMixin extends ASTWrapperPsiElement {
};
}
}
+
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java
index 536dfc4f2cc..897217c082c 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixin.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixin.java
@@ -1,16 +1,18 @@
-package org.intellij.sdk.language.psi.impl;
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.psi.impl;
import com.intellij.lang.ASTNode;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiNamedElement;
-import org.intellij.sdk.language.psi.SdElementFactory;
-import org.intellij.sdk.language.psi.SdIdentifier;
-import org.intellij.sdk.language.psi.SdIdentifierVal;
+import ai.vespa.intellij.schema.psi.SdElementFactory;
+import ai.vespa.intellij.schema.psi.SdIdentifier;
+import ai.vespa.intellij.schema.psi.SdIdentifierVal;
import org.jetbrains.annotations.NotNull;
/**
* This abstract class is used for methods' implementations for SdIdentifier. Connected with "mixin" to IdentifierVal and
* IdentifierWithDashVal rules in sd.bnf
+ *
* @author Shahar Ariel
*/
public abstract class SdIdentifierMixin extends SdIdentifierMixinImpl implements PsiNamedElement {
@@ -41,6 +43,4 @@ public abstract class SdIdentifierMixin extends SdIdentifierMixinImpl implements
return this;
}
-
-
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java
index 7b738cc1b0f..3f69952d0b6 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdIdentifierMixinImpl.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdIdentifierMixinImpl.java
@@ -1,15 +1,17 @@
-package org.intellij.sdk.language.psi.impl;
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.psi.impl;
import com.intellij.extapi.psi.ASTWrapperPsiElement;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiReference;
-import org.intellij.sdk.language.SdReference;
-import org.intellij.sdk.language.psi.SdIdentifier;
+import ai.vespa.intellij.schema.SdReference;
+import ai.vespa.intellij.schema.psi.SdIdentifier;
import org.jetbrains.annotations.NotNull;
/**
* This class is used for methods' implementations for SdIdentifier. The abstract class SdIdentifierMixin extents it.
+ *
* @author Shahar Ariel
*/
public class SdIdentifierMixinImpl extends ASTWrapperPsiElement implements SdIdentifier {
@@ -22,4 +24,5 @@ public class SdIdentifierMixinImpl extends ASTWrapperPsiElement implements SdIde
public PsiReference getReference() {
return new SdReference(this, new TextRange(0, getNode().getText().length()));
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java
index 2c2a7aae3b1..ee328f43075 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdNamedElementImpl.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdNamedElementImpl.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.psi.impl;
+package ai.vespa.intellij.schema.psi.impl;
import com.intellij.extapi.psi.ASTWrapperPsiElement;
import com.intellij.icons.AllIcons;
@@ -7,29 +7,29 @@ import com.intellij.lang.ASTNode;
import com.intellij.navigation.ItemPresentation;
import com.intellij.psi.PsiElement;
import com.intellij.psi.util.PsiTreeUtil;
-import org.intellij.sdk.language.SdIcons;
-import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.psi.SdAnnotationFieldDefinition;
-import org.intellij.sdk.language.psi.SdArgumentDefinition;
-import org.intellij.sdk.language.psi.SdDeclarationType;
-import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition;
-import org.intellij.sdk.language.psi.SdDocumentDefinition;
-import org.intellij.sdk.language.psi.SdDocumentFieldDefinition;
-import org.intellij.sdk.language.psi.SdDocumentStructDefinition;
-import org.intellij.sdk.language.psi.SdDocumentStructFieldDefinition;
-import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition;
-import org.intellij.sdk.language.psi.SdElementFactory;
-import org.intellij.sdk.language.psi.SdFunctionDefinition;
-import org.intellij.sdk.language.psi.SdIdentifier;
-import org.intellij.sdk.language.psi.SdImportFieldDefinition;
-import org.intellij.sdk.language.psi.SdItemRawScoreDefinition;
-import org.intellij.sdk.language.psi.SdNamedElement;
-import org.intellij.sdk.language.psi.SdQueryDefinition;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
-import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition;
-import org.intellij.sdk.language.psi.SdSchemaFieldDefinition;
-import org.intellij.sdk.language.psi.SdStructFieldDefinition;
-import org.intellij.sdk.language.psi.SdTypes;
+import ai.vespa.intellij.schema.SdIcons;
+import ai.vespa.intellij.schema.SdUtil;
+import ai.vespa.intellij.schema.psi.SdAnnotationFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdArgumentDefinition;
+import ai.vespa.intellij.schema.psi.SdDeclarationType;
+import ai.vespa.intellij.schema.psi.SdDocumentAnnotationDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentStructDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentStructFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition;
+import ai.vespa.intellij.schema.psi.SdElementFactory;
+import ai.vespa.intellij.schema.psi.SdFunctionDefinition;
+import ai.vespa.intellij.schema.psi.SdIdentifier;
+import ai.vespa.intellij.schema.psi.SdImportFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdItemRawScoreDefinition;
+import ai.vespa.intellij.schema.psi.SdNamedElement;
+import ai.vespa.intellij.schema.psi.SdQueryDefinition;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition;
+import ai.vespa.intellij.schema.psi.SdSchemaFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdStructFieldDefinition;
+import ai.vespa.intellij.schema.psi.SdTypes;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -200,4 +200,5 @@ public abstract class SdNamedElementImpl extends ASTWrapperPsiElement implements
}
};
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java
index 085b1e04102..71d93641939 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/psi/impl/SdSummaryDefinitionMixin.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/psi/impl/SdSummaryDefinitionMixin.java
@@ -1,10 +1,11 @@
-package org.intellij.sdk.language.psi.impl;
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.psi.impl;
import com.intellij.extapi.psi.ASTWrapperPsiElement;
import com.intellij.lang.ASTNode;
import com.intellij.navigation.ItemPresentation;
-import org.intellij.sdk.language.SdIcons;
-import org.intellij.sdk.language.psi.SdTypes;
+import ai.vespa.intellij.schema.SdIcons;
+import ai.vespa.intellij.schema.psi.SdTypes;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -13,6 +14,7 @@ import javax.swing.Icon;
/**
* This class is used for methods' implementations for SdSummaryDefinition. Connected with "mixin" to SummaryDefinition
* rule in sd.bnf
+ *
* @author Shahar Ariel
*/
public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement {
@@ -54,4 +56,5 @@ public abstract class SdSummaryDefinitionMixin extends ASTWrapperPsiElement {
}
};
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java
index 67bda9ced19..76662c5cd31 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewElement.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewElement.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.structure;
+package ai.vespa.intellij.schema.structure;
import com.intellij.ide.projectView.PresentationData;
import com.intellij.ide.structureView.StructureViewTreeElement;
@@ -8,14 +8,14 @@ import com.intellij.ide.util.treeView.smartTree.TreeElement;
import com.intellij.navigation.ItemPresentation;
import com.intellij.psi.NavigatablePsiElement;
import com.intellij.psi.PsiElement;
-import org.intellij.sdk.language.SdUtil;
-import org.intellij.sdk.language.psi.SdDocumentAnnotationDefinition;
-import org.intellij.sdk.language.psi.SdDocumentDefinition;
-import org.intellij.sdk.language.psi.SdDocumentStructDefinition;
-import org.intellij.sdk.language.psi.SdDocumentSummaryDefinition;
-import org.intellij.sdk.language.psi.SdFile;
-import org.intellij.sdk.language.psi.SdRankProfileDefinition;
-import org.intellij.sdk.language.psi.SdSchemaAnnotationDefinition;
+import ai.vespa.intellij.schema.SdUtil;
+import ai.vespa.intellij.schema.psi.SdDocumentAnnotationDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentStructDefinition;
+import ai.vespa.intellij.schema.psi.SdDocumentSummaryDefinition;
+import ai.vespa.intellij.schema.psi.SdFile;
+import ai.vespa.intellij.schema.psi.SdRankProfileDefinition;
+import ai.vespa.intellij.schema.psi.SdSchemaAnnotationDefinition;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
@@ -23,6 +23,7 @@ import java.util.List;
/**
* This class is used for the presentation of an element in the "Structure View" window.
+ *
* @author Shahar Ariel
*/
public class SdStructureViewElement implements StructureViewTreeElement, SortableTreeElement {
@@ -91,4 +92,5 @@ public class SdStructureViewElement implements StructureViewTreeElement, Sortabl
}
return treeElements.toArray(new TreeElement[0]);
}
+
}
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java
index 2e902beb2ad..3d7b3a1a275 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewFactory.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewFactory.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.structure;
+package ai.vespa.intellij.schema.structure;
import com.intellij.ide.structureView.StructureViewBuilder;
import com.intellij.ide.structureView.StructureViewModel;
@@ -13,6 +13,7 @@ import org.jetbrains.annotations.Nullable;
/**
* This class is used for the extension (in plugin.xml) to the class SdStructureViewModel. It make the IDE use our
* plugin's code when creating the "Structure View" window.
+ *
* @author Shahar Ariel
*/
public class SdStructureViewFactory implements PsiStructureViewFactory {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java
index 00cf18565aa..1f1c271dcf6 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/structure/SdStructureViewModel.java
+++ b/integration/intellij/src/main/java/ai/vespa/intellij/schema/structure/SdStructureViewModel.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package org.intellij.sdk.language.structure;
+package ai.vespa.intellij.schema.structure;
import com.intellij.ide.structureView.StructureViewModel;
import com.intellij.ide.structureView.StructureViewModelBase;
@@ -10,6 +10,8 @@ import org.jetbrains.annotations.NotNull;
/**
* This class represents the "Structure View" window.
+ *
+ * @author Shahar Ariel
*/
public class SdStructureViewModel extends StructureViewModelBase implements StructureViewModel.ElementInfoProvider {
public SdStructureViewModel(PsiFile psiFile) {
diff --git a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex b/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex
index 585a6672189..78334de1961 100644
--- a/sd-plugin/src/main/java/org/intellij/sdk/language/lexer/sd.flex
+++ b/integration/intellij/src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex
@@ -1,19 +1,21 @@
-package org.intellij.sdk.language.lexer;
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.intellij.schema.lexer;
import com.intellij.lexer.FlexLexer;
import com.intellij.psi.tree.IElementType;
import com.intellij.ui.components.MultiColumnList;
-import org.intellij.sdk.language.psi.SdTokenType;
+import ai.vespa.intellij.schema.psi.SdTokenType;
-import static org.intellij.sdk.language.psi.SdTypes.*; // That is the class which is specified as `elementTypeHolderClass` in bnf
+import static ai.vespa.intellij.schema.psi.SdTypes.*; // That is the class which is specified as `elementTypeHolderClass` in bnf
// grammar file. This will contain all other tokens which we will use.
import static com.intellij.psi.TokenType.BAD_CHARACTER; // Pre-defined bad character token.
import static com.intellij.psi.TokenType.WHITE_SPACE; // Pre-defined whitespace character token.
/*
-* This file is used for the generation of the lexer of the SD language.
-* @author Shahar Ariel
-*/
+ * Vespa schema parser lexer
+ *
+ * @author Shahar Ariel
+ */
%%
@@ -36,7 +38,7 @@ SYMBOL= [!$|:{}(),.\[\]]
INTEGER = [0-9]+
FLOAT = {INTEGER}[.][0-9]+[e]?
COMPARISON_OPERATOR = [<>]|(==)|(<=)|(>=)|(\~=)
-ARITHMETIC_OPERATOR = [\-+*/]
+ARITHMETIC_OPERATOR = [\-+*/%]
STRING = \"([^\"\\]*(\\.[^\"\\]*)*)\"
WORD = \w+
diff --git a/integration/intellij/src/main/resources/META-INF/plugin.xml b/integration/intellij/src/main/resources/META-INF/plugin.xml
new file mode 100644
index 00000000000..49db6c59b3e
--- /dev/null
+++ b/integration/intellij/src/main/resources/META-INF/plugin.xml
@@ -0,0 +1,41 @@
+<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<idea-plugin>
+ <id>ai.vespa</id>
+ <name>Vespa</name>
+
+ <!-- Text to display as company information on Preferences/Settings | Plugin page -->
+ <vendor>vespa.ai</vendor>
+
+ <!-- Product and plugin compatibility requirements -->
+ <depends>com.intellij.java</depends>
+ <depends>com.intellij.modules.platform</depends>
+
+ <!-- Text to display as description on Preferences/Settings | Plugin page -->
+ <description><![CDATA[
+ <p>Vespa.ai schema file support</p>
+ ]]></description>
+
+ <!-- Extension points defined by the plugin -->
+ <extensions defaultExtensionNs="com.intellij">
+ <fileType name="Sd File" implementationClass="ai.vespa.intellij.schema.SdFileType" fieldName="INSTANCE"
+ language="Sd" extensions="sd"/>
+ <lang.parserDefinition language="Sd" implementationClass="ai.vespa.intellij.schema.parser.SdParserDefinition"/>
+ <lang.syntaxHighlighterFactory language="Sd" implementationClass="ai.vespa.intellij.schema.SdSyntaxHighlighterFactory"/>
+ <completion.contributor language="Sd" implementationClass="ai.vespa.intellij.schema.SdCompletionContributor"/>
+
+ <lang.findUsagesProvider language="Sd" implementationClass="ai.vespa.intellij.schema.findUsages.SdFindUsagesProvider"/>
+ <findUsagesHandlerFactory implementation="ai.vespa.intellij.schema.findUsages.SdFindUsagesHandlerFactory"/>
+ <fileStructureGroupRuleProvider implementation="ai.vespa.intellij.schema.findUsages.SdRankProfileGroupingRuleProvider"/>
+ <fileStructureGroupRuleProvider implementation="ai.vespa.intellij.schema.findUsages.SdDocumentSummaryGroupingRuleProvider"/>
+ <elementDescriptionProvider implementation="ai.vespa.intellij.schema.psi.SdElementDescriptionProvider"/>
+
+ <lang.psiStructureViewFactory language="Sd" implementationClass="ai.vespa.intellij.schema.structure.SdStructureViewFactory"/>
+ <codeStyleSettingsProvider implementation="ai.vespa.intellij.schema.SdCodeStyleSettingsProvider"/>
+ <langCodeStyleSettingsProvider implementation="ai.vespa.intellij.schema.SdLanguageCodeStyleSettingsProvider"/>
+ <lang.commenter language="Sd" implementationClass="ai.vespa.intellij.schema.SdCommenter"/>
+ <lang.refactoringSupport language="Sd" implementationClass="ai.vespa.intellij.schema.SdRefactoringSupportProvider"/>
+ <gotoSymbolContributor implementation="ai.vespa.intellij.schema.SdChooseByNameContributor"/>
+ <callHierarchyProvider language="Sd" implementationClass="ai.vespa.intellij.schema.hierarchy.SdCallHierarchyProvider"/>
+ </extensions>
+
+</idea-plugin> \ No newline at end of file
diff --git a/integration/intellij/src/main/resources/META-INF/pluginIcon.svg b/integration/intellij/src/main/resources/META-INF/pluginIcon.svg
new file mode 100644
index 00000000000..3a795976efd
--- /dev/null
+++ b/integration/intellij/src/main/resources/META-INF/pluginIcon.svg
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg id="Layer_1" data-name="Layer 1" viewBox="0 0 40 40" preserveAspectRatio="xMidYMin slice" xmlns="http://www.w3.org/2000/svg">
+ <defs>
+ <style>.cls-1{fill:#ffc43c;}.cls-2{fill:url(#linear-gradient);}.cls-3{fill:#1a7db6;}.cls-4{fill:url(#linear-gradient-2);}.cls-5{fill:#303030;}</style>
+ <linearGradient id="linear-gradient" x1="964.75" y1="34.82" x2="919.68" y2="63.4" gradientTransform="matrix(-0.276923, 0, 0, 0.276923, 291.653198, 2)" gradientUnits="userSpaceOnUse">
+ <stop offset="0" stop-color="#da9728"/>
+ <stop offset="0.54" stop-color="#ffc43c"/>
+ </linearGradient>
+ <linearGradient id="linear-gradient-2" x1="46.38" y1="94.56" x2="-29.88" y2="53" gradientUnits="userSpaceOnUse" gradientTransform="matrix(0.276923, 0, 0, 0.276923, 3.212999, 2)">
+ <stop offset="0" stop-color="#005a8e"/>
+ <stop offset="0.54" stop-color="#1a7db6"/>
+ </linearGradient>
+ </defs>
+ <title>VespaLogo copy</title>
+ <polygon class="cls-1" points="22.736 2 8.685 11.543 8.685 27.971 22.736 18.064 36.787 27.971 36.787 11.543 22.736 2"/>
+ <polygon class="cls-2" points="22.736 18.064 22.736 2 36.787 11.543 36.787 27.97 22.736 18.064"/>
+ <polygon class="cls-3" points="17.264 21.936 3.213 12.029 3.213 28.457 17.264 38 31.315 28.457 31.315 12.029 17.264 21.936"/>
+ <polygon class="cls-4" points="17.264 21.936 17.264 38 3.213 28.457 3.213 12.03 17.264 21.936"/>
+</svg> \ No newline at end of file
diff --git a/sd-plugin/src/main/resources/icons/document_icon.png b/integration/intellij/src/main/resources/icons/document_icon.png
index 497c218ab91..497c218ab91 100644
--- a/sd-plugin/src/main/resources/icons/document_icon.png
+++ b/integration/intellij/src/main/resources/icons/document_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/document_summary_icon.png b/integration/intellij/src/main/resources/icons/document_summary_icon.png
index 72f42c68902..72f42c68902 100644
--- a/sd-plugin/src/main/resources/icons/document_summary_icon.png
+++ b/integration/intellij/src/main/resources/icons/document_summary_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/first_phase_icon.png b/integration/intellij/src/main/resources/icons/first_phase_icon.png
index 18f24e9e462..18f24e9e462 100644
--- a/sd-plugin/src/main/resources/icons/first_phase_icon.png
+++ b/integration/intellij/src/main/resources/icons/first_phase_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/imported_field_icon.png b/integration/intellij/src/main/resources/icons/imported_field_icon.png
index 1ec9b1d6a96..1ec9b1d6a96 100644
--- a/sd-plugin/src/main/resources/icons/imported_field_icon.png
+++ b/integration/intellij/src/main/resources/icons/imported_field_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/macro_icon.png b/integration/intellij/src/main/resources/icons/macro_icon.png
index 736d8168119..736d8168119 100644
--- a/sd-plugin/src/main/resources/icons/macro_icon.png
+++ b/integration/intellij/src/main/resources/icons/macro_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/override_macro_icon.png b/integration/intellij/src/main/resources/icons/override_macro_icon.png
index 024eb777eef..024eb777eef 100644
--- a/sd-plugin/src/main/resources/icons/override_macro_icon.png
+++ b/integration/intellij/src/main/resources/icons/override_macro_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/sd_icon.png b/integration/intellij/src/main/resources/icons/sd_icon.png
index 1c96aaee633..1c96aaee633 100644
--- a/sd-plugin/src/main/resources/icons/sd_icon.png
+++ b/integration/intellij/src/main/resources/icons/sd_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/struct_field_icon.png b/integration/intellij/src/main/resources/icons/struct_field_icon.png
index 7f5074630d4..7f5074630d4 100644
--- a/sd-plugin/src/main/resources/icons/struct_field_icon.png
+++ b/integration/intellij/src/main/resources/icons/struct_field_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/struct_icon.png b/integration/intellij/src/main/resources/icons/struct_icon.png
index c2f6fdc8440..c2f6fdc8440 100644
--- a/sd-plugin/src/main/resources/icons/struct_icon.png
+++ b/integration/intellij/src/main/resources/icons/struct_icon.png
Binary files differ
diff --git a/sd-plugin/src/main/resources/icons/summary_def_icon.png b/integration/intellij/src/main/resources/icons/summary_def_icon.png
index a8faf12601f..a8faf12601f 100644
--- a/sd-plugin/src/main/resources/icons/summary_def_icon.png
+++ b/integration/intellij/src/main/resources/icons/summary_def_icon.png
Binary files differ
diff --git a/parent/pom.xml b/parent/pom.xml
index eac9cf53d71..22925af4502 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -874,7 +874,7 @@
<aircompressor.version>0.21</aircompressor.version>
<airline.version>0.7</airline.version>
<antlr.version>3.5.2</antlr.version>
- <antlr4.version>4.9.3</antlr4.version>
+ <antlr4.version>4.5</antlr4.version>
<apache.httpclient.version>4.5.13</apache.httpclient.version>
<apache.httpcore.version>4.4.13</apache.httpcore.version>
<apache.httpclient5.version>5.1.1</apache.httpclient5.version>
@@ -895,6 +895,7 @@
<curator.version>5.2.0</curator.version>
<commons.codec.version>1.15</commons.codec.version>
<commons.math3.version>3.6.1</commons.math3.version>
+ <gson.version>2.8.9</gson.version>
<jna.version>5.9.0</jna.version>
<junit.version>5.7.0</junit.version>
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
@@ -917,7 +918,7 @@
<protobuf.version>3.11.4</protobuf.version>
<spifly.version>1.3.3</spifly.version>
<surefire.version>2.22.2</surefire.version>
- <zookeeper.client.version>3.6.3</zookeeper.client.version>
+ <zookeeper.client.version>3.7.0</zookeeper.client.version>
<doclint>all</doclint>
<test.hide>true</test.hide>
diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp
index 495522d3cf5..810f2ad2356 100644
--- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp
+++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp
@@ -3,13 +3,13 @@
#include <vespa/document/base/testdocman.h>
#include <vespa/persistence/conformancetest/conformancetest.h>
#include <vespa/persistence/spi/test.h>
+#include <vespa/persistence/spi/catchresult.h>
#include <vespa/persistence/spi/resource_usage_listener.h>
#include <vespa/document/fieldset/fieldsets.h>
#include <vespa/document/update/documentupdate.h>
#include <vespa/document/update/assignvalueupdate.h>
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/test/make_bucket_space.h>
-#include <vespa/document/util/bytebuffer.h>
#include <vespa/vdslib/state/state.h>
#include <vespa/vdslib/state/node.h>
#include <vespa/vdslib/state/nodestate.h>
@@ -18,7 +18,6 @@
#include <vespa/vespalib/util/idestructorcallback.h>
#include <vespa/vespalib/util/size_literals.h>
#include <vespa/config-stor-distribution.h>
-#include <algorithm>
#include <limits>
#include <gtest/gtest.h>
@@ -796,6 +795,40 @@ TEST_F(ConformanceTest, testRemove)
EXPECT_FALSE(getResult.hasDocument());
}
+TEST_F(ConformanceTest, testRemoveMulti)
+{
+ document::TestDocMan testDocMan;
+ _factory->clear();
+ PersistenceProviderUP spi(getSpi(*_factory, testDocMan));
+
+ BucketId bucketId1(8, 0x01);
+ Bucket bucket1(makeSpiBucket(bucketId1));
+ Context context(Priority(0), Trace::TraceLevel(0));
+ spi->createBucket(bucket1, context);
+
+ std::vector<Document::SP> docs;
+ for (size_t i(0); i < 30; i++) {
+ docs.push_back(testDocMan.createRandomDocumentAtLocation(0x01, i));
+ }
+
+ std::vector<PersistenceProvider::TimeStampAndDocumentId> ids;
+ for (size_t i(0); i < docs.size(); i++) {
+ spi->put(bucket1, Timestamp(i), docs[i], context);
+ if (i & 0x1) {
+ ids.emplace_back(Timestamp(i), docs[i]->getId());
+ }
+ }
+
+ auto onDone = std::make_unique<CatchResult>();
+ auto future = onDone->future_result();
+ spi->removeAsync(bucket1, ids, context, std::move(onDone));
+ auto result = future.get();
+ ASSERT_TRUE(result);
+ auto removeResult = dynamic_cast<spi::RemoveResult *>(result.get());
+ ASSERT_TRUE(removeResult != nullptr);
+ EXPECT_EQ(15u, removeResult->num_removed());
+}
+
TEST_F(ConformanceTest, testRemoveMerge)
{
document::TestDocMan testDocMan;
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
index c7365f39f02..9d9f31b63a3 100644
--- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
+++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
@@ -471,30 +471,33 @@ DummyPersistence::updateAsync(const Bucket& bucket, Timestamp ts, DocumentUpdate
}
void
-DummyPersistence::removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context &, OperationComplete::UP onComplete)
+DummyPersistence::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, Context &, OperationComplete::UP onComplete)
{
DUMMYPERSISTENCE_VERIFY_INITIALIZED;
- LOG(debug, "remove(%s, %" PRIu64 ", %s)",
- b.toString().c_str(),
- uint64_t(t),
- did.toString().c_str());
assert(b.getBucketSpace() == FixedBucketSpaces::default_space());
-
BucketContentGuard::UP bc(acquireBucketWithLock(b));
- while (!bc) {
- internal_create_bucket(b);
- bc = acquireBucketWithLock(b);
- }
- DocEntry::SP entry((*bc)->getEntry(did));
- bool foundPut(entry.get() && !entry->isRemove());
- auto remEntry = std::make_unique<DocEntry>(t, REMOVE_ENTRY, did);
-
- if ((*bc)->hasTimestamp(t)) {
- (*bc)->eraseEntry(t);
+
+ uint32_t numRemoves(0);
+ for (const TimeStampAndDocumentId & stampedId : ids) {
+ const DocumentId & id = stampedId.second;
+ Timestamp t = stampedId.first;
+ LOG(debug, "remove(%s, %" PRIu64 ", %s)", b.toString().c_str(), uint64_t(t), id.toString().c_str());
+
+ while (!bc) {
+ internal_create_bucket(b);
+ bc = acquireBucketWithLock(b);
+ }
+ DocEntry::SP entry((*bc)->getEntry(id));
+ numRemoves += (entry.get() && !entry->isRemove()) ? 1 : 0;
+ auto remEntry = std::make_unique<DocEntry>(t, REMOVE_ENTRY, id);
+
+ if ((*bc)->hasTimestamp(t)) {
+ (*bc)->eraseEntry(t);
+ }
+ (*bc)->insert(std::move(remEntry));
}
- (*bc)->insert(std::move(remEntry));
bc.reset();
- onComplete->onComplete(std::make_unique<RemoveResult>(foundPut));
+ onComplete->onComplete(std::make_unique<RemoveResult>(numRemoves));
}
GetResult
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h
index 4e0d088d5d6..a7a784d0479 100644
--- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h
+++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h
@@ -159,7 +159,7 @@ public:
BucketInfoResult getBucketInfo(const Bucket&) const override;
GetResult get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const override;
void putAsync(const Bucket&, Timestamp, DocumentSP, Context&, OperationComplete::UP) override;
- void removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context&, OperationComplete::UP) override;
+ void removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP) override;
void updateAsync(const Bucket&, Timestamp, DocumentUpdateSP, Context&, OperationComplete::UP) override;
CreateIteratorResult
diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp
index 951dbf97cff..35654240ec7 100644
--- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp
+++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp
@@ -1,10 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "abstractpersistenceprovider.h"
-#include <vespa/document/datatype/documenttype.h>
-#include <vespa/document/update/documentupdate.h>
-#include <vespa/document/fieldset/fieldsets.h>
-#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/base/documentid.h>
#include <vespa/vespalib/util/idestructorcallback.h>
namespace storage::spi {
@@ -13,7 +10,9 @@ void
AbstractPersistenceProvider::removeIfFoundAsync(const Bucket& b, Timestamp timestamp,
const DocumentId& id, Context& context, OperationComplete::UP onComplete)
{
- removeAsync(b, timestamp, id, context, std::move(onComplete));
+ std::vector<TimeStampAndDocumentId> ids;
+ ids.emplace_back(timestamp, id);
+ removeAsync(b, std::move(ids), context, std::move(onComplete));
}
BucketIdListResult
diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp
index 31db08a6f4f..e6733dc4150 100644
--- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp
+++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp
@@ -2,6 +2,7 @@
#include "persistenceprovider.h"
#include "catchresult.h"
+#include <vespa/document/base/documentid.h>
#include <future>
namespace storage::spi {
@@ -44,7 +45,9 @@ RemoveResult
PersistenceProvider::remove(const Bucket& bucket, Timestamp timestamp, const DocumentId & docId, Context& context) {
auto catcher = std::make_unique<CatchResult>();
auto future = catcher->future_result();
- removeAsync(bucket, timestamp, docId, context, std::move(catcher));
+ std::vector<TimeStampAndDocumentId> ids;
+ ids.emplace_back(timestamp, docId);
+ removeAsync(bucket, std::move(ids), context, std::move(catcher));
return dynamic_cast<const RemoveResult &>(*future.get());
}
diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h
index 269175f7d26..8c62e691daf 100644
--- a/persistence/src/vespa/persistence/spi/persistenceprovider.h
+++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h
@@ -54,6 +54,7 @@ struct PersistenceProvider
{
using BucketSpace = document::BucketSpace;
using FieldSetSP = std::shared_ptr<document::FieldSet>;
+ using TimeStampAndDocumentId = std::pair<Timestamp, DocumentId>;
virtual ~PersistenceProvider();
@@ -168,7 +169,7 @@ struct PersistenceProvider
* @param timestamp The timestamp for the new bucket entry.
* @param id The ID to remove
*/
- virtual void removeAsync(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&, OperationComplete::UP) = 0;
+ virtual void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP) = 0;
/**
* @see remove()
diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h
index 73838e99a18..70bd37590a1 100644
--- a/persistence/src/vespa/persistence/spi/result.h
+++ b/persistence/src/vespa/persistence/spi/result.h
@@ -142,6 +142,7 @@ public:
: _numRemoved(numRemoved) { }
bool wasFound() const { return _numRemoved > 0; }
uint32_t num_removed() const { return _numRemoved; }
+ void inc_num_removed(uint32_t add) { _numRemoved += add; }
private:
uint32_t _numRemoved;
diff --git a/pom.xml b/pom.xml
index d43affb3565..9cf3630e463 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,6 +85,7 @@
<module>hosted-zone-api</module>
<module>http-utils</module>
<module>indexinglanguage</module>
+ <!--<module>integration/intellij</module>-->
<module>jaxrs_client_utils</module>
<module>jaxrs_utils</module>
<module>jdisc-cloud-aws</module>
diff --git a/sd-plugin/BACKLOG.md b/sd-plugin/BACKLOG.md
deleted file mode 100644
index 19e185dcc6b..00000000000
--- a/sd-plugin/BACKLOG.md
+++ /dev/null
@@ -1,34 +0,0 @@
-### Open Issues
-
-1. In some cases, the grammar prefers not to enforce bad syntax, because if the parser encounters bad syntax it stops
-and can't build the PSI tree. That means none of the features will work for a file like that. For example, in cases
-where an element supposes to have zero-to-one occurrences, the grammar will treat it as zero-to-many.
-2. In order to enable the grammar recognize some keywords as identifiers (e.g. "filter" as a field's name), the
-identifier rule (named "IdentifierVal") wraps the regex (ID_REG) and the KeywordOrIdentifier rule (which contains all
-the keywords in the language).
-3. The implementation of the GoTo Declaration feature is not exactly the same as IntelliJ. In IntelliJ if a reference
-has several declarations, after clicking "Goto Declaration" there is a little window with all the declarations to choose
-from. It can be done by changing the method "multiResolve" in SdReference.java to return more than one declaration. The
-problem with that is that it causes the "Find Usages" feature to not work. For now, I decided to make the plugin
-"Goto Declaration" feature show only the most specific declaration by the right rank-profile scope.
-4. The "Find Usages" window can group usages only under rank-profiles and document-summaries. Other usages appear
-directly under the .sd file. In order to create another group type of usages' group, you'll need to create 2 classes:
-one for the extension "fileStructureGroupRuleProvider" (e.g. SdRankProfileGroupingRuleProvider.java), and one for the
-grouping rule itself (e.g. SdRankProfileGroupingRule.java).
-Another open problem is that the navigation isn't working in the current grouping rules. It means that when clicking on
-the group headline (e.g. some name of a rank-profile) the IDE doesn't "jump" to the matching declaration.
-5. Goto declaration doesn't work for document's inherits. e.g. if document A inherits from document B, B doesn't have a
-reference to its declaration.
-6. There aren't any tests for the plugin.
-
-### Some useful links:
-1. JetBrains official tutorials: https://plugins.jetbrains.com/docs/intellij/custom-language-support.html and
-https://plugins.jetbrains.com/docs/intellij/custom-language-support-tutorial.html
-2. Grammar-Kit HOWTO: Helps to understand the BNF syntax.
- https://github.com/JetBrains/Grammar-Kit/blob/master/HOWTO.md
-3. How to deal with left-recursion in the grammar (in SD for example it happens in expressions). Last answer here:
-https://intellij-support.jetbrains.com/hc/en-us/community/posts/360001258300-What-s-the-alternative-to-left-recursion-in-GrammarKit-
-4. Great tutorial for a custom-language-plugin, but only for the basics (mainly the parser and lexer):
- https://medium.com/@shan1024/custom-language-plugin-development-for-intellij-idea-part-01-d6a41ab96bc9
-5. Code of Dart (some custom language) plugin for IntelliJ:
-https://github.com/JetBrains/intellij-plugins/tree/0f07ca63355d5530b441ca566c98f17c560e77f8/Dart \ No newline at end of file
diff --git a/sd-plugin/README.md b/sd-plugin/README.md
deleted file mode 100644
index 1cab356f78e..00000000000
--- a/sd-plugin/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-This directory holds the code for an IntelliJ plugin for reading SD files.
-
-NOTE: This is the source code, not the plugin itself. In order to be able to use the plugin you'll need to download it
-from JetBrains Marketplace or create a zip file and load it to IntelliJ by choosing "Install Plugin from Disk".
-
-Before starting, you should:
-1. Download Gradle 7 (if you don't have it already).
-2. Make sure that the bundled Plugin DevKit plugin is enabled (inside IntelliJ).
-3. Optional- Download Grammar-Kit plugin from JetBrains Marketplace (inside IntelliJ). It helps with reading the .bnf file.
-4. Optional- Download PsiViewer plugin from JetBrains Marketplace (inside IntelliJ). It helps to test the grammar defined
-in the .bnf file.
-
-### Working Process
-The grammar is defined in 2 files:
-- sd.bnf
-- sd.flex
-
-In order to generate the lexer and parser's code, you need to run in the command line:
-
- ./gradlew generateSdParser
- ./gradlew generateSdLexer
-
-You should now have a "gen" folder next to the "java" folder, and it contains all the parser and lexer code.
-
-NOTE- Running those tasks would reset the "gen" folder, and all the previous generated files would be deleted before the
-new ones would be generated.
-
-Now, you can run the gradle task "intellij/runIde" (or "./gradlew runIde" in the command line), open a project with some sd file and see how the plugin works on it.
-
-### Build the Plugin
-In order to build the plugin and create a zip file from it, you should run the command:
-
- ./gradlew buildPlugin
-
-Or since it's a default task you can just run:
-
- ./gradlew
-
-This task also invokes the tasks generateSdParser and generateSdLexer as a part of the building process.
-
-Now, you'll have a zip file in the directory build\distributions.
diff --git a/sd-plugin/gradle/wrapper/gradle-wrapper.jar b/sd-plugin/gradle/wrapper/gradle-wrapper.jar
deleted file mode 100644
index 7454180f2ae..00000000000
--- a/sd-plugin/gradle/wrapper/gradle-wrapper.jar
+++ /dev/null
Binary files differ
diff --git a/sd-plugin/gradle/wrapper/gradle-wrapper.properties b/sd-plugin/gradle/wrapper/gradle-wrapper.properties
deleted file mode 100644
index 69a9715077f..00000000000
--- a/sd-plugin/gradle/wrapper/gradle-wrapper.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
diff --git a/sd-plugin/gradlew b/sd-plugin/gradlew
deleted file mode 100755
index 744e882ed57..00000000000
--- a/sd-plugin/gradlew
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env sh
-
-#
-# Copyright 2015 the original author or authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##############################################################################
-##
-## Gradle start up script for UN*X
-##
-##############################################################################
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
- ls=`ls -ld "$PRG"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "$PRG"`"/$link"
- fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn () {
- echo "$*"
-}
-
-die () {
- echo
- echo "$*"
- echo
- exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "`uname`" in
- CYGWIN* )
- cygwin=true
- ;;
- Darwin* )
- darwin=true
- ;;
- MSYS* | MINGW* )
- msys=true
- ;;
- NONSTOP* )
- nonstop=true
- ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
- if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
- # IBM's JDK on AIX uses strange locations for the executables
- JAVACMD="$JAVA_HOME/jre/sh/java"
- else
- JAVACMD="$JAVA_HOME/bin/java"
- fi
- if [ ! -x "$JAVACMD" ] ; then
- die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
- fi
-else
- JAVACMD="java"
- which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
- MAX_FD_LIMIT=`ulimit -H -n`
- if [ $? -eq 0 ] ; then
- if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
- MAX_FD="$MAX_FD_LIMIT"
- fi
- ulimit -n $MAX_FD
- if [ $? -ne 0 ] ; then
- warn "Could not set maximum file descriptor limit: $MAX_FD"
- fi
- else
- warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
- fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
- GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin or MSYS, switch paths to Windows format before running java
-if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
- APP_HOME=`cygpath --path --mixed "$APP_HOME"`
- CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
- JAVACMD=`cygpath --unix "$JAVACMD"`
-
- # We build the pattern for arguments to be converted via cygpath
- ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
- SEP=""
- for dir in $ROOTDIRSRAW ; do
- ROOTDIRS="$ROOTDIRS$SEP$dir"
- SEP="|"
- done
- OURCYGPATTERN="(^($ROOTDIRS))"
- # Add a user-defined pattern to the cygpath arguments
- if [ "$GRADLE_CYGPATTERN" != "" ] ; then
- OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
- fi
- # Now convert the arguments - kludge to limit ourselves to /bin/sh
- i=0
- for arg in "$@" ; do
- CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
- CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
-
- if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
- eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
- else
- eval `echo args$i`="\"$arg\""
- fi
- i=`expr $i + 1`
- done
- case $i in
- 0) set -- ;;
- 1) set -- "$args0" ;;
- 2) set -- "$args0" "$args1" ;;
- 3) set -- "$args0" "$args1" "$args2" ;;
- 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
- 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
- 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
- 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
- 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
- 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
- esac
-fi
-
-# Escape application args
-save () {
- for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
- echo " "
-}
-APP_ARGS=`save "$@"`
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
-
-exec "$JAVACMD" "$@"
diff --git a/sd-plugin/gradlew.bat b/sd-plugin/gradlew.bat
deleted file mode 100644
index 107acd32c4e..00000000000
--- a/sd-plugin/gradlew.bat
+++ /dev/null
@@ -1,89 +0,0 @@
-@rem
-@rem Copyright 2015 the original author or authors.
-@rem
-@rem Licensed under the Apache License, Version 2.0 (the "License");
-@rem you may not use this file except in compliance with the License.
-@rem You may obtain a copy of the License at
-@rem
-@rem https://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-@rem
-
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Resolve any "." and ".." in APP_HOME to make it shorter.
-for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto execute
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
diff --git a/sd-plugin/settings.gradle b/sd-plugin/settings.gradle
deleted file mode 100644
index 5d47bd4197c..00000000000
--- a/sd-plugin/settings.gradle
+++ /dev/null
@@ -1,3 +0,0 @@
-// Copyright 2000-2020 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
-
-rootProject.name = 'sd_language_plugin'
diff --git a/sd-plugin/src/main/resources/META-INF/plugin.xml b/sd-plugin/src/main/resources/META-INF/plugin.xml
deleted file mode 100644
index dd268bec16f..00000000000
--- a/sd-plugin/src/main/resources/META-INF/plugin.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<idea-plugin>
- <id>org.intellij.sdk.language</id>
- <name>SdReader</name>
-
- <!-- Text to display as company information on Preferences/Settings | Plugin page -->
- <vendor>Yahoo</vendor>
-
- <!-- Product and plugin compatibility requirements -->
- <depends>com.intellij.java</depends>
- <depends>com.intellij.modules.platform</depends>
-
- <!-- Text to display as description on Preferences/Settings | Plugin page -->
- <description><![CDATA[
- <p>Support for reading Vespa's SD files.</p>
- <p>Features:</p>
- <ul>
- <li>Structure View</li>
- <li>Find Usages</li>
- <li>Call Hierarchy (for functions/macros)</li>
- <li>Go To Declaration</li>
- <li>Go to Symbol</li>
- <li>Refactoring (Only for uses of the same implementation)</li>
- <li>Syntax Highlighting</li>
- <li>Commenter- enable turning lines into a comment with "Code -> Comment with line comment"</li>
- </ul>
- <p>Find more information on Vespa's SD files <a href="https://docs.vespa.ai/en/reference/schema-reference.html">here</a>.</p>
- ]]></description>
-
- <!-- Extension points defined by the plugin -->
- <extensions defaultExtensionNs="com.intellij">
- <fileType name="Sd File" implementationClass="org.intellij.sdk.language.SdFileType" fieldName="INSTANCE"
- language="Sd" extensions="sd"/>
- <lang.parserDefinition language="Sd" implementationClass="org.intellij.sdk.language.parser.SdParserDefinition"/>
- <lang.syntaxHighlighterFactory language="Sd" implementationClass="org.intellij.sdk.language.SdSyntaxHighlighterFactory"/>
- <completion.contributor language="Sd" implementationClass="org.intellij.sdk.language.SdCompletionContributor"/>
-
- <lang.findUsagesProvider language="Sd" implementationClass="org.intellij.sdk.language.findUsages.SdFindUsagesProvider"/>
- <findUsagesHandlerFactory implementation="org.intellij.sdk.language.findUsages.SdFindUsagesHandlerFactory"/>
- <fileStructureGroupRuleProvider implementation="org.intellij.sdk.language.findUsages.SdRankProfileGroupingRuleProvider"/>
- <fileStructureGroupRuleProvider implementation="org.intellij.sdk.language.findUsages.SdDocumentSummaryGroupingRuleProvider"/>
- <elementDescriptionProvider implementation="org.intellij.sdk.language.psi.SdElementDescriptionProvider"/>
-
- <lang.psiStructureViewFactory language="Sd" implementationClass="org.intellij.sdk.language.structure.SdStructureViewFactory"/>
- <codeStyleSettingsProvider implementation="org.intellij.sdk.language.SdCodeStyleSettingsProvider"/>
- <langCodeStyleSettingsProvider implementation="org.intellij.sdk.language.SdLanguageCodeStyleSettingsProvider"/>
- <lang.commenter language="Sd" implementationClass="org.intellij.sdk.language.SdCommenter"/>
- <lang.refactoringSupport language="Sd" implementationClass="org.intellij.sdk.language.SdRefactoringSupportProvider"/>
- <gotoSymbolContributor implementation="org.intellij.sdk.language.SdChooseByNameContributor"/>
- <callHierarchyProvider language="Sd" implementationClass="org.intellij.sdk.language.hierarchy.SdCallHierarchyProvider"/>
- </extensions>
-
-</idea-plugin> \ No newline at end of file
diff --git a/sd-plugin/src/main/resources/META-INF/pluginIcon.svg b/sd-plugin/src/main/resources/META-INF/pluginIcon.svg
deleted file mode 100644
index caacd6a053e..00000000000
--- a/sd-plugin/src/main/resources/META-INF/pluginIcon.svg
+++ /dev/null
@@ -1,10 +0,0 @@
-<svg version="1.2" baseProfile="tiny-ps" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32">
- <title>New Project</title>
- <defs>
- <image width="30" height="29" id="img1" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAAdCAMAAACKeiw+AAAAAXNSR0IB2cksfwAAAUpQTFRFAAAAHKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0lOzY/wAAAG50Uk5TAA5Acarj/8dVMgdq1LgjXdzxzbAfO/iV1epHHGO/f9hcFZ3lQiR08oAPeL7CuWkIYKieoI8u9/n69en2/eAX6GY6rljt7MzmWv4ac5jwEO5RM7381k8skCnX3srvyDdB6/N8jNLnBfT7wXqSr7W47lQGAAABpElEQVR4nIVT6z8CQRS9+7qVNk3FKiqUIvJWbSLvjQh5hkgeef//X83s2rb6hfNhZvecmXvP3LkD8AOOF0QJKWyi3cFBO3ocThmbcDl7e1pVzk08Xp+9j333Cy6ieN2t8oA/IA42Iw4FQ+gPW2oIh0fa0o1GomjqnBujYx1mYvFxv9vIP+jFCejEaAi9vfpm0eOkexOTU8npmVmYm19YXEpRA6LCaHAQ4qDTUlrJqFkeJGU5t7KaBwiii/F29LEka+r6xmZ6C7Z3+uJaYZdG3UM7QFEMCCyHpwCQ2s9Tmf6sanQ4QLEIMRtJMTlbCh+yeYfJyhEdjtEWgzJqutUT9fTs3JRz24xCLDdlmCmkoxe/ywCXauQvGVQJDGuWXJGudGvXN9RR1ZAT8i1l7lCqNA+WVmoJtQRwX394fHrmzYM1BKMs2mlpLeMDkF+ymVeS18siNAD4gF5UOMnW3yj7/vGZ/KoBK6rMW1fSCfNKIEi6X6gR9L92AE7u1kwDFiNjZytiqGX1P40MXFAMYPRKoyBVJC2hzAV/PSKGRrFSpogVGxb3De/eO/7N6EMHAAAAAElFTkSuQmCC"/>
- </defs>
- <style>
- tspan { white-space:pre }
- </style>
- <use id="Layer 2" href="#img1" x="1" y="2" />
-</svg> \ No newline at end of file
diff --git a/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg b/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg
deleted file mode 100644
index caacd6a053e..00000000000
--- a/sd-plugin/src/main/resources/META-INF/pluginIcon_dark.svg
+++ /dev/null
@@ -1,10 +0,0 @@
-<svg version="1.2" baseProfile="tiny-ps" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32">
- <title>New Project</title>
- <defs>
- <image width="30" height="29" id="img1" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAAdCAMAAACKeiw+AAAAAXNSR0IB2cksfwAAAUpQTFRFAAAAHKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0HKy0lOzY/wAAAG50Uk5TAA5Acarj/8dVMgdq1LgjXdzxzbAfO/iV1epHHGO/f9hcFZ3lQiR08oAPeL7CuWkIYKieoI8u9/n69en2/eAX6GY6rljt7MzmWv4ac5jwEO5RM7381k8skCnX3srvyDdB6/N8jNLnBfT7wXqSr7W47lQGAAABpElEQVR4nIVT6z8CQRS9+7qVNk3FKiqUIvJWbSLvjQh5hkgeef//X83s2rb6hfNhZvecmXvP3LkD8AOOF0QJKWyi3cFBO3ocThmbcDl7e1pVzk08Xp+9j333Cy6ieN2t8oA/IA42Iw4FQ+gPW2oIh0fa0o1GomjqnBujYx1mYvFxv9vIP+jFCejEaAi9vfpm0eOkexOTU8npmVmYm19YXEpRA6LCaHAQ4qDTUlrJqFkeJGU5t7KaBwiii/F29LEka+r6xmZ6C7Z3+uJaYZdG3UM7QFEMCCyHpwCQ2s9Tmf6sanQ4QLEIMRtJMTlbCh+yeYfJyhEdjtEWgzJqutUT9fTs3JRz24xCLDdlmCmkoxe/ywCXauQvGVQJDGuWXJGudGvXN9RR1ZAT8i1l7lCqNA+WVmoJtQRwX394fHrmzYM1BKMs2mlpLeMDkF+ymVeS18siNAD4gF5UOMnW3yj7/vGZ/KoBK6rMW1fSCfNKIEi6X6gR9L92AE7u1kwDFiNjZytiqGX1P40MXFAMYPRKoyBVJC2hzAV/PSKGRrFSpogVGxb3De/eO/7N6EMHAAAAAElFTkSuQmCC"/>
- </defs>
- <style>
- tspan { white-space:pre }
- </style>
- <use id="Layer 2" href="#img1" x="1" y="2" />
-</svg> \ No newline at end of file
diff --git a/sd-plugin/src/main/resources/flex/idea-flex.skeleton b/sd-plugin/src/main/resources/flex/idea-flex.skeleton
deleted file mode 100644
index 218399976ca..00000000000
--- a/sd-plugin/src/main/resources/flex/idea-flex.skeleton
+++ /dev/null
@@ -1,248 +0,0 @@
-
- /** This character denotes the end of file */
- public static final int YYEOF = -1;
-
- /** initial size of the lookahead buffer */
---- private static final int ZZ_BUFFERSIZE = ...;
-
- /** lexical states */
---- lexical states, charmap
-
- /* error codes */
- private static final int ZZ_UNKNOWN_ERROR = 0;
- private static final int ZZ_NO_MATCH = 1;
- private static final int ZZ_PUSHBACK_2BIG = 2;
-
- /* error messages for the codes above */
- private static final String[] ZZ_ERROR_MSG = {
- "Unknown internal scanner error",
- "Error: could not match input",
- "Error: pushback value was too large"
- };
-
---- isFinal list
- /** the input device */
- private java.io.Reader zzReader;
-
- /** the current state of the DFA */
- private int zzState;
-
- /** the current lexical state */
- private int zzLexicalState = YYINITIAL;
-
- /** this buffer contains the current text to be matched and is
- the source of the yytext() string */
- private CharSequence zzBuffer = "";
-
- /** the textposition at the last accepting state */
- private int zzMarkedPos;
-
- /** the current text position in the buffer */
- private int zzCurrentPos;
-
- /** startRead marks the beginning of the yytext() string in the buffer */
- private int zzStartRead;
-
- /** endRead marks the last character in the buffer, that has been read
- from input */
- private int zzEndRead;
-
- /**
- * zzAtBOL == true <=> the scanner is currently at the beginning of a line
- */
- private boolean zzAtBOL = true;
-
- /** zzAtEOF == true <=> the scanner is at the EOF */
- private boolean zzAtEOF;
-
- /** denotes if the user-EOF-code has already been executed */
- private boolean zzEOFDone;
-
---- user class code
-
---- constructor declaration
-
- public final int getTokenStart() {
- return zzStartRead;
- }
-
- public final int getTokenEnd() {
- return getTokenStart() + yylength();
- }
-
- public void reset(CharSequence buffer, int start, int end, int initialState) {
- zzBuffer = buffer;
- zzCurrentPos = zzMarkedPos = zzStartRead = start;
- zzAtEOF = false;
- zzAtBOL = true;
- zzEndRead = end;
- yybegin(initialState);
- }
-
- /**
- * Refills the input buffer.
- *
- * @return {@code false}, iff there was new input.
- *
- * @exception java.io.IOException if any I/O-Error occurs
- */
- private boolean zzRefill() throws java.io.IOException {
- return true;
- }
-
-
- /**
- * Returns the current lexical state.
- */
- public final int yystate() {
- return zzLexicalState;
- }
-
-
- /**
- * Enters a new lexical state
- *
- * @param newState the new lexical state
- */
- public final void yybegin(int newState) {
- zzLexicalState = newState;
- }
-
-
- /**
- * Returns the text matched by the current regular expression.
- */
- public final CharSequence yytext() {
- return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
- }
-
-
- /**
- * Returns the character at position {@code pos} from the
- * matched text.
- *
- * It is equivalent to yytext().charAt(pos), but faster
- *
- * @param pos the position of the character to fetch.
- * A value from 0 to yylength()-1.
- *
- * @return the character at position pos
- */
- public final char yycharat(int pos) {
- return zzBuffer.charAt(zzStartRead+pos);
- }
-
-
- /**
- * Returns the length of the matched text region.
- */
- public final int yylength() {
- return zzMarkedPos-zzStartRead;
- }
-
-
- /**
- * Reports an error that occurred while scanning.
- *
- * In a wellformed scanner (no or only correct usage of
- * yypushback(int) and a match-all fallback rule) this method
- * will only be called with things that "Can't Possibly Happen".
- * If this method is called, something is seriously wrong
- * (e.g. a JFlex bug producing a faulty scanner etc.).
- *
- * Usual syntax/scanner level error handling should be done
- * in error fallback rules.
- *
- * @param errorCode the code of the errormessage to display
- */
---- zzScanError declaration
- String message;
- try {
- message = ZZ_ERROR_MSG[errorCode];
- }
- catch (ArrayIndexOutOfBoundsException e) {
- message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
- }
-
---- throws clause
- }
-
-
- /**
- * Pushes the specified amount of characters back into the input stream.
- *
- * They will be read again by then next call of the scanning method
- *
- * @param number the number of characters to be read again.
- * This number must not be greater than yylength()!
- */
---- yypushback decl (contains zzScanError exception)
- if ( number > yylength() )
- zzScanError(ZZ_PUSHBACK_2BIG);
-
- zzMarkedPos -= number;
- }
-
-
---- zzDoEOF
- /**
- * Resumes scanning until the next regular expression is matched,
- * the end of input is encountered or an I/O-Error occurs.
- *
- * @return the next token
- * @exception java.io.IOException if any I/O-Error occurs
- */
---- yylex declaration
- int zzInput;
- int zzAction;
-
- // cached fields:
- int zzCurrentPosL;
- int zzMarkedPosL;
- int zzEndReadL = zzEndRead;
- CharSequence zzBufferL = zzBuffer;
-
---- local declarations
-
- while (true) {
- zzMarkedPosL = zzMarkedPos;
-
---- start admin (line, char, col count)
- zzAction = -1;
-
- zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
-
---- start admin (lexstate etc)
-
- zzForAction: {
- while (true) {
-
---- next input, line, col, char count, next transition, isFinal action
- zzAction = zzState;
- zzMarkedPosL = zzCurrentPosL;
---- line count update
- }
-
- }
- }
-
- // store back cached position
- zzMarkedPos = zzMarkedPosL;
---- char count update
-
- if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
- zzAtEOF = true;
---- eofvalue
- }
- else {
---- actions
- default:
---- no match
- }
- }
- }
- }
-
---- main
-
-}
diff --git a/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar b/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar
deleted file mode 100644
index c152c2c3384..00000000000
--- a/sd-plugin/src/main/resources/flex/jflex-1.7.0-2.jar
+++ /dev/null
Binary files differ
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index d4ad85d88d9..3397b424ed0 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -28,7 +28,6 @@
#include <vespa/searchlib/attribute/bitvector_search_cache.h>
#include <vespa/searchlib/attribute/imported_attribute_vector.h>
#include <vespa/searchlib/attribute/imported_attribute_vector_factory.h>
-#include <vespa/searchlib/attribute/integerbase.h>
#include <vespa/searchlib/attribute/predicate_attribute.h>
#include <vespa/vespalib/util/idestructorcallback.h>
#include <vespa/searchlib/index/docbuilder.h>
@@ -40,7 +39,6 @@
#include <vespa/searchlib/test/directory_handler.h>
#include <vespa/vespalib/btree/btreeroot.hpp>
#include <vespa/vespalib/gtest/gtest.h>
-#include <vespa/vespalib/io/fileutil.h>
#include <vespa/vespalib/test/insertion_operators.h>
#include <vespa/vespalib/util/destructor_callbacks.h>
#include <vespa/vespalib/util/exceptions.h>
@@ -165,12 +163,10 @@ public:
AttributeVector::SP addAttribute(const AttributeSpec &spec) {
auto ret = _mgr->addAttribute(spec.getName(),
AttributeFactory::createAttribute(spec.getName(), spec.getConfig()));
- allocAttributeWriter();
return ret;
}
void add_attribute(AttributeVector::SP attr) {
_mgr->addAttribute(attr->getName(), std::move(attr));
- allocAttributeWriter();
}
void put(SerialNum serialNum, const Document &doc, DocumentIdT lid) {
_aw->put(serialNum, doc, lid, emptyCallback);
@@ -225,6 +221,7 @@ TEST_F(AttributeWriterTest, handles_put)
auto a2 = addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)});
auto a3 = addAttribute({"a3", AVConfig(AVBasicType::FLOAT)});
auto a4 = addAttribute({"a4", AVConfig(AVBasicType::STRING)});
+ allocAttributeWriter();
attribute::IntegerContent ibuf;
attribute::FloatContent fbuf;
@@ -303,6 +300,7 @@ TEST_F(AttributeWriterTest, handles_predicate_put)
DocBuilder idb(s);
auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)});
+ allocAttributeWriter();
PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex();
@@ -349,6 +347,7 @@ TEST_F(AttributeWriterTest, handles_remove)
{
auto a1 = addAttribute("a1");
auto a2 = addAttribute("a2");
+ allocAttributeWriter();
fillAttribute(a1, 1, 10, 1);
fillAttribute(a2, 1, 20, 1);
@@ -371,6 +370,7 @@ TEST_F(AttributeWriterTest, handles_batch_remove)
{
auto a1 = addAttribute("a1");
auto a2 = addAttribute("a2");
+ allocAttributeWriter();
fillAttribute(a1, 4, 22, 1);
fillAttribute(a2, 4, 33, 1);
@@ -397,6 +397,7 @@ verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::string
TEST_F(AttributeWriterTest, visibility_delay_is_honoured)
{
auto a1 = addAttribute({"a1", AVConfig(AVBasicType::STRING)});
+ allocAttributeWriter();
Schema s;
s.addAttributeField(Schema::AttributeField("a1", schema::DataType::STRING, CollectionType::SINGLE));
DocBuilder idb(s);
@@ -442,6 +443,7 @@ TEST_F(AttributeWriterTest, visibility_delay_is_honoured)
TEST_F(AttributeWriterTest, handles_predicate_remove)
{
auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)});
+ allocAttributeWriter();
Schema s;
s.addAttributeField(
Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE));
@@ -465,6 +467,7 @@ TEST_F(AttributeWriterTest, handles_update)
{
auto a1 = addAttribute("a1");
auto a2 = addAttribute("a2");
+ allocAttributeWriter();
fillAttribute(a1, 1, 10, 1);
fillAttribute(a2, 1, 20, 1);
@@ -504,6 +507,7 @@ TEST_F(AttributeWriterTest, handles_update)
TEST_F(AttributeWriterTest, handles_predicate_update)
{
auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)});
+ allocAttributeWriter();
Schema schema;
schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE));
@@ -680,6 +684,7 @@ createTensorPutDoc(DocBuilder &builder, const Value &tensor) {
TEST_F(AttributeWriterTest, can_write_to_tensor_attribute)
{
auto a1 = createTensorAttribute(*this);
+ allocAttributeWriter();
Schema s = createTensorSchema();
DocBuilder builder(s);
auto tensor = make_tensor(TensorSpec(sparse_tensor)
@@ -697,6 +702,7 @@ TEST_F(AttributeWriterTest, can_write_to_tensor_attribute)
TEST_F(AttributeWriterTest, handles_tensor_assign_update)
{
auto a1 = createTensorAttribute(*this);
+ allocAttributeWriter();
Schema s = createTensorSchema();
DocBuilder builder(s);
auto tensor = make_tensor(TensorSpec(sparse_tensor)
@@ -762,6 +768,7 @@ putAttributes(AttributeWriterTest &t, std::vector<uint32_t> expExecuteHistory)
auto a1 = t.addAttribute(a1_name);
auto a2 = t.addAttribute(a2_name);
auto a3 = t.addAttribute(a3_name);
+ t.allocAttributeWriter();
EXPECT_EQ(1u, a1->getNumDocs());
EXPECT_EQ(1u, a2->getNumDocs());
@@ -899,6 +906,7 @@ public:
tensor = make_tensor(TensorSpec(dense_tensor)
.add({{"x", 0}}, 3).add({{"x", 1}}, 5));
attr->exp_tensor = tensor.get();
+ allocAttributeWriter();
}
void expect_tensor_attr_calls(size_t exp_prepare_cnt,
size_t exp_complete_cnt,
@@ -1028,6 +1036,7 @@ TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribut
TEST_F(AttributeWriterTest, ignores_force_commit_serial_not_greater_than_create_serial)
{
auto a1 = addAttribute("a1");
+ allocAttributeWriter();
a1->setCreateSerialNum(100);
EXPECT_EQ(0u, test_force_commit(*a1, 50u));
EXPECT_EQ(0u, test_force_commit(*a1, 100u));
@@ -1112,6 +1121,7 @@ StructArrayWriterTest::~StructArrayWriterTest() = default;
TEST_F(StructArrayWriterTest, update_with_doc_argument_updates_struct_field_attributes)
{
+ allocAttributeWriter();
auto doc = makeDoc(10, {11, 12});
put(10, *doc, 1);
checkAttrs(1, 10, {11, 12});
@@ -1169,6 +1179,7 @@ public:
TEST_F(StructMapWriterTest, update_with_doc_argument_updates_struct_field_attributes)
{
+ allocAttributeWriter();
auto doc = makeDoc(10, {{1, 11}, {2, 12}});
put(10, *doc, 1);
checkAttrs(1, 10, {{1, 11}, {2, 12}});
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index b2acc8703f3..cd694cb938c 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -287,7 +287,7 @@ struct FastAccessFixture
vespalib::mkdir(BASE_DIR);
}
~FastAccessFixture() {
- _writeService.sync_all_executors();
+ _writeService.shutdown();
}
};
@@ -377,18 +377,12 @@ struct FeedViewComparer
void expect_equal_index_adapter() {
EXPECT_EQUAL(_old->getIndexWriter().get(), _new->getIndexWriter().get());
}
- void expect_equal_attribute_writer() {
- EXPECT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get());
- }
void expect_not_equal_attribute_writer() {
EXPECT_NOT_EQUAL(_old->getAttributeWriter().get(), _new->getAttributeWriter().get());
}
void expect_equal_summary_adapter() {
EXPECT_EQUAL(_old->getSummaryAdapter().get(), _new->getSummaryAdapter().get());
}
- void expect_equal_schema() {
- EXPECT_EQUAL(_old->getSchema().get(), _new->getSchema().get());
- }
void expect_not_equal_schema() {
EXPECT_NOT_EQUAL(_old->getSchema().get(), _new->getSchema().get());
}
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 091292b3151..e3e6ac6321e 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -34,6 +34,7 @@
#include <vespa/vespalib/util/lambdatask.h>
#include <vespa/vespalib/util/size_literals.h>
#include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/destructor_callbacks.h>
using namespace cloud::config::filedistribution;
using namespace document;
@@ -65,7 +66,6 @@ typedef StoreOnlyDocSubDB::Config StoreOnlyConfig;
typedef StoreOnlyDocSubDB::Context StoreOnlyContext;
typedef FastAccessDocSubDB::Config FastAccessConfig;
typedef FastAccessDocSubDB::Context FastAccessContext;
-typedef SearchableDocSubDB::Config SearchableConfig;
typedef SearchableDocSubDB::Context SearchableContext;
typedef std::vector<AttributeGuard> AttributeGuardList;
@@ -203,9 +203,9 @@ MyFastAccessContext::~MyFastAccessContext() = default;
struct MySearchableConfig
{
- SearchableConfig _cfg;
+ FastAccessConfig _cfg;
MySearchableConfig()
- : _cfg(MyFastAccessConfig<false>()._cfg, 1)
+ : _cfg(MyFastAccessConfig<false>()._cfg)
{
}
};
@@ -313,9 +313,17 @@ struct FixtureBase
init();
}
~FixtureBase() {
- _writeService.sync_all_executors();
- _writeService.master().execute(makeLambdaTask([this]() { _subDb.close(); }));
- _writeService.sync_all_executors();
+ _writeService.master().execute(makeLambdaTask([this]() { _subDb.close(); }));
+ _writeService.shutdown();
+ }
+ void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc) {
+ vespalib::Gate gate;
+ _subDb.setBucketStateCalculator(calc, std::make_shared<vespalib::GateCallback>(gate));
+ gate.await();
+ }
+ template <typename FunctionType>
+ void runInMasterAndSync(FunctionType func) {
+ proton::test::runInMasterAndSync(_writeService, func);
}
template <typename FunctionType>
void runInMaster(FunctionType func) {
@@ -328,16 +336,16 @@ struct FixtureBase
initializer::TaskRunner taskRunner(executor);
taskRunner.runTask(task);
auto sessionMgr = std::make_shared<SessionManager>(1);
- runInMaster([&] () { _subDb.initViews(*_snapshot->_cfg, sessionMgr); });
+ runInMasterAndSync([&]() { _subDb.initViews(*_snapshot->_cfg, sessionMgr); });
}
void basicReconfig(SerialNum serialNum) {
- runInMaster([&] () { performReconfig(serialNum, TwoAttrSchema(), ConfigDir2::dir()); });
+ runInMasterAndSync([&]() { performReconfig(serialNum, TwoAttrSchema(), ConfigDir2::dir()); });
}
void reconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) {
- runInMaster([&] () { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); });
+ runInMasterAndSync([&]() { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); });
}
void performReconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) {
- MyConfigSnapshot::UP newCfg(new MyConfigSnapshot(reconfigSchema, reconfigConfigDir));
+ auto newCfg = std::make_unique<MyConfigSnapshot>(reconfigSchema, reconfigConfigDir);
DocumentDBConfig::ComparisonResult cmpResult;
cmpResult.attributesChanged = true;
cmpResult.documenttypesChanged = true;
@@ -353,9 +361,7 @@ struct FixtureBase
}
_subDb.onReprocessDone(serialNum);
}
- void sync() {
- _writeService.master().sync();
- }
+
proton::IAttributeManager::SP getAttributeManager() {
return _subDb.getAttributeManager();
}
@@ -386,20 +392,20 @@ typedef BaseTraitsT<OneAttrSchema, ConfigDir1> BaseTraits;
struct StoreOnlyTraits : public BaseTraits
{
- typedef MyStoreOnlyConfig Config;
- typedef MyStoreOnlyContext Context;
- typedef StoreOnlyDocSubDB SubDB;
- typedef StoreOnlyFeedView FeedView;
+ using Config = MyStoreOnlyConfig;
+ using Context = MyStoreOnlyContext;
+ using SubDB = StoreOnlyDocSubDB;
+ using FeedView = StoreOnlyFeedView;
};
typedef FixtureBase<StoreOnlyTraits> StoreOnlyFixture;
struct FastAccessTraits : public BaseTraits
{
- typedef MyFastAccessConfig<false> Config;
- typedef MyFastAccessContext Context;
- typedef FastAccessDocSubDB SubDB;
- typedef FastAccessFeedView FeedView;
+ using Config = MyFastAccessConfig<false>;
+ using Context = MyFastAccessContext;
+ using SubDB = FastAccessDocSubDB;
+ using FeedView = FastAccessFeedView;
};
typedef FixtureBase<FastAccessTraits> FastAccessFixture;
@@ -407,10 +413,10 @@ typedef FixtureBase<FastAccessTraits> FastAccessFixture;
template <typename ConfigDirT>
struct FastAccessOnlyTraitsBase : public BaseTraitsT<TwoAttrSchema, ConfigDirT>
{
- typedef MyFastAccessConfig<true> Config;
- typedef MyFastAccessContext Context;
- typedef FastAccessDocSubDB SubDB;
- typedef FastAccessFeedView FeedView;
+ using Config = MyFastAccessConfig<true>;
+ using Context = MyFastAccessContext;
+ using SubDB = FastAccessDocSubDB;
+ using FeedView = FastAccessFeedView;
};
// Setup with 1 fast-access attribute
@@ -420,10 +426,10 @@ typedef FixtureBase<FastAccessOnlyTraits> FastAccessOnlyFixture;
template <typename SchemaT, typename ConfigDirT>
struct SearchableTraitsBase : public BaseTraitsT<SchemaT, ConfigDirT>
{
- typedef MySearchableConfig Config;
- typedef MySearchableContext Context;
- typedef SearchableDocSubDB SubDB;
- typedef proton::SearchableFeedView FeedView;
+ using Config = MySearchableConfig;
+ using Context = MySearchableContext;
+ using SubDB = SearchableDocSubDB;
+ using FeedView = proton::SearchableFeedView;
};
typedef SearchableTraitsBase<OneAttrSchema, ConfigDir1> SearchableTraits;
@@ -567,14 +573,14 @@ TEST_F("require that subdb reflect retirement", FastAccessFixture)
auto calc = std::make_shared<proton::test::BucketStateCalculator>();
calc->setNodeRetired(true);
- f._subDb.setBucketStateCalculator(calc);
+ f.setBucketStateCalculator(calc);
EXPECT_TRUE(f._subDb.isNodeRetired());
auto retired_cfg = f._subDb.computeCompactionStrategy(cfg);
EXPECT_TRUE(cfg != retired_cfg);
EXPECT_TRUE(search::CompactionStrategy(0.5, 0.5) == retired_cfg);
calc->setNodeRetired(false);
- f._subDb.setBucketStateCalculator(calc);
+ f.setBucketStateCalculator(calc);
EXPECT_FALSE(f._subDb.isNodeRetired());
unretired_cfg = f._subDb.computeCompactionStrategy(cfg);
EXPECT_TRUE(cfg == unretired_cfg);
@@ -590,21 +596,18 @@ TEST_F("require that attribute compaction config reflect retirement", FastAccess
auto calc = std::make_shared<proton::test::BucketStateCalculator>();
calc->setNodeRetired(true);
- f._subDb.setBucketStateCalculator(calc);
- f._writeService.sync_all_executors();
+ f.setBucketStateCalculator(calc);
guard = f._subDb.getAttributeManager()->getAttribute("attr1");
EXPECT_EQUAL(retired_cfg, (*guard)->getConfig().getCompactionStrategy());
EXPECT_EQUAL(retired_cfg, dynamic_cast<const proton::DocumentMetaStore &>(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy());
f.basicReconfig(10);
- f._writeService.sync_all_executors();
guard = f._subDb.getAttributeManager()->getAttribute("attr1");
EXPECT_EQUAL(retired_cfg, (*guard)->getConfig().getCompactionStrategy());
EXPECT_EQUAL(retired_cfg, dynamic_cast<const proton::DocumentMetaStore &>(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy());
calc->setNodeRetired(false);
- f._subDb.setBucketStateCalculator(calc);
- f._writeService.sync_all_executors();
+ f.setBucketStateCalculator(calc);
guard = f._subDb.getAttributeManager()->getAttribute("attr1");
EXPECT_EQUAL(default_cfg, (*guard)->getConfig().getCompactionStrategy());
EXPECT_EQUAL(default_cfg, dynamic_cast<const proton::DocumentMetaStore &>(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy());
@@ -783,27 +786,34 @@ struct DocumentHandler
}
void putDoc(PutOperation &op) {
IFeedView::SP feedView = _f._subDb.getFeedView();
+ vespalib::Gate gate;
_f.runInMaster([&]() {
feedView->preparePut(op);
feedView->handlePut(FeedToken(), op);
- feedView->forceCommit(op.getSerialNum());
- } );
+ feedView->forceCommit(CommitParam(op.getSerialNum()), std::make_shared<vespalib::GateCallback>(gate));
+ });
+ gate.await();
}
void moveDoc(MoveOperation &op) {
IFeedView::SP feedView = _f._subDb.getFeedView();
+ vespalib::Gate gate;
_f.runInMaster([&]() {
- feedView->handleMove(op, IDestructorCallback::SP());
- feedView->forceCommit(op.getSerialNum());
- } );
+ auto onDone = std::make_shared<vespalib::GateCallback>(gate);
+ feedView->handleMove(op, onDone);
+ feedView->forceCommit(CommitParam(op.getSerialNum()), onDone);
+ });
+ gate.await();
}
void removeDoc(RemoveOperation &op)
{
IFeedView::SP feedView = _f._subDb.getFeedView();
+ vespalib::Gate gate;
_f.runInMaster([&]() {
feedView->prepareRemove(op);
feedView->handleRemove(FeedToken(), op);
- feedView->forceCommit(op.getSerialNum());
- } );
+ feedView->forceCommit(CommitParam(op.getSerialNum()), std::make_shared<vespalib::GateCallback>(gate));
+ });
+ gate.await();
}
void putDocs() {
PutOperation putOp = createPut(std::move(createDoc(1, 22, 33)), Timestamp(10), 10);
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
index 977c899ab11..bffedfd8dab 100644
--- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
@@ -185,13 +185,13 @@ struct MyFeedView : public test::DummyFeedView {
int remove_count;
int move_count;
int prune_removed_count;
+
int update_count;
SerialNum update_serial;
const DocumentType *documentType;
MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr,
const DocTypeName &docTypeName);
~MyFeedView() override;
- void resetPutLatch(uint32_t count) { putLatch = std::make_unique<vespalib::CountDownLatch>(count); }
void preparePut(PutOperation &op) override {
prepareDocumentOperation(op, op.getDocument()->getId().getGlobalId());
}
@@ -389,16 +389,6 @@ FeedTokenContext::FeedTokenContext()
FeedTokenContext::~FeedTokenContext() = default;
-struct PutContext {
- FeedTokenContext tokenCtx;
- DocumentContext docCtx;
- typedef std::shared_ptr<PutContext> SP;
- PutContext(const vespalib::string &docId, DocBuilder &builder) :
- tokenCtx(),
- docCtx(docId, builder)
- {}
-};
-
struct MyTlsWriter : TlsWriter {
int store_count;
int erase_count;
@@ -455,40 +445,18 @@ struct FeedHandlerFixture
}
~FeedHandlerFixture() {
- writeService.sync_all_executors();
+ writeService.shutdown();
}
template <class FunctionType>
inline void runAsMaster(FunctionType &&function) {
writeService.master().execute(makeLambdaTask(std::move(function)));
- writeService.master().sync();
+ syncMaster();
}
void syncMaster() {
writeService.master().sync();
}
};
-
-struct MyConfigStore : ConfigStore {
- SerialNum getBestSerialNum() const override { return 1; }
- SerialNum getOldestSerialNum() const override { return 1; }
- void saveConfig(const DocumentDBConfig &, SerialNum) override {}
- void loadConfig(const DocumentDBConfig &, SerialNum, DocumentDBConfig::SP &) override {}
- void removeInvalid() override {}
- void prune(SerialNum) override {}
- bool hasValidSerial(SerialNum) const override { return true; }
- SerialNum getPrevValidSerial(SerialNum) const override { return 1; }
- void serializeConfig(SerialNum, vespalib::nbostream &) override {}
- void deserializeConfig(SerialNum, vespalib::nbostream &) override {}
- void setProtonConfig(const ProtonConfigSP &) override { }
-};
-
-
-struct ReplayTransactionLogContext {
- MyConfigStore config_store;
- DocumentDBConfig::SP cfgSnap;
-};
-
-
TEST_F("require that heartBeat calls FeedView's heartBeat",
FeedHandlerFixture)
{
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index c35cd49e498..a030ce1c455 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -22,6 +22,7 @@
#include <vespa/searchcore/proton/test/threading_service_observer.h>
#include <vespa/searchlib/attribute/attributefactory.h>
#include <vespa/document/update/documentupdate.h>
+#include <vespa/vespalib/util/destructor_callbacks.h>
#include <vespa/searchlib/index/docbuilder.h>
#include <vespa/log/log.h>
@@ -375,6 +376,9 @@ struct MyAttributeWriter : public IAttributeWriter
++_commitCount;
_tracer.traceCommit(attributeAdapterTypeName, param.lastSerialNum());
}
+ void drain(OnWriteDoneType onDone) override {
+ (void) onDone;
+ }
void onReplayDone(uint32_t ) override { }
bool hasStructFieldAttribute() const override { return false; }
@@ -415,16 +419,6 @@ struct MyTransport : public feedtoken::ITransport
MyTransport::MyTransport(MyTracer &tracer) : lastResult(), _gate(), _tracer(tracer) {}
MyTransport::~MyTransport() = default;
-struct MyResultHandler : public IGenericResultHandler
-{
- vespalib::Gate _gate;
- MyResultHandler() : _gate() {}
- void handle(const storage::spi::Result &) override {
- _gate.countDown();
- }
- void await() { _gate.await(); }
-};
-
struct SchemaContext
{
Schema::SP _schema;
@@ -447,7 +441,6 @@ SchemaContext::SchemaContext() :
}
SchemaContext::~SchemaContext() = default;
-
struct DocumentContext
{
Document::SP doc;
@@ -511,18 +504,6 @@ struct FixtureBase
virtual ~FixtureBase();
- void syncMaster() {
- _writeService.master().sync();
- }
-
- void syncIndex() {
- _writeService.sync_all_executors();
- }
-
- void sync() {
- _writeServiceReal.sync_all_executors();
- }
-
const test::DocumentMetaStoreObserver &metaStoreObserver() {
return _dmsc->getObserver();
}
@@ -532,6 +513,10 @@ struct FixtureBase
}
template <typename FunctionType>
+ void runInMasterAndSyncAll(FunctionType func) {
+ test::runInMasterAndSyncAll(_writeService, func);
+ }
+ template <typename FunctionType>
void runInMaster(FunctionType func) {
test::runInMaster(_writeService, func);
}
@@ -578,7 +563,7 @@ struct FixtureBase
void putAndWait(const DocumentContext &docCtx) {
FeedTokenContext token(_tracer);
PutOperation op(docCtx.bid, docCtx.ts, docCtx.doc);
- runInMaster([this, ft=std::move(token.ft), &op] () mutable { performPut(std::move(ft), op); });
+ runInMaster([this, ft = std::move(token.ft), &op]() mutable { performPut(std::move(ft), op); });
token.mt.await();
}
@@ -591,7 +576,7 @@ struct FixtureBase
void updateAndWait(const DocumentContext &docCtx) {
FeedTokenContext token(_tracer);
UpdateOperation op(docCtx.bid, docCtx.ts, docCtx.upd);
- runInMaster([this, ft=std::move(token.ft), &op] () mutable { performUpdate(std::move(ft), op); });
+ runInMaster([this, ft = std::move(token.ft), &op]() mutable { performUpdate(std::move(ft), op); });
token.mt.await();
}
@@ -606,7 +591,7 @@ struct FixtureBase
void removeAndWait(const DocumentContext &docCtx) {
FeedTokenContext token(_tracer);
RemoveOperationWithDocId op(docCtx.bid, docCtx.ts, docCtx.doc->getId());
- runInMaster([this, ft=std::move(token.ft), &op] () mutable { performRemove(std::move(ft), op); });
+ runInMaster([this, ft = std::move(token.ft), &op]() mutable { performRemove(std::move(ft), op); });
token.mt.await();
}
@@ -616,15 +601,17 @@ struct FixtureBase
}
}
- void performMove(MoveOperation &op) {
+ void performMove(MoveOperation &op, IDestructorCallback::SP onDone) {
op.setSerialNum(++serial);
- getFeedView().handleMove(op, IDestructorCallback::SP());
+ getFeedView().handleMove(op, std::move(onDone));
}
void moveAndWait(const DocumentContext &docCtx, uint32_t fromLid, uint32_t toLid) {
MoveOperation op(docCtx.bid, docCtx.ts, docCtx.doc, DbDocumentId(pc._params._subDbId, fromLid), pc._params._subDbId);
op.setTargetLid(toLid);
- runInMaster([&]() { performMove(op); });
+ vespalib::Gate gate;
+ runInMaster([&, onDone=std::make_shared<vespalib::GateCallback>(gate)]() { performMove(op, std::move(onDone)); });
+ gate.await();
}
void performDeleteBucket(DeleteBucketOperation &op) {
@@ -635,7 +622,7 @@ struct FixtureBase
void performForceCommit() { getFeedView().forceCommit(serial); }
void forceCommitAndWait() {
- runInMaster([&]() { performForceCommit(); });
+ runInMasterAndSyncAll([&]() { performForceCommit(); });
}
bool assertTrace(const vespalib::string &exp) {
@@ -662,7 +649,7 @@ struct FixtureBase
fv.handleCompactLidSpace(op);
}
void compactLidSpaceAndWait(uint32_t wantedLidLimit) {
- runInMaster([&] () { performCompactLidSpace(wantedLidLimit); });
+ runInMasterAndSyncAll([&]() { performCompactLidSpace(wantedLidLimit); });
}
void assertChangeHandler(document::GlobalId expGid, uint32_t expLid, uint32_t expChanges) {
_gidToLidChangeHandler->assertChanges(expGid, expLid, expChanges);
@@ -674,6 +661,8 @@ struct FixtureBase
_gidToLidChangeHandler->assertLid(gid, expLid);
}
void populateBeforeCompactLidSpace();
+
+ void dms_commit() { _dmsc->get().commit(search::CommitParam(serial)); }
};
@@ -701,7 +690,7 @@ FixtureBase::FixtureBase()
}
FixtureBase::~FixtureBase() {
- _writeServiceReal.sync_all_executors();
+ _writeServiceReal.shutdown();
}
void
@@ -789,6 +778,7 @@ TEST_F("require that put() updates document meta store with bucket info",
{
DocumentContext dc = f.doc1();
f.putAndWait(dc);
+ f.dms_commit();
assertBucketInfo(dc.bid, dc.ts, 1, f.getMetaStore());
// TODO: rewrite to use getBucketInfo() when available
@@ -828,6 +818,7 @@ TEST_F("require that update() updates document meta store with bucket info",
f.putAndWait(dc1);
BucketChecksum bcs = f.getBucketDB()->get(dc1.bid).getChecksum();
f.updateAndWait(dc2);
+ f.dms_commit();
assertBucketInfo(dc1.bid, Timestamp(20), 1, f.getMetaStore());
// TODO: rewrite to use getBucketInfo() when available
@@ -857,6 +848,7 @@ TEST_F("require that remove() updates document meta store with bucket info",
f.putAndWait(dc2);
BucketChecksum bcs2 = f.getBucketDB()->get(dc2.bid).getChecksum();
f.removeAndWait(DocumentContext("id:test:searchdocument:n=1:2", 20, f.getBuilder()));
+ f.dms_commit();
assertBucketInfo(dc1.bid, Timestamp(10), 1, f.getMetaStore());
EXPECT_FALSE(f.getMetaStore().validLid(2)); // don't remember remove
@@ -934,6 +926,7 @@ TEST_F("require that handleDeleteBucket() removes documents", SearchableFeedView
TEST_DO(f.assertChangeNotified(docs[2].gid(), 3));
TEST_DO(f.assertChangeNotified(docs[3].gid(), 4));
TEST_DO(f.assertChangeNotified(docs[4].gid(), 5));
+ f.dms_commit();
DocumentIdT lid;
EXPECT_TRUE(f.getMetaStore().getLid(docs[0].doc->getId().getGlobalId(), lid));
@@ -945,7 +938,8 @@ TEST_F("require that handleDeleteBucket() removes documents", SearchableFeedView
// delete bucket for user 1
DeleteBucketOperation op(docs[0].bid);
- f.runInMaster([&] () { f.performDeleteBucket(op); });
+ f.runInMasterAndSyncAll([&]() { f.performDeleteBucket(op); });
+ f.dms_commit();
EXPECT_EQUAL(0u, f.getBucketDB()->get(docs[0].bid).getDocumentCount());
EXPECT_EQUAL(2u, f.getBucketDB()->get(docs[3].bid).getDocumentCount());
@@ -1047,7 +1041,7 @@ TEST_F("require that removes are not remembered", SearchableFeedViewFixture)
TEST_F("require that heartbeat propagates to index- and attributeadapter",
SearchableFeedViewFixture)
{
- f.runInMaster([&] () { f.fv.heartBeat(2); });
+ f.runInMasterAndSyncAll([&]() { f.fv.heartBeat(2); });
EXPECT_EQUAL(1, f.miw._heartBeatCount);
EXPECT_EQUAL(1, f.maw._heartBeatCount);
}
@@ -1143,7 +1137,7 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc
f.compactLidSpaceAndWait(2);
// performIndexForceCommit in index thread, then completion callback
// in master thread.
- EXPECT_TRUE(assertThreadObserver(7, 6, 6, f.writeServiceObserver()));
+ EXPECT_TRUE(assertThreadObserver(7, 7, 7, f.writeServiceObserver()));
EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit);
EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit);
EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
@@ -1159,32 +1153,29 @@ TEST_F("require that compactLidSpace() doesn't propagate to "
EXPECT_TRUE(assertThreadObserver(5, 4, 4, f.writeServiceObserver()));
CompactLidSpaceOperation op(0, 2);
op.setSerialNum(0);
- f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); });
+ f.runInMasterAndSyncAll([&]() { f.fv.handleCompactLidSpace(op); });
// Delayed holdUnblockShrinkLidSpace() in index thread, then master thread
- EXPECT_TRUE(assertThreadObserver(6, 5, 4, f.writeServiceObserver()));
+ EXPECT_TRUE(assertThreadObserver(6, 6, 5, f.writeServiceObserver()));
EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit);
EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit);
EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
}
-TEST_F("require that compactLidSpace() propagates to attributeadapter",
- FastAccessFeedViewFixture)
+TEST_F("require that compactLidSpace() propagates to attributeadapter", FastAccessFeedViewFixture)
{
f.populateBeforeCompactLidSpace();
f.compactLidSpaceAndWait(2);
EXPECT_EQUAL(2u, f.maw._wantedLidLimit);
}
-TEST_F("require that compactLidSpace() propagates to index writer",
- SearchableFeedViewFixture)
+TEST_F("require that compactLidSpace() propagates to index writer", SearchableFeedViewFixture)
{
f.populateBeforeCompactLidSpace();
f.compactLidSpaceAndWait(2);
EXPECT_EQUAL(2u, f.miw._wantedLidLimit);
}
-TEST_F("require that commit is not implicitly called",
- SearchableFeedViewFixture)
+TEST_F("require that commit is not implicitly called", SearchableFeedViewFixture)
{
DocumentContext dc = f.doc1();
f.putAndWait(dc);
@@ -1204,8 +1195,7 @@ TEST_F("require that commit is not implicitly called",
f.forceCommitAndWait();
}
-TEST_F("require that forceCommit updates docid limit",
- SearchableFeedViewFixture)
+TEST_F("require that forceCommit updates docid limit", SearchableFeedViewFixture)
{
DocumentContext dc = f.doc1();
f.putAndWait(dc);
diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
index eba87447f86..7ccaa6e9fbf 100644
--- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
@@ -14,6 +14,7 @@
#include <vespa/searchcore/proton/test/mock_summary_adapter.h>
#include <vespa/searchcore/proton/test/thread_utils.h>
#include <vespa/searchlib/index/docbuilder.h>
+#include <vespa/vespalib/util/destructor_callbacks.h>
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/log/log.h>
@@ -240,13 +241,24 @@ struct FixtureBase {
}
template <typename FunctionType>
+ void runInMasterAndSyncAll(FunctionType func) {
+ test::runInMasterAndSyncAll(writeService, func);
+ }
+ template <typename FunctionType>
+ void runInMasterAndSync(FunctionType func) {
+ test::runInMasterAndSync(writeService, func);
+ }
+ template <typename FunctionType>
void runInMaster(FunctionType func) {
test::runInMaster(writeService, func);
}
void force_commit() {
- runInMaster([this] () { static_cast<IFeedView&>(*feedview).forceCommit(serial_num); });
- writeService.sync_all_executors();
+ vespalib::Gate gate;
+ runInMaster([this, &gate]() {
+ feedview->forceCommit(search::CommitParam(serial_num), std::make_shared<vespalib::GateCallback>(gate));
+ });
+ gate.await();
}
};
@@ -281,7 +293,7 @@ TEST_F("require that prepareMove sets target db document id", Fixture)
{
Document::SP doc(new Document);
MoveOperation op(BucketId(20, 42), Timestamp(10), doc, 1, subdb_id + 1);
- f.runInMaster([&] () { f.feedview->prepareMove(op); });
+ f.runInMasterAndSync([&]() { f.feedview->prepareMove(op); });
DbDocumentId targetId = op.getDbDocumentId();
EXPECT_EQUAL(subdb_id, targetId.getSubDbId());
@@ -309,8 +321,8 @@ TEST_F("require that handleMove() adds document to target and removes it from so
{ // move from (subdb_id + 1) -> this (subdb_id)
MoveOperation::UP op = makeMoveOp(DbDocumentId(subdb_id + 1, 1), subdb_id);
TEST_DO(f.assertPutCount(0));
- f.runInMaster([&]() { f.feedview->prepareMove(*op); });
- f.runInMaster([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); });
+ f.runInMasterAndSync([&]() { f.feedview->prepareMove(*op); });
+ f.runInMasterAndSyncAll([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); });
TEST_DO(f.assertPutCount(1));
TEST_DO(f.assertAndClearMoveOp());
lid = op->getDbDocumentId().getLid();
@@ -322,7 +334,7 @@ TEST_F("require that handleMove() adds document to target and removes it from so
MoveOperation::UP op = makeMoveOp(DbDocumentId(subdb_id, 1), subdb_id + 1);
op->setDbDocumentId(DbDocumentId(subdb_id + 1, 1));
TEST_DO(f.assertRemoveCount(0));
- f.runInMaster([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); });
+ f.runInMasterAndSyncAll([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); });
EXPECT_FALSE(f.metaStore->validLid(lid));
TEST_DO(f.assertRemoveCount(1));
TEST_DO(f.assertAndClearMoveOp());
@@ -334,19 +346,23 @@ TEST_F("require that handleMove() handles move within same subdb and propagates
Document::SP doc(new Document);
DocumentId doc1id("id:test:foo:g=foo:1");
uint32_t docSize = 1;
- f.runInMaster([&] () { f.metaStore->put(doc1id.getGlobalId(),
- doc1id.getGlobalId().convertToBucketId(),
- Timestamp(9), docSize, 1, 0u); });
- f.runInMaster([&] () { f.metaStore->put(doc->getId().getGlobalId(),
- doc->getId().getGlobalId().convertToBucketId(),
- Timestamp(10), docSize, 2, 0u); });
- f.runInMaster([&] () { f.metaStore->remove(1, 0u); });
+ f.runInMasterAndSync([&]() {
+ f.metaStore->put(doc1id.getGlobalId(),
+ doc1id.getGlobalId().convertToBucketId(),
+ Timestamp(9), docSize, 1, 0u);
+ });
+ f.runInMasterAndSync([&]() {
+ f.metaStore->put(doc->getId().getGlobalId(),
+ doc->getId().getGlobalId().convertToBucketId(),
+ Timestamp(10), docSize, 2, 0u);
+ });
+ f.runInMasterAndSync([&]() { f.metaStore->remove(1, 0u); });
f.metaStore->removes_complete({ 1 });
MoveOperation::UP op = makeMoveOp(doc, DbDocumentId(subdb_id, 2), subdb_id);
op->setTargetLid(1);
TEST_DO(f.assertPutCount(0));
TEST_DO(f.assertRemoveCount(0));
- f.runInMaster([&] () { f.feedview->handleMove(*op, f.beginMoveOp()); });
+ f.runInMasterAndSyncAll([&]() { f.feedview->handleMove(*op, f.beginMoveOp()); });
TEST_DO(f.assertPutCount(1));
TEST_DO(f.assertRemoveCount(1));
TEST_DO(f.assertAndClearMoveOp());
@@ -366,7 +382,7 @@ TEST_F("require that prune removed documents removes documents",
PruneRemovedDocumentsOperation op(lids->getDocIdLimit(), subdb_id);
op.setLidsToRemove(lids);
op.setSerialNum(1); // allows use of meta store.
- f.runInMaster([&] () { f.feedview->handlePruneRemovedDocuments(op); });
+ f.runInMasterAndSyncAll([&]() { f.feedview->handlePruneRemovedDocuments(op); });
EXPECT_EQUAL(2, f.removeCount);
EXPECT_FALSE(f.metaStore->validLid(1));
@@ -382,7 +398,7 @@ TEST_F("require that heartbeat propagates and commits meta store", Fixture)
EXPECT_EQUAL(0, f.feedview->heartBeatIndexedFieldsCount);
EXPECT_EQUAL(0, f.feedview->heartBeatAttributesCount);
EXPECT_EQUAL(0, f.heartbeatCount);
- f.runInMaster([&] () { f.feedview->heartBeat(2); });
+ f.runInMasterAndSyncAll([&]() { f.feedview->heartBeat(2); });
EXPECT_EQUAL(2u, f.metaStore->getStatus().getLastSyncToken());
EXPECT_EQUAL(1, f.feedview->heartBeatIndexedFieldsCount);
EXPECT_EQUAL(1, f.feedview->heartBeatAttributesCount);
diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
index 50a8349b859..a59b3e9bc6f 100644
--- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
@@ -43,7 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con
TEST_DO(f.assertIndexingThreads(3, 18));
TEST_DO(f.assertIndexingThreads(4, 19));
TEST_DO(f.assertIndexingThreads(4, 24));
- TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4
+ TEST_DO(f.assertIndexingThreads(11, 64));
}
TEST_F("require that indexing threads is always >= 1", Fixture(0))
diff --git a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt
index 3c9a4e05b50..c3d70bd7410 100644
--- a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt
+++ b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt
@@ -11,5 +11,4 @@ vespa_add_executable(searchcore_documentmetastore_test_app TEST
searchcore_fconfig
GTest::GTest
)
-vespa_add_test(NAME searchcore_documentmetastore_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/documentmetastore_test.sh
- DEPENDS searchcore_documentmetastore_test_app)
+vespa_add_test(NAME searchcore_documentmetastore_test_app COMMAND searchcore_documentmetastore_test_app)
diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
index 722ea25a14e..c96b0825789 100644
--- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
@@ -336,20 +336,22 @@ TEST(DocumentMetaStore, gids_can_be_cleared)
assertLid(1, gid1, dms);
EXPECT_EQ(1u, dms.getNumUsedLids());
EXPECT_TRUE(dms.remove(1, 0u));
- dms.removes_complete({ 1 });
+ dms.commit();
EXPECT_EQ(0u, dms.getNumUsedLids());
EXPECT_TRUE(!dms.getGid(1, gid));
EXPECT_TRUE(!dms.getLid(gid1, lid));
+ dms.removes_complete({ 1 });
// reuse lid
addGid(dms, gid2, bucketId2, time2);
assertGid(gid2, 1, dms);
assertLid(1, gid2, dms);
EXPECT_EQ(1u, dms.getNumUsedLids());
EXPECT_TRUE(dms.remove(1, 0u));
- dms.removes_complete({ 1 });
+ dms.commit();
EXPECT_EQ(0u, dms.getNumUsedLids());
EXPECT_TRUE(!dms.getGid(1, gid));
EXPECT_TRUE(!dms.getLid(gid2, lid));
+ dms.removes_complete({ 1 });
EXPECT_TRUE(!dms.remove(1, 0u)); // not used
EXPECT_TRUE(!dms.remove(2, 0u)); // outside range
}
@@ -519,6 +521,7 @@ TEST(DocumentMetaStoreTest, gids_can_be_saved_and_loaded)
EXPECT_EQ(numLids + 1, dms2.getNumDocs());
EXPECT_EQ(numLids - (3 - i), dms2.getNumUsedLids());
}
+ vespalib::unlink("documentmetastore2.dat");
}
TEST(DocumentMetaStoreTest, bucket_used_bits_are_lbounded_at_load_time)
@@ -544,6 +547,7 @@ TEST(DocumentMetaStoreTest, bucket_used_bits_are_lbounded_at_load_time)
BucketId expected_bucket(storage::spi::BucketLimits::MinUsedBits, gid.convertToBucketId().getRawId());
assertGid(gid, lid, dms2, expected_bucket, Timestamp(1000));
+ vespalib::unlink("documentmetastore2.dat");
}
TEST(DocumentMetaStore, stats_are_updated)
@@ -1732,6 +1736,7 @@ remove(uint32_t startLid, uint32_t shrinkTarget, DocumentMetaStore &dms)
{
for (uint32_t lid = startLid; lid >= shrinkTarget; --lid) {
dms.remove(lid, 0u);
+ dms.commit();
dms.removes_complete({ lid });
}
}
@@ -1765,8 +1770,6 @@ TEST(DocumentMetaStoreTest, shrink_via_flush_target_works)
DummyFileHeaderContext fileHeaderContext;
DummyTlsSyncer dummyTlsSyncer;
HwInfo hwInfo;
- vespalib::rmdir("dmsflush", true);
- vespalib::mkdir("dmsflush");
using Type = IFlushTarget::Type;
using Component = IFlushTarget::Component;
IFlushTarget::SP ft(std::make_shared<ShrinkLidSpaceFlushTarget>
@@ -1903,6 +1906,8 @@ TEST(DocumentMetaStoreTest, document_sizes_are_saved)
assertSize(dms4, 1, 1);
assertSize(dms4, 2, 1);
assertSize(dms4, 3, 1);
+ vespalib::unlink("documentmetastore3.dat");
+ vespalib::unlink("documentmetastore4.dat");
}
namespace {
@@ -1944,12 +1949,12 @@ TEST(DocumentMetaStoreTest, multiple_lids_can_be_removed_with_removeBatch)
assertLidGidFound(4, dms);
dms.removeBatch({1, 3}, 5);
- dms.removes_complete({1, 3});
-
+ dms.commit();
assertLidGidNotFound(1, dms);
assertLidGidFound(2, dms);
assertLidGidNotFound(3, dms);
assertLidGidFound(4, dms);
+ dms.removes_complete({1, 3});
}
class MockOperationListener : public documentmetastore::OperationListener {
diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh
deleted file mode 100755
index 199a71106f8..00000000000
--- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-set -e
-$VALGRIND ./searchcore_documentmetastore_test_app
-rm -rf documentmetastore*.dat
-rm -rf dmsflush
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
index 33b58e32669..8265585a038 100644
--- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
@@ -136,8 +136,13 @@ public:
}
template <typename FunctionType>
- void runInMaster(FunctionType func) {
- test::runInMaster(_writeService, func);
+ void runInMasterAndSyncAll(FunctionType func) {
+ test::runInMasterAndSyncAll(_writeService, func);
+ }
+
+ template <typename FunctionType>
+ void runInMasterAndSync(FunctionType func) {
+ test::runInMasterAndSync(_writeService, func);
}
void cycledLids(const std::vector<uint32_t> &lids) {
@@ -155,19 +160,16 @@ public:
}
void delayReuse(uint32_t lid) {
- runInMaster([&] () { _lidReuseDelayer->delayReuse(lid); } );
+ runInMasterAndSync([&]() { _lidReuseDelayer->delayReuse(lid); });
}
void delayReuse(const std::vector<uint32_t> &lids) {
- runInMaster([&] () { _lidReuseDelayer->delayReuse(lids); });
+ runInMasterAndSync([&]() { _lidReuseDelayer->delayReuse(lids); });
}
void commit() {
- runInMaster([&] () { cycleLids(_lidReuseDelayer->getReuseLids()); });
+ runInMasterAndSyncAll([&]() { cycleLids(_lidReuseDelayer->getReuseLids()); });
}
-
- void sync() { _writeService.sync_all_executors(); }
-
};
TEST_F("require that nothing happens before free list is active", Fixture)
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
index d34e2ae667e..d6bbc77aa09 100644
--- a/searchcore/src/tests/proton/index/indexmanager_test.cpp
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -104,8 +104,6 @@ void push_documents_and_wait(search::memoryindex::DocumentInverter &inverter) {
gate.await();
}
-std::shared_ptr<vespalib::IDestructorCallback> emptyDestructorCallback;
-
struct IndexManagerTest : public ::testing::Test {
SerialNum _serial_num;
IndexManagerDummyReconfigurer _reconfigurer;
@@ -128,7 +126,6 @@ struct IndexManagerTest : public ::testing::Test {
{
removeTestData();
vespalib::mkdir(index_dir, false);
- _writeService.sync_all_executors();
resetIndexManager();
}
@@ -138,22 +135,31 @@ struct IndexManagerTest : public ::testing::Test {
template <class FunctionType>
inline void runAsMaster(FunctionType &&function) {
- _writeService.master().execute(makeLambdaTask(std::move(function)));
- _writeService.master().sync();
+ vespalib::Gate gate;
+ _writeService.master().execute(makeLambdaTask([&gate,function = std::move(function)]() {
+ function();
+ gate.countDown();
+ }));
+ gate.await();
}
template <class FunctionType>
inline void runAsIndex(FunctionType &&function) {
- _writeService.index().execute(makeLambdaTask(std::move(function)));
- _writeService.index().sync();
+ vespalib::Gate gate;
+ _writeService.index().execute(makeLambdaTask([&gate,function = std::move(function)]() {
+ function();
+ gate.countDown();
+ }));
+ gate.await();
}
void flushIndexManager();
Document::UP addDocument(uint32_t docid);
void resetIndexManager();
void removeDocument(uint32_t docId, SerialNum serialNum) {
vespalib::Gate gate;
- runAsIndex([&]() { _index_manager->removeDocument(docId, serialNum);
- _index_manager->commit(serialNum, std::make_shared<vespalib::GateCallback>(gate));
- });
+ runAsIndex([&]() {
+ _index_manager->removeDocument(docId, serialNum);
+ _index_manager->commit(serialNum, std::make_shared<vespalib::GateCallback>(gate));
+ });
gate.await();
}
void removeDocument(uint32_t docId) {
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
index b64e972c4cf..3474a4297c7 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp
@@ -38,6 +38,7 @@ class ListenerStats {
uint32_t _createdListeners;
uint32_t _registeredListeners;
uint32_t _destroyedListeners;
+ std::vector<GlobalId> _initial_removes;
public:
ListenerStats() noexcept
@@ -46,7 +47,8 @@ public:
_removeChanges(0u),
_createdListeners(0u),
_registeredListeners(0u),
- _destroyedListeners(0u)
+ _destroyedListeners(0u),
+ _initial_removes()
{
}
@@ -64,7 +66,7 @@ public:
++_removeChanges;
}
void markCreatedListener() { lock_guard guard(_lock); ++_createdListeners; }
- void markRegisteredListener() { lock_guard guard(_lock); ++_registeredListeners; }
+ void markRegisteredListener(const std::vector<GlobalId>& removes) { lock_guard guard(_lock); ++_registeredListeners; _initial_removes = removes; }
void markDestroyedListener() { lock_guard guard(_lock); ++_destroyedListeners; }
uint32_t getCreatedListeners() const { return _createdListeners; }
@@ -84,6 +86,7 @@ public:
EXPECT_EQUAL(expPutChanges, _putChanges);
EXPECT_EQUAL(expRemoveChanges, _removeChanges);
}
+ const std::vector<GlobalId>& get_initial_removes() const noexcept { return _initial_removes; }
};
class MyListener : public IGidToLidChangeListener
@@ -105,7 +108,7 @@ public:
~MyListener() override { _stats.markDestroyedListener(); }
void notifyPutDone(IDestructorCallbackSP, GlobalId, uint32_t) override { _stats.notifyPutDone(); }
void notifyRemove(IDestructorCallbackSP, GlobalId) override { _stats.notifyRemove(); }
- void notifyRegistered() override { _stats.markRegisteredListener(); }
+ void notifyRegistered(const std::vector<GlobalId>& removes) override { _stats.markRegisteredListener(removes); }
const vespalib::string &getName() const override { return _name; }
const vespalib::string &getDocTypeName() const override { return _docTypeName; }
};
@@ -233,6 +236,16 @@ TEST_F("Test that we keep old listener when registering duplicate", Fixture)
TEST_DO(stats.assertListeners(2, 1, 1));
}
+TEST_F("Test that pending removes are passed on to new listener", Fixture)
+{
+ auto& stats = f.addStats();
+ auto listener = std::make_unique<MyListener>(stats, "test1", "testdoc");
+ f.notifyRemove(toGid(doc1), 20);
+ f.addListener(std::move(listener));
+ EXPECT_TRUE((std::vector<GlobalId>{ toGid(doc1) }) == stats.get_initial_removes());
+ f.commit();
+}
+
class StatsFixture : public Fixture
{
ListenerStats &_stats;
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
index 15281563b93..e3b1fd3aa15 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
@@ -103,8 +103,8 @@ struct Fixture
gate.await();
}
- void notifyListenerRegistered() {
- _listener->notifyRegistered();
+ void notifyListenerRegistered(const std::vector<GlobalId>& removes) {
+ _listener->notifyRegistered(removes);
}
};
@@ -144,7 +144,7 @@ TEST_F("Test that target lids are populated when listener is registered", Fixtur
std::make_shared<MyGidToLidMapperFactory>();
f._attr->setGidToLidMapperFactory(factory);
f.allocListener();
- f.notifyListenerRegistered();
+ f.notifyListenerRegistered({});
TEST_DO(f.assertTargetLid(10, 1));
TEST_DO(f.assertTargetLid(17, 2));
TEST_DO(f.assertTargetLid(10, 3));
@@ -152,6 +152,31 @@ TEST_F("Test that target lids are populated when listener is registered", Fixtur
TEST_DO(f.assertNoTargetLid(5));
}
+TEST_F("Test that removed target lids are pruned when listener is registered", Fixture)
+{
+ f.ensureDocIdLimit(6);
+ f.set(1, toGid(doc1));
+ f.set(2, toGid(doc2));
+ f.set(3, toGid(doc1));
+ f.set(4, toGid(doc3));
+ f.commit();
+ TEST_DO(f.assertTargetLid(0, 1));
+ TEST_DO(f.assertTargetLid(0, 2));
+ TEST_DO(f.assertTargetLid(0, 3));
+ TEST_DO(f.assertTargetLid(0, 4));
+ TEST_DO(f.assertNoTargetLid(5));
+ std::shared_ptr<search::IGidToLidMapperFactory> factory =
+ std::make_shared<MyGidToLidMapperFactory>();
+ f._attr->setGidToLidMapperFactory(factory);
+ f.allocListener();
+ f.notifyListenerRegistered({ toGid(doc1) });
+ TEST_DO(f.assertTargetLid(0, 1));
+ TEST_DO(f.assertTargetLid(17, 2));
+ TEST_DO(f.assertTargetLid(0, 3));
+ TEST_DO(f.assertTargetLid(0, 4));
+ TEST_DO(f.assertNoTargetLid(5));
+}
+
}
TEST_MAIN()
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
index 8cff0b1ea6c..c2efa8c2389 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp
@@ -26,7 +26,7 @@ public:
~MyListener() override { }
void notifyPutDone(IDestructorCallbackSP, document::GlobalId, uint32_t) override { }
void notifyRemove(IDestructorCallbackSP, document::GlobalId) override { }
- void notifyRegistered() override { }
+ void notifyRegistered(const std::vector<document::GlobalId>&) override { }
const vespalib::string &getName() const override { return _name; }
const vespalib::string &getDocTypeName() const override { return _docTypeName; }
};
diff --git a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp
index eccae8fe8ab..d5421eaaeca 100644
--- a/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp
+++ b/searchcore/src/vespa/searchcore/bmcluster/spi_bm_feed_handler.cpp
@@ -134,7 +134,9 @@ SpiBmFeedHandler::remove(const document::Bucket& bucket, const DocumentId& docum
auto provider = get_provider(bucket);
if (provider) {
Bucket spi_bucket(bucket);
- provider->removeAsync(spi_bucket, Timestamp(timestamp), document_id, context, std::make_unique<MyOperationComplete>(provider, _errors, spi_bucket, tracker));
+ std::vector<storage::spi::PersistenceProvider::TimeStampAndDocumentId> ids;
+ ids.emplace_back(Timestamp(timestamp), document_id);
+ provider->removeAsync(spi_bucket, std::move(ids), context, std::make_unique<MyOperationComplete>(provider, _errors, spi_bucket, tracker));
} else {
++_errors;
}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index f3d90d37e42..77b6d8d1392 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -259,7 +259,7 @@ applyCommit(CommitParam param, AttributeWriter::OnWriteDoneType , AttributeVecto
void
applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum, AttributeVector &attr)
{
- if (attr.getStatus().getLastSyncToken() < serialNum) {
+ if (attr.getStatus().getLastSyncToken() <= serialNum) {
/*
* If the attribute is an empty placeholder attribute due to
* later config changes removing the attribute then it might
@@ -657,7 +657,6 @@ AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr)
_attributeFieldWriter(_mgr->getAttributeFieldWriter()),
_shared_executor(_mgr->get_shared_executor()),
_writeContexts(),
- _dataType(nullptr),
_hasStructFieldAttribute(false),
_attrMap()
{
@@ -673,9 +672,18 @@ void AttributeWriter::setupAttributeMapping() {
}
-AttributeWriter::~AttributeWriter()
-{
- _attributeFieldWriter.sync_all();
+AttributeWriter::~AttributeWriter() {
+ vespalib::Gate gate;
+ drain(std::make_shared<vespalib::GateCallback>(gate));
+ gate.await();
+}
+
+void
+AttributeWriter::drain(OnWriteDoneType onDone) {
+
+ for (const auto &wc : _writeContexts) {
+ _attributeFieldWriter.executeLambda(wc.getExecutorId(), [onDone] () { (void) onDone; });
+ }
}
std::vector<search::AttributeVector *>
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
index 726005ae04b..dc543c19222 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
@@ -83,13 +83,11 @@ public:
private:
using AttrMap = vespalib::hash_map<vespalib::string, AttributeWithInfo>;
std::vector<WriteContext> _writeContexts;
- const DataType *_dataType;
bool _hasStructFieldAttribute;
AttrMap _attrMap;
void setupWriteContexts();
void setupAttributeMapping();
- void buildFieldPaths(const DocumentType &docType, const DataType *dataType);
void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid,
bool allAttributes, OnWriteDoneType onWriteDone);
void internalRemove(SerialNum serialNum, DocumentIdT lid, OnWriteDoneType onWriteDone);
@@ -120,6 +118,7 @@ public:
void onReplayDone(uint32_t docIdLimit) override;
bool hasStructFieldAttribute() const override;
+ void drain(OnWriteDoneType onWriteDone) override;
// Should only be used for unit testing.
const std::vector<WriteContext>& get_write_contexts() const {
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index 032be9e1dc8..eee6264b9f4 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -22,6 +22,8 @@
#include <vespa/vespalib/util/threadexecutor.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/gate.h>
+#include <vespa/vespalib/util/destructor_callbacks.h>
#include <vespa/log/log.h>
LOG_SETUP(".proton.attribute.attributemanager");
@@ -59,14 +61,14 @@ search::SerialNum estimateShrinkSerialNum(const AttributeVector &attr)
return std::max(attr.getStatus().getLastSyncToken(), serialNum);
}
-std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector::SP &attr, vespalib::ISequencedTaskExecutor &attributeFieldWriter, AttributeDiskLayout &diskLayout)
+std::shared_ptr<ShrinkLidSpaceFlushTarget>
+allocShrinker(const AttributeVector::SP &attr, vespalib::ISequencedTaskExecutor & executor, AttributeDiskLayout &diskLayout)
{
using Type = IFlushTarget::Type;
using Component = IFlushTarget::Component;
- auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter,
- attributeFieldWriter.getExecutorIdFromName(
- attr->getNamePrefix()));
+ auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, executor,
+ executor.getExecutorIdFromName(attr->getNamePrefix()));
const vespalib::string &name = attr->getName();
auto dir = diskLayout.createAttributeDir(name);
search::SerialNum shrinkSerialNum = estimateShrinkSerialNum(*attr);
@@ -167,24 +169,32 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr,
const Spec &newSpec,
Spec::AttributeList &toBeAdded)
{
- for (const auto &aspec : newSpec.getAttributes()) {
- AttributeVector::SP av = currMgr.findAttribute(aspec.getName());
- if (matchingTypes(av, aspec.getConfig())) { // transfer attribute
- LOG(debug, "Transferring attribute vector '%s' with %u docs and serial number %" PRIu64 " from current manager",
- av->getName().c_str(), av->getNumDocs(), av->getStatus().getLastSyncToken());
- auto wrap = currMgr.findFlushable(aspec.getName());
- assert(wrap != nullptr);
- auto shrinker = wrap->getShrinker();
- assert(shrinker);
- addAttribute(AttributeWrap::normalAttribute(av), shrinker);
- auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix());
- auto cfg = aspec.getConfig();
- _attributeFieldWriter.execute(id, [av, cfg]() { av->update_config(cfg); });
- } else {
- toBeAdded.push_back(aspec);
+ vespalib::Gate gate;
+ {
+ auto gateCallback = std::make_shared<vespalib::GateCallback>(gate);
+ for (const auto &aspec: newSpec.getAttributes()) {
+ AttributeVector::SP av = currMgr.findAttribute(aspec.getName());
+ if (matchingTypes(av, aspec.getConfig())) { // transfer attribute
+ LOG(debug,
+ "Transferring attribute vector '%s' with %u docs and serial number %" PRIu64 " from current manager",
+ av->getName().c_str(), av->getNumDocs(), av->getStatus().getLastSyncToken());
+ auto wrap = currMgr.findFlushable(aspec.getName());
+ assert(wrap != nullptr);
+ auto shrinker = wrap->getShrinker();
+ assert(shrinker);
+ addAttribute(AttributeWrap::normalAttribute(av), shrinker);
+ auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix());
+ auto cfg = aspec.getConfig();
+ _attributeFieldWriter.execute(id, [av, cfg, gateCallback]() {
+ (void) gateCallback;
+ av->update_config(cfg);
+ });
+ } else {
+ toBeAdded.push_back(aspec);
+ }
}
}
- _attributeFieldWriter.sync_all();
+ gate.await();
}
void
@@ -585,7 +595,7 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor>
}
void
-AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const
+AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const
{
for (const auto &attr : _attributes) {
if (attr.second.isExtra()) {
@@ -594,7 +604,10 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func)
}
AttributeVector::SP attrsp = attr.second.getAttribute();
_attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()),
- [attrsp, func]() { (*func)(*attrsp); });
+ [attrsp, func, onDone]() {
+ (void) onDone;
+ (*func)(*attrsp);
+ });
}
}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
index e2b9550435d..08e2d511d70 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -178,7 +178,7 @@ public:
const std::vector<search::AttributeVector *> &getWritableAttributes() const override;
void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const override;
- void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override;
+ void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const override;
void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override;
ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
index c7ab83ae590..5f162281d96 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
@@ -206,7 +206,7 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFun
}
void
-FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const
+FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const
{
// Run by document db master thread
std::vector<AttributeGuard> completeList;
@@ -217,7 +217,10 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>
// Name must be extracted in document db master thread or attribute
// writer thread
attributeFieldWriter.execute(attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()),
- [attrsp, func]() { (*func)(*attrsp); });
+ [attrsp, func, onDone]() {
+ (void) onDone;
+ (*func)(*attrsp);
+ });
}
}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
index 1ae5f452218..1512ab32d62 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -52,7 +52,7 @@ public:
search::AttributeVector * getWritableAttribute(const vespalib::string &name) const override;
const std::vector<search::AttributeVector *> & getWritableAttributes() const override;
void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const override;
- void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const override;
+ void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const override;
ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
const ImportedAttributesRepo *getImportedAttributes() const override;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
index d55cd45d014..b8968ba9d2e 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -31,7 +31,7 @@ class ImportedAttributesRepo;
struct IAttributeManager : public search::IAttributeManager
{
using SP = std::shared_ptr<IAttributeManager>;
- using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &;
+ using OnDone = std::shared_ptr<vespalib::IDestructorCallback>;
using IAttributeFunctor = search::attribute::IAttributeFunctor;
using IConstAttributeFunctor = search::attribute::IConstAttributeFunctor;
@@ -98,7 +98,7 @@ struct IAttributeManager : public search::IAttributeManager
virtual const std::vector<search::AttributeVector *> &getWritableAttributes() const = 0;
virtual void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const = 0;
- virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func) const = 0;
+ virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const = 0;
virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
index 1e959a4fb77..0f739f6ffea 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h
@@ -64,6 +64,7 @@ public:
virtual void onReplayDone(uint32_t docIdLimit) = 0;
virtual bool hasStructFieldAttribute() const = 0;
+ virtual void drain(OnWriteDoneType onWriteDone) = 0;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
index 8fb1415cee8..1342324b727 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
@@ -28,7 +28,6 @@ DocumentStoreAdapter::writeStringField(const char * buf, uint32_t buflen, ResTyp
case RES_STRING:
return _resultPacker.AddString(buf, buflen);
case RES_LONG_STRING:
- case RES_XMLSTRING:
case RES_JSONSTRING:
return _resultPacker.AddLongString(buf, buflen);
default:
@@ -57,7 +56,6 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type)
return _resultPacker.AddDouble(value.getAsDouble());
case RES_STRING:
case RES_LONG_STRING:
- case RES_XMLSTRING:
case RES_JSONSTRING:
{
if (value.getClass().inherits(LiteralFieldValueB::classId)) {
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
index 778273c9777..3170654409b 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
@@ -207,6 +207,8 @@ void
DocumentMetaStore::onCommit()
{
if (consider_compact_gid_to_lid_map()) {
+ incGeneration();
+ _changesSinceCommit = 0;
_gidToLidMap.compact_worst();
_gid_to_lid_map_write_itr_prepare_serial_num = 0u;
_gid_to_lid_map_write_itr.begin(_gidToLidMap.getRoot());
@@ -690,7 +692,7 @@ DocumentMetaStore::removeBatch(const std::vector<DocId> &lidsToRemove, const uin
bucketdb::Guard bucketGuard = _bucketDB->takeGuard();
bucketGuard->remove_batch(bdb_removed, _subDbType);
}
- incGeneration();
+ ++_changesSinceCommit;
if (_op_listener) {
_op_listener->notify_remove_batch();
}
diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h
index 758281794b0..bc0d09e5cd4 100644
--- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h
+++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.h
@@ -41,7 +41,6 @@ class TaskRunner {
}
void setDone() { execute(std::move(_doneTask)); }
const InitializerTask::SP &rootTask() { return _rootTask; }
- void schedulePoll();
};
void getReadyTasks(const InitializerTask::SP task, TaskList &readyTasks, TaskSet &checked);
void setTaskRunning(InitializerTask &task);
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
index 9d9e1e0a344..7e21e619419 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
@@ -368,23 +368,61 @@ PersistenceEngine::putAsync(const Bucket &bucket, Timestamp ts, storage::spi::Do
}
void
-PersistenceEngine::removeAsync(const Bucket& b, Timestamp t, const DocumentId& did, Context&, OperationComplete::UP onComplete)
+PersistenceEngine::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP onComplete)
+{
+ if (ids.size() == 1) {
+ removeAsyncSingle(b, ids[0].first, ids[0].second, std::move(onComplete));
+ } else {
+ removeAsyncMulti(b, std::move(ids), std::move(onComplete));
+ }
+}
+
+void
+PersistenceEngine::removeAsyncMulti(const Bucket& b, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP onComplete) {
+ ReadGuard rguard(_rwMutex);
+ //TODO Group per document type/handler and handle in one go.
+ for (const TimeStampAndDocumentId & stampedId : ids) {
+ const document::DocumentId & id = stampedId.second;
+ if (!id.hasDocType()) {
+ return onComplete->onComplete(
+ std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR,
+ fmt("Old id scheme not supported in elastic mode (%s)", id.toString().c_str())));
+ }
+ DocTypeName docType(id.getDocType());
+ IPersistenceHandler *handler = getHandler(rguard, b.getBucketSpace(), docType);
+ if (!handler) {
+ return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR,
+ fmt("No handler for document type '%s'",
+ docType.toString().c_str())));
+ }
+ }
+ auto transportContext = std::make_shared<AsyncRemoveTransportContext>(ids.size(), std::move(onComplete));
+ for (const TimeStampAndDocumentId & stampedId : ids) {
+ const document::DocumentId & id = stampedId.second;
+ DocTypeName docType(id.getDocType());
+ IPersistenceHandler *handler = getHandler(rguard, b.getBucketSpace(), docType);
+ handler->handleRemove(feedtoken::make(transportContext), b, stampedId.first, id);
+ }
+}
+
+void
+PersistenceEngine::removeAsyncSingle(const Bucket& b, Timestamp t, const DocumentId& id, OperationComplete::UP onComplete)
{
ReadGuard rguard(_rwMutex);
LOG(spam, "remove(%s, %" PRIu64 ", \"%s\")", b.toString().c_str(),
- static_cast<uint64_t>(t.getValue()), did.toString().c_str());
- if (!did.hasDocType()) {
+ static_cast<uint64_t>(t.getValue()), id.toString().c_str());
+ if (!id.hasDocType()) {
return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR,
- fmt("Old id scheme not supported in elastic mode (%s)", did.toString().c_str())));
+ fmt("Old id scheme not supported in elastic mode (%s)", id.toString().c_str())));
}
- DocTypeName docType(did.getDocType());
+ DocTypeName docType(id.getDocType());
IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType);
if (!handler) {
return onComplete->onComplete(std::make_unique<RemoveResult>(Result::ErrorType::PERMANENT_ERROR,
fmt("No handler for document type '%s'", docType.toString().c_str())));
}
auto transportContext = std::make_shared<AsyncTransportContext>(1, std::move(onComplete));
- handler->handleRemove(feedtoken::make(std::move(transportContext)), b, t, did);
+ handler->handleRemove(feedtoken::make(std::move(transportContext)), b, t, id);
}
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
index fe564d01459..7c8040fae9d 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
@@ -88,7 +88,8 @@ private:
void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc);
ClusterState::SP savedClusterState(BucketSpace bucketSpace) const;
std::shared_ptr<BucketExecutor> get_bucket_executor() noexcept { return _bucket_executor.lock(); }
-
+ void removeAsyncSingle(const Bucket&, Timestamp, const document::DocumentId &id, OperationComplete::UP);
+ void removeAsyncMulti(const Bucket&, std::vector<TimeStampAndDocumentId> ids, OperationComplete::UP);
public:
typedef std::unique_ptr<PersistenceEngine> UP;
@@ -106,7 +107,7 @@ public:
void setActiveStateAsync(const Bucket&, BucketInfo::ActiveState, OperationComplete::UP) override;
BucketInfoResult getBucketInfo(const Bucket&) const override;
void putAsync(const Bucket &, Timestamp, storage::spi::DocumentSP, Context &context, OperationComplete::UP) override;
- void removeAsync(const Bucket&, Timestamp, const document::DocumentId&, Context&, OperationComplete::UP) override;
+ void removeAsync(const Bucket&, std::vector<TimeStampAndDocumentId> ids, Context&, OperationComplete::UP) override;
void updateAsync(const Bucket&, Timestamp, storage::spi::DocumentUpdateSP, Context&, OperationComplete::UP) override;
GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override;
CreateIteratorResult
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
index 8a0955b3147..bf70304c4f4 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
@@ -5,6 +5,7 @@
using vespalib::make_string;
using storage::spi::Result;
+using storage::spi::RemoveResult;
namespace proton {
@@ -30,14 +31,19 @@ TransportMerger::mergeResult(ResultUP result, bool documentWasFound) {
}
}
+Result::UP
+TransportMerger::merge(ResultUP accum, ResultUP incoming, bool documentWasFound) {
+ return documentWasFound ? std::move(incoming) : std::move(accum);
+}
+
void
TransportMerger::mergeWithLock(ResultUP result, bool documentWasFound) {
if (!_result) {
_result = std::move(result);
} else if (result->hasError()) {
_result = std::make_unique<Result>(mergeErrorResults(*_result, *result));
- } else if (documentWasFound) {
- _result = std::move(result);
+ } else {
+ _result = merge(std::move(_result), std::move(result), documentWasFound);
}
completeIfDone();
}
@@ -93,4 +99,11 @@ AsyncTransportContext::send(ResultUP result, bool documentWasFound)
mergeResult(std::move(result), documentWasFound);
}
+Result::UP
+AsyncRemoveTransportContext::merge(ResultUP accum, ResultUP incoming, bool) {
+ // TODO This can be static cast if necessary.
+ dynamic_cast<RemoveResult *>(accum.get())->inc_num_removed(dynamic_cast<RemoveResult *>(incoming.get())->num_removed());
+ return accum;
+}
+
} // proton
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h
index 91cadfedcd5..b792124136d 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.h
@@ -23,6 +23,7 @@ protected:
~TransportMerger() override;
void mergeResult(ResultUP result, bool documentWasFound);
virtual void completeIfDone() { } // Called with lock held if necessary on every merge
+ virtual ResultUP merge(ResultUP accum, ResultUP incoming, bool documentWasFound);
ResultUP _result;
private:
@@ -47,15 +48,10 @@ public:
void await() {
_latch.await();
}
- const UpdateResult &getUpdateResult() const {
- return dynamic_cast<const UpdateResult &>(*_result);
- }
+
const Result &getResult() const {
return *_result;
}
- const RemoveResult &getRemoveResult() const {
- return dynamic_cast<const RemoveResult &>(*_result);
- }
};
@@ -77,5 +73,11 @@ public:
void send(ResultUP result, bool documentWasFound) override;
};
-} // namespace proton
+class AsyncRemoveTransportContext : public AsyncTransportContext {
+public:
+ using AsyncTransportContext::AsyncTransportContext;
+protected:
+ ResultUP merge(ResultUP accum, ResultUP incoming, bool documentWasFound) override;
+};
+}
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp
index 3de1ac09c09..ed672cc9974 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp
@@ -142,7 +142,13 @@ GidToLidChangeHandler::addListener(std::unique_ptr<IGidToLidChangeListener> list
}
}
_listeners.emplace_back(std::move(listener));
- _listeners.back()->notifyRegistered();
+ std::vector<GlobalId> removes;
+ for (auto& change : _pending_changes) {
+ if (change.is_remove()) {
+ removes.emplace_back(change.get_gid());
+ }
+ }
+ _listeners.back()->notifyRegistered(removes);
} else {
assert(_listeners.empty());
}
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
index be03b57e088..7f0f27e3c58 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
@@ -1,19 +1,20 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "gid_to_lid_change_listener.h"
+#include <vespa/vespalib/util/gate.h>
#include <future>
using vespalib::RetainGuard;
namespace proton {
-GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor &attributeFieldWriter,
+GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor & executor,
std::shared_ptr<search::attribute::ReferenceAttribute> attr,
RetainGuard retainGuard,
const vespalib::string &name,
const vespalib::string &docTypeName)
- : _attributeFieldWriter(attributeFieldWriter),
- _executorId(_attributeFieldWriter.getExecutorIdFromName(attr->getNamePrefix())),
+ : _executor(executor),
+ _executorId(_executor.getExecutorIdFromName(attr->getNamePrefix())),
_attr(std::move(attr)),
_retainGuard(std::move(retainGuard)),
_name(name),
@@ -22,14 +23,16 @@ GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor
GidToLidChangeListener::~GidToLidChangeListener()
{
- _attributeFieldWriter.sync_all();
+ vespalib::Gate gate;
+ _executor.executeLambda(_executorId, [&gate]() { gate.countDown(); });
+ gate.await();
}
void
GidToLidChangeListener::notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid)
{
- _attributeFieldWriter.executeLambda(_executorId,
- [this, context=std::move(context), gid, lid]() {
+ _executor.executeLambda(_executorId,
+ [this, context=std::move(context), gid, lid]() {
(void) context;
_attr->notifyReferencedPut(gid, lid);
});
@@ -38,21 +41,21 @@ GidToLidChangeListener::notifyPutDone(IDestructorCallbackSP context, document::G
void
GidToLidChangeListener::notifyRemove(IDestructorCallbackSP context, document::GlobalId gid)
{
- _attributeFieldWriter.executeLambda(_executorId,
- [this, context = std::move(context), gid]() {
+ _executor.executeLambda(_executorId,
+ [this, context = std::move(context), gid]() {
(void) context;
_attr->notifyReferencedRemove(gid);
});
}
void
-GidToLidChangeListener::notifyRegistered()
+GidToLidChangeListener::notifyRegistered(const std::vector<document::GlobalId>& removes)
{
std::promise<void> promise;
auto future = promise.get_future();
- _attributeFieldWriter.executeLambda(_executorId,
- [this, &promise]() {
- _attr->populateTargetLids();
+ _executor.executeLambda(_executorId,
+ [this, &promise, removes(std::move(removes))]() {
+ _attr->populateTargetLids(removes);
promise.set_value();
});
future.wait();
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
index 28e9684ed86..441c6377128 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
@@ -15,7 +15,7 @@ namespace proton {
*/
class GidToLidChangeListener : public IGidToLidChangeListener
{
- vespalib::ISequencedTaskExecutor &_attributeFieldWriter;
+ vespalib::ISequencedTaskExecutor &_executor;
vespalib::ISequencedTaskExecutor::ExecutorId _executorId;
std::shared_ptr<search::attribute::ReferenceAttribute> _attr;
vespalib::RetainGuard _retainGuard;
@@ -31,7 +31,7 @@ public:
~GidToLidChangeListener() override;
void notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) override;
void notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) override;
- void notifyRegistered() override;
+ void notifyRegistered(const std::vector<document::GlobalId>& removes) override;
const vespalib::string &getName() const override;
const vespalib::string &getDocTypeName() const override;
const std::shared_ptr<search::attribute::ReferenceAttribute> &getReferenceAttribute() const { return _attr; }
diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h
index 9bdeb2f2e23..75da3adc973 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h
@@ -4,6 +4,7 @@
#include <vespa/vespalib/util/idestructorcallback.h>
#include <vespa/vespalib/stllike/string.h>
+#include <vector>
namespace document { class GlobalId; }
@@ -20,7 +21,7 @@ public:
virtual ~IGidToLidChangeListener() { }
virtual void notifyPutDone(IDestructorCallbackSP context, document::GlobalId gid, uint32_t lid) = 0;
virtual void notifyRemove(IDestructorCallbackSP context, document::GlobalId gid) = 0;
- virtual void notifyRegistered() = 0;
+ virtual void notifyRegistered(const std::vector<document::GlobalId>& removes) = 0;
virtual const vespalib::string &getName() const = 0;
virtual const vespalib::string &getDocTypeName() const = 0;
};
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
index 56210fe0c85..511adbe66e9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
@@ -54,6 +54,7 @@ vespa_add_library(searchcore_server STATIC
health_adapter.cpp
heart_beat_job.cpp
idocumentdbowner.cpp
+ ifeedview.cpp
ireplayconfig.cpp
job_tracked_maintenance_job.cpp
lid_space_compaction_handler.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h
index 55d0cd479e3..d95dfd1314e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h
+++ b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h
@@ -13,8 +13,8 @@ namespace proton {
struct AttributeWriterFactory : public IAttributeWriterFactory
{
AttributeWriterFactory() {}
- virtual IAttributeWriter::SP create(const IAttributeWriter::SP &old,
- const AttributeCollectionSpec &attrSpec) const override
+ IAttributeWriter::SP create(const IAttributeWriter::SP &old,
+ const AttributeCollectionSpec &attrSpec) const override
{
const AttributeWriter &oldAdapter = dynamic_cast<const AttributeWriter &>(*old.get());
const proton::IAttributeManager::SP &oldMgr = oldAdapter.getAttributeManager();
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 427d435aae7..cc05fc7169c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -29,7 +29,6 @@
#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h>
#include <vespa/searchcore/proton/reference/document_db_reference_resolver.h>
#include <vespa/searchcore/proton/reference/i_document_db_reference_registry.h>
-#include <vespa/searchlib/attribute/attributefactory.h>
#include <vespa/searchlib/attribute/configconverter.h>
#include <vespa/searchlib/engine/docsumreply.h>
#include <vespa/searchlib/engine/searchreply.h>
@@ -104,6 +103,16 @@ public:
}
};
+template<typename T>
+void
+forceCommitAndWait(std::shared_ptr<IFeedView> feedView, SerialNum serialNum, T keepAlive) {
+ vespalib::Gate gate;
+ using Keep = vespalib::KeepAlive<std::pair<T, std::shared_ptr<IDestructorCallback>>>;
+ feedView->forceCommit(CommitParam(serialNum),
+ std::make_shared<Keep>(std::make_pair(std::move(keepAlive), std::make_shared<GateCallback>(gate))));
+ gate.await();
+}
+
}
template <typename FunctionType>
@@ -196,9 +205,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
_transient_usage_provider(std::make_shared<DocumentDBResourceUsageProvider>(*this)),
_feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)),
_subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext,
- metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir,
- DocumentSubDBCollection::Config(protonCfg.numsearcherthreads),
- hwInfo),
+ metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, hwInfo),
_maintenanceController(_writeService.master(), sharedExecutor, _refCount, _docTypeName),
_jobTrackers(),
_calc(),
@@ -378,12 +385,8 @@ DocumentDB::enterOnlineState()
{
// Called by executor thread
assert(_writeService.master().isCurrentThread());
- {
- vespalib::Gate gate;
- // Ensure that all replayed operations are committed to memory structures
- _feedView.get()->forceCommit(CommitParam(_feedHandler->getSerialNum()), std::make_shared<GateCallback>(gate));
- gate.await();
- }
+ // Ensure that all replayed operations are committed to memory structures
+ _feedView.get()->forceCommitAndWait(CommitParam(_feedHandler->getSerialNum()));
(void) _state.enterOnlineState();
// Consider delayed pruning of transaction log and config history
@@ -469,11 +472,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum
}
{
bool elidedConfigSave = equalReplayConfig && tlsReplayDone;
- vespalib::Gate gate;
- // Flush changes to attributes and memory index, cf. visibilityDelay
- _feedView.get()->forceCommit(CommitParam(elidedConfigSave ? serialNum : serialNum - 1),
- std::make_shared<vespalib::KeepAlive<std::pair<FeedHandler::CommitResult, std::shared_ptr<IDestructorCallback>>>>(std::make_pair(std::move(commit_result), std::make_shared<GateCallback>(gate))));
- gate.await();
+ forceCommitAndWait(_feedView.get(), elidedConfigSave ? serialNum : serialNum - 1, std::move(commit_result));
}
if (params.shouldMaintenanceControllerChange()) {
_maintenanceController.killJobs();
@@ -517,30 +516,6 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum
}
}
-
-void
-DocumentDB::performDropFeedView(IFeedView::SP feedView)
-{
- // Delays when feed view is dropped.
- assert(_writeService.master().isCurrentThread());
- _writeService.attributeFieldWriter().sync_all();
- _writeService.summary().sync();
-
- // Feed view is kept alive in the closure's shared ptr.
- _writeService.index().execute(makeLambdaTask([this, feedView] () { performDropFeedView2(feedView); }));
-}
-
-
-void
-DocumentDB::performDropFeedView2(IFeedView::SP feedView) {
- // Delays when feed view is dropped.
- assert(_writeService.index().isCurrentThread());
- _writeService.indexFieldInverter().sync_all();
- _writeService.indexFieldWriter().sync_all();
- masterExecute([feedView]() { (void) feedView; });
-}
-
-
void
DocumentDB::tearDownReferences()
{
@@ -567,8 +542,10 @@ DocumentDB::close()
}
// Abort any ongoing maintenance
stopMaintenance();
- _writeService.master().sync(); // Complete all tasks that didn't observe shutdown
- masterExecute([this]() { tearDownReferences(); });
+ masterExecute([this]() {
+ _feedView.get()->forceCommitAndWait(search::CommitParam(getCurrentSerialNumber()));
+ tearDownReferences();
+ });
_writeService.master().sync();
// Wait until inflight feed operations to this document db has left.
// Caller should have removed document DB from feed router.
@@ -580,8 +557,11 @@ DocumentDB::close()
DocumentDBTaggedMetrics &metrics = getMetrics();
_metricsWireService.cleanAttributes(metrics.ready.attributes);
_metricsWireService.cleanAttributes(metrics.notReady.attributes);
- _writeService.sync_all_executors();
- masterExecute([this] () { closeSubDBs(); } );
+
+ masterExecute([this] () {
+ _feedView.get()->forceCommitAndWait(search::CommitParam(getCurrentSerialNumber()));
+ closeSubDBs();
+ });
_writeService.sync_all_executors();
// What about queued tasks ?
_writeService.shutdown();
@@ -912,20 +892,14 @@ void
DocumentDB::syncFeedView()
{
assert(_writeService.master().isCurrentThread());
- IFeedView::SP oldFeedView(_feedView.get());
IFeedView::SP newFeedView(_subDBs.getFeedView());
_maintenanceController.killJobs();
- _writeService.sync_all_executors();
_feedView.set(newFeedView);
_feedHandler->setActiveFeedView(newFeedView.get());
_subDBs.createRetrievers();
_subDBs.maintenanceSync(_maintenanceController);
-
- // Ensure that old feed view is referenced until all index executor tasks
- // depending on it has completed.
- performDropFeedView(oldFeedView);
}
bool
@@ -990,7 +964,6 @@ void
DocumentDB::stopMaintenance()
{
_maintenanceController.stop();
- _writeService.sync_all_executors();
}
void
@@ -1023,7 +996,7 @@ DocumentDB::notifyClusterStateChanged(const std::shared_ptr<IBucketStateCalculat
if (cfv != nullptr)
cfv->setCalculator(newCalc);
}
- _subDBs.setBucketStateCalculator(newCalc);
+ _subDBs.setBucketStateCalculator(newCalc, std::shared_ptr<vespalib::IDestructorCallback>());
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 6b855cd40a8..391c11df276 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -160,18 +160,6 @@ private:
void enterApplyLiveConfigState();
/**
- * Drop old field view in a controlled manner. The feed view will
- * be kept alive until the index executor is done with all current
- * tasks.
- *
- * Called by executor thread.
- *
- * @param feedView shared pointer to feed view to be dropped.
- */
- void performDropFeedView(IFeedView::SP feedView);
- void performDropFeedView2(IFeedView::SP feedView);
-
- /**
* Implements IFeedHandlerOwner
*/
void onTransactionLogReplayDone() override __attribute__((noinline));
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 4a62a43709f..6576e4ead97 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -20,10 +20,6 @@ using vespalib::makeLambdaTask;
namespace proton {
-DocumentSubDBCollection::Config::Config(size_t numSearchThreads)
- : _numSearchThreads(numSearchThreads)
-{ }
-
DocumentSubDBCollection::DocumentSubDBCollection(
IDocumentSubDBOwner &owner,
search::transactionlog::SyncProxy &tlSyncer,
@@ -38,7 +34,6 @@ DocumentSubDBCollection::DocumentSubDBCollection(
const vespalib::Clock &clock,
std::mutex &configMutex,
const vespalib::string &baseDir,
- const Config & cfg,
const HwInfo &hwInfo)
: _subDBs(),
_owner(owner),
@@ -58,30 +53,22 @@ DocumentSubDBCollection::DocumentSubDBCollection(
StoreOnlyDocSubDB::Context context(owner, tlSyncer, getSerialNum, fileHeaderContext, writeService,
_bucketDB, *_bucketDBHandler, metrics, configMutex, hwInfo);
_subDBs.push_back
- (new SearchableDocSubDB(
- SearchableDocSubDB::Config(
- FastAccessDocSubDB::Config(
- StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir,
- _readySubDbId, SubDbType::READY),
- true, true, false),
- cfg.getNumSearchThreads()),
- SearchableDocSubDB::Context(
- FastAccessDocSubDB::Context(context, metrics.ready.attributes, metricsWireService),
- queryLimiter, clock, warmupExecutor)));
+ (new SearchableDocSubDB(FastAccessDocSubDB::Config(
+ StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir,_readySubDbId, SubDbType::READY),
+ true, true, false),
+ SearchableDocSubDB::Context(
+ FastAccessDocSubDB::Context(context, metrics.ready.attributes, metricsWireService),
+ queryLimiter, clock, warmupExecutor)));
_subDBs.push_back
- (new StoreOnlyDocSubDB(
- StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir,
- _remSubDbId, SubDbType::REMOVED),
- context));
+ (new StoreOnlyDocSubDB(StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, _remSubDbId, SubDbType::REMOVED),
+ context));
_subDBs.push_back
- (new FastAccessDocSubDB(
- FastAccessDocSubDB::Config(
- StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir,
- _notReadySubDbId, SubDbType::NOTREADY),
- true, true, true),
- FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService)));
+ (new FastAccessDocSubDB(FastAccessDocSubDB::Config(
+ StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir,_notReadySubDbId, SubDbType::NOTREADY),
+ true, true, true),
+ FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService)));
}
@@ -316,11 +303,11 @@ DocumentSubDBCollection::close()
}
void
-DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc)
+DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc, OnDone onDone)
{
_calc = calc;
for (auto subDb : _subDBs) {
- subDb->setBucketStateCalculator(calc);
+ subDb->setBucketStateCalculator(calc, onDone);
}
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index 880fb57840b..515a886969c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -7,6 +7,7 @@
#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
#include <vespa/searchlib/common/serialnum.h>
#include <vespa/vespalib/util/varholder.h>
+#include <vespa/vespalib/util/idestructorcallback.h>
#include <mutex>
namespace vespalib {
@@ -58,13 +59,7 @@ public:
using SubDBVector = std::vector<IDocumentSubDB *>;
using const_iterator = SubDBVector::const_iterator;
using SerialNum = search::SerialNum;
- class Config {
- public:
- Config(size_t numSearchThreads);
- size_t getNumSearchThreads() const noexcept { return _numSearchThreads; }
- private:
- const size_t _numSearchThreads;
- };
+ using OnDone = std::shared_ptr<vespalib::IDestructorCallback>;
private:
using IFeedViewSP = std::shared_ptr<IFeedView>;
@@ -99,11 +94,10 @@ public:
const vespalib::Clock &clock,
std::mutex &configMutex,
const vespalib::string &baseDir,
- const Config & cfg,
const HwInfo &hwInfo);
~DocumentSubDBCollection();
- void setBucketStateCalculator(const IBucketStateCalculatorSP &calc);
+ void setBucketStateCalculator(const IBucketStateCalculatorSP &calc, OnDone onDone);
void createRetrievers();
void maintenanceSync(MaintenanceController &mc);
diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
index 7e0a1851bf5..dcd29c9ddcb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
@@ -48,6 +48,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha
: _sharedExecutor(sharedExecutor),
_masterExecutor(1, stackSize, master_executor),
+ _shared_field_writer(cfg.shared_field_writer()),
_master_task_limit(cfg.master_task_limit()),
_indexExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), index_executor)),
_summaryExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), summary_executor)),
@@ -62,7 +63,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha
_index_field_writer_ptr(),
_attribute_field_writer_ptr()
{
- if (cfg.shared_field_writer() == SharedFieldWriterExecutor::INDEX) {
+ if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) {
_field_writer = SequencedTaskExecutor::create(field_writer_executor, cfg.indexingThreads() * 2, cfg.defaultTaskLimit());
_attributeFieldWriter = SequencedTaskExecutor::create(attribute_field_writer_executor, cfg.indexingThreads(), cfg.defaultTaskLimit(),
cfg.optimize(), cfg.kindOfwatermark(), cfg.reactionTime());
@@ -70,7 +71,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha
_index_field_writer_ptr = _field_writer.get();
_attribute_field_writer_ptr = _attributeFieldWriter.get();
- } else if (cfg.shared_field_writer() == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) {
+ } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) {
_field_writer = SequencedTaskExecutor::create(field_writer_executor, cfg.indexingThreads() * 3, cfg.defaultTaskLimit(),
cfg.optimize(), cfg.kindOfwatermark(), cfg.reactionTime());
_index_field_inverter_ptr = _field_writer.get();
@@ -154,13 +155,28 @@ ExecutorThreadingService::set_task_limits(uint32_t master_task_limit,
ExecutorThreadingServiceStats
ExecutorThreadingService::getStats()
{
- return ExecutorThreadingServiceStats(_masterExecutor.getStats(),
- _indexExecutor->getStats(),
- _summaryExecutor->getStats(),
- _sharedExecutor.getStats(),
- _index_field_inverter_ptr->getStats(),
- _index_field_writer_ptr->getStats(),
- _attribute_field_writer_ptr->getStats());
+ auto master_stats = _masterExecutor.getStats();
+ auto index_stats = _indexExecutor->getStats();
+ auto summary_stats = _summaryExecutor->getStats();
+ auto shared_stats = _sharedExecutor.getStats();
+ if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) {
+ auto field_writer_stats = _field_writer->getStats();
+ return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats,
+ field_writer_stats,
+ field_writer_stats,
+ _attribute_field_writer_ptr->getStats());
+ } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) {
+ auto field_writer_stats = _field_writer->getStats();
+ return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats,
+ field_writer_stats,
+ field_writer_stats,
+ field_writer_stats);
+ } else {
+ return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats,
+ _index_field_inverter_ptr->getStats(),
+ _index_field_writer_ptr->getStats(),
+ _attribute_field_writer_ptr->getStats());
+ }
}
vespalib::ISequencedTaskExecutor &
diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
index 1890ca300e2..629c5043ed7 100644
--- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
+++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
@@ -2,13 +2,13 @@
#pragma once
#include "executor_thread_service.h"
+#include "threading_service_config.h"
#include <vespa/searchcorespi/index/ithreadingservice.h>
#include <vespa/vespalib/util/threadstackexecutor.h>
namespace proton {
class ExecutorThreadingServiceStats;
-class ThreadingServiceConfig;
/**
* Implementation of IThreadingService using 2 underlying thread stack executors
@@ -19,6 +19,7 @@ class ExecutorThreadingService : public searchcorespi::index::IThreadingService
private:
vespalib::ThreadExecutor & _sharedExecutor;
vespalib::ThreadStackExecutor _masterExecutor;
+ ThreadingServiceConfig::SharedFieldWriterExecutor _shared_field_writer;
std::atomic<uint32_t> _master_task_limit;
std::unique_ptr<vespalib::SyncableThreadExecutor> _indexExecutor;
std::unique_ptr<vespalib::SyncableThreadExecutor> _summaryExecutor;
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index 40a1a1a45f3..f8a6253ead1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -276,7 +276,10 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const
}
_iFeedView.set(_fastAccessFeedView.get());
if (isNodeRetired()) {
- reconfigureAttributesConsideringNodeState();
+ // TODO Should probably ahve a similar OnDone callback to applyConfig too.
+ vespalib::Gate gate;
+ reconfigureAttributesConsideringNodeState(std::make_shared<vespalib::GateCallback>(gate));
+ gate.await();
}
}
return tasks;
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
index db2bb7ed2cb..e2b3887c60c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -6,7 +6,6 @@
#include "removedonecontext.h"
#include "putdonecontext.h"
#include <vespa/searchcore/proton/feedoperation/operations.h>
-#include <vespa/vespalib/util/isequencedtaskexecutor.h>
using document::Document;
using document::DocumentUpdate;
@@ -73,7 +72,7 @@ void
FastAccessFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
{
// Drain pending PutDoneContext and ForceCommitContext objects
- _writeService.sync_all_executors();
+ forceCommitAndWait(search::CommitParam(op.getSerialNum()));
_docIdLimit.set(op.getLidLimit());
getAttributeWriter()->compactLidSpace(op.getLidLimit(), op.getSerialNum());
Parent::handleCompactLidSpace(op);
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index ea63d59c830..bb03f48882f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -303,7 +303,7 @@ void
FeedHandler::performEof()
{
assert(_writeService.master().isCurrentThread());
- _writeService.sync_all_executors();
+ _activeFeedView->forceCommitAndWait(CommitParam(_serialNum));
LOG(debug, "Visiting done for transaction log domain '%s', eof received", _tlsMgr.getDomainName().c_str());
// Replay must be complete
if (_replay_end_serial_num != _serialNum) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
index 00849d6ad31..f84352a4558 100644
--- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
@@ -6,6 +6,7 @@
#include <vespa/searchlib/common/serialnum.h>
#include <vespa/searchlib/util/searchable_stats.h>
#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/idestructorcallback.h>
namespace search::index { class Schema; }
@@ -60,6 +61,7 @@ public:
using SchemaSP = std::shared_ptr<Schema>;
using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>;
using IndexConfig = index::IndexConfig;
+ using OnDone = std::shared_ptr<vespalib::IDestructorCallback>;
public:
IDocumentSubDB() { }
virtual ~IDocumentSubDB() { }
@@ -77,7 +79,7 @@ public:
virtual IReprocessingTask::List
applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot,
SerialNum serialNum, const ReconfigParams &params, IDocumentDBReferenceResolver &resolver) = 0;
- virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) = 0;
+ virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone) = 0;
virtual std::shared_ptr<ISearchHandler> getSearchView() const = 0;
virtual std::shared_ptr<IFeedView> getFeedView() const = 0;
diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp
new file mode 100644
index 00000000000..2e07c839d76
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.cpp
@@ -0,0 +1,16 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "ifeedview.h"
+#include <vespa/vespalib/util/destructor_callbacks.h>
+#include <vespa/vespalib/util/gate.h>
+
+namespace proton {
+
+void
+IFeedView::forceCommitAndWait(CommitParam param) {
+ vespalib::Gate gate;
+ forceCommit(param, std::make_shared<vespalib::GateCallback>(gate));
+ gate.await();
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
index 4feb7d013ea..14e6d6811e7 100644
--- a/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/ifeedview.h
@@ -62,6 +62,7 @@ public:
virtual void forceCommit(const CommitParam & param, DoneCallback onDone) = 0;
void forceCommit(CommitParam param) { forceCommit(param, DoneCallback()); }
void forceCommit(search::SerialNum serialNum) { forceCommit(CommitParam(serialNum)); }
+ void forceCommitAndWait(CommitParam param);
virtual void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation & pruneOp) = 0;
virtual void handleCompactLidSpace(const CompactLidSpaceOperation &op) = 0;
};
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
index c4826bba8ea..6b606298026 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
@@ -31,9 +31,9 @@ public:
};
bool
-isRunningOrRunnable(const MaintenanceJobRunner & job, const Executor * master) {
+isRunnable(const MaintenanceJobRunner & job, const Executor * master) {
return (&job.getExecutor() == master)
- ? job.isRunning()
+ ? false
: job.isRunnable();
}
@@ -99,14 +99,14 @@ MaintenanceController::killJobs()
job->stop(); // Make sure no more tasks are added to the executor
}
for (auto &job : _jobs) {
- while (isRunningOrRunnable(*job, &_masterThread)) {
+ while (isRunnable(*job, &_masterThread)) {
std::this_thread::sleep_for(1ms);
}
}
- JobList tmpJobs = _jobs;
+ JobList tmpJobs;
{
Guard guard(_jobsLock);
- _jobs.clear();
+ tmpJobs.swap(_jobs);
}
// Hold jobs until existing tasks have been drained
_masterThread.execute(makeLambdaTask([this, jobs=std::move(tmpJobs)]() {
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp
index 16d74479ebe..9eb0596ff1f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp
@@ -77,13 +77,6 @@ MaintenanceJobRunner::MaintenanceJobRunner(Executor &executor, IMaintenanceJob::
}
bool
-MaintenanceJobRunner::isRunning() const
-{
- Guard guard(_lock);
- return _running;
-}
-
-bool
MaintenanceJobRunner::isRunnable() const
{
Guard guard(_lock);
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h
index 151cc302cd3..17f244e6621 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.h
@@ -31,7 +31,6 @@ public:
MaintenanceJobRunner(vespalib::Executor &executor, IMaintenanceJob::UP job);
void run() override;
void stop();
- bool isRunning() const;
bool isRunnable() const;
const vespalib::Executor & getExecutor() const { return _executor; }
const IMaintenanceJob &getJob() const { return *_job; }
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index bf3589457f9..6e87f33e1c6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -29,7 +29,7 @@ using namespace searchcorespi;
namespace proton {
SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx)
- : FastAccessDocSubDB(cfg._fastUpdCfg, ctx._fastUpdCtx),
+ : FastAccessDocSubDB(cfg, ctx._fastUpdCtx),
IIndexManager::Reconfigurer(),
_indexMgr(),
_indexWriter(),
@@ -180,9 +180,9 @@ SearchableDocSubDB::propagateFlushConfig()
}
void
-SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc)
+SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone onDone)
{
- FastAccessDocSubDB::setBucketStateCalculator(calc);
+ FastAccessDocSubDB::setBucketStateCalculator(calc, std::move(onDone));
propagateFlushConfig();
}
@@ -244,7 +244,7 @@ SearchableDocSubDB::reconfigure(std::unique_ptr<Configure> configure)
{
assert(_writeService.master().isCurrentThread());
- _writeService.sync_all_executors();
+ getFeedView()->forceCommitAndWait(search::CommitParam(_getSerialNum.getSerialNum()));
// Everything should be quiet now.
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index 2e7aac0a8d3..ea758443cd4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -6,6 +6,7 @@
#include "searchable_feed_view.h"
#include "searchview.h"
#include "summaryadapter.h"
+#include "igetserialnum.h"
#include <vespa/eval/eval/value_cache/constant_tensor_loader.h>
#include <vespa/eval/eval/value_cache/constant_value_cache.h>
#include <vespa/searchcore/proton/attribute/attributemanager.h>
@@ -39,15 +40,6 @@ SearchableDocSubDB : public FastAccessDocSubDB,
{
public:
- struct Config {
- const FastAccessDocSubDB::Config _fastUpdCfg;
- const size_t _numSearcherThreads;
-
- Config(const FastAccessDocSubDB::Config &fastUpdCfg, size_t numSearcherThreads)
- : _fastUpdCfg(fastUpdCfg),
- _numSearcherThreads(numSearcherThreads)
- { }
- };
struct Context {
const FastAccessDocSubDB::Context _fastUpdCtx;
@@ -113,7 +105,7 @@ public:
IReprocessingTask::List
applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot,
SerialNum serialNum, const ReconfigParams &params, IDocumentDBReferenceResolver &resolver) override;
- void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override;
+ void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone onDone) override;
void clearViews() override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index 97e55c37aff..06d174497b3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -460,7 +460,7 @@ StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config,
}
void
-StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc) {
+StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc, OnDone onDone) {
bool wasNodeRetired = isNodeRetired();
_nodeRetired = calc->nodeRetired();
if (wasNodeRetired != isNodeRetired()) {
@@ -468,16 +468,16 @@ StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCa
auto cfg = _dms->getConfig();
cfg.setCompactionStrategy(compactionStrategy);
_dms->update_config(cfg);
- reconfigureAttributesConsideringNodeState();
+ reconfigureAttributesConsideringNodeState(std::move(onDone));
}
}
void
-StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState() {
+StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState(OnDone onDone) {
search::CompactionStrategy compactionStrategy = computeCompactionStrategy(_lastConfiguredCompactionStrategy);
auto attrMan = getAttributeManager();
if (attrMan) {
- attrMan->asyncForEachAttribute(std::make_shared<UpdateConfig>(compactionStrategy));
+ attrMan->asyncForEachAttribute(std::make_shared<UpdateConfig>(compactionStrategy), std::move(onDone));
}
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index 7051722f605..b53dfe89f59 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -146,10 +146,10 @@ protected:
vespalib::VarHolder<IFeedView::SP> _iFeedView;
std::mutex &_configMutex;
HwInfo _hwInfo;
+ const IGetSerialNum &_getSerialNum;
private:
- const IGetSerialNum &_getSerialNum;
- TlsSyncer _tlsSyncer;
- DocumentMetaStoreFlushTarget::SP _dmsFlushTarget;
+ TlsSyncer _tlsSyncer;
+ DocumentMetaStoreFlushTarget::SP _dmsFlushTarget;
std::shared_ptr<ShrinkLidSpaceFlushTarget> _dmsShrinkTarget;
std::shared_ptr<PendingLidTrackerBase> _pendingLidsForCommit;
bool _nodeRetired;
@@ -183,7 +183,7 @@ protected:
StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams();
vespalib::string getSubDbName() const;
void reconfigure(const search::LogDocumentStore::Config & protonConfig, const AllocStrategy& alloc_strategy);
- void reconfigureAttributesConsideringNodeState();
+ void reconfigureAttributesConsideringNodeState(OnDone onDone);
public:
StoreOnlyDocSubDB(const Config &cfg, const Context &ctx);
~StoreOnlyDocSubDB() override;
@@ -203,7 +203,7 @@ public:
IReprocessingTask::List
applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot,
SerialNum serialNum, const ReconfigParams &params, IDocumentDBReferenceResolver &resolver) override;
- void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override;
+ void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc, OnDone onDone) override;
ISearchHandler::SP getSearchView() const override { return _iSearchView.get(); }
IFeedView::SP getFeedView() const override { return _iFeedView.get(); }
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 5a7afcc584d..527eaff07cc 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -417,7 +417,6 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp)
bool updateOk = _metaStore.updateMetaData(updOp.getLid(), updOp.getBucketId(), updOp.getTimestamp());
assert(updateOk);
(void) updateOk;
- _metaStore.commit(CommitParam(serialNum));
}
auto onWriteDone = createUpdateDoneContext(std::move(token), get_pending_lid_token(updOp), updOp.getUpdate());
@@ -605,7 +604,6 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId &
gate.await();
removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED);
}
- _metaStore.commit(CommitParam(serialNum));
}
}
@@ -621,9 +619,6 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo
const SerialNum serialNum = op.getSerialNum();
const LidVectorContext::SP &ctx = op.getLidsToRemove(_params._subDbId);
if (!ctx) {
- if (useDocumentMetaStore(serialNum)) {
- _metaStore.commit(CommitParam(serialNum));
- }
return 0;
}
const LidVector &lidsToRemove(ctx->getLidVector());
@@ -634,7 +629,6 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo
_gidToLidChangeHandler.notifyRemoves(std::make_shared<vespalib::GateCallback>(gate), gidsToRemove, serialNum);
gate.await();
_metaStore.removeBatch(lidsToRemove, ctx->getDocIdLimit());
- _metaStore.commit(CommitParam(serialNum));
_lidReuseDelayer.delayReuse(lidsToRemove);
}
std::shared_ptr<vespalib::IDestructorCallback> onWriteDone;
diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
index ff75a59c41b..7982e8a8414 100644
--- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
@@ -33,13 +33,6 @@ uint32_t
calculateIndexingThreads(const ProtonConfig::Indexing & indexing, double concurrency, const HwInfo::Cpu &cpuInfo)
{
double scaledCores = cpuInfo.cores() * concurrency;
- if (indexing.optimize != ProtonConfig::Indexing::Optimize::ADAPTIVE) {
- // We are capping at 12 threads to reduce cost of waking up threads
- // to achieve a better throughput.
- // TODO: Fix this in a simpler/better way.
- scaledCores = std::min(12.0, scaledCores);
- }
-
uint32_t indexingThreads = std::max((int32_t)std::ceil(scaledCores / 3), indexing.threads);
return std::max(indexingThreads, 1u);
}
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
index 5cc0ac5a186..03ddcf3605b 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -58,7 +58,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
{
return IReprocessingTask::List();
}
- void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) override { }
+ void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &, OnDone) override { }
ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); }
IFeedView::SP getFeedView() const override { return IFeedView::SP(); }
void clearViews() override {}
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
index abc8eb679dd..743cd9af8fc 100644
--- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
@@ -87,7 +87,7 @@ public:
return _writables;
}
void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor>) const override { }
- void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>) const override { }
+ void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>, OnDone) const override { }
ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &) const override {
return ExclusiveAttributeReadAccessor::UP();
diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h
index 6b08eecf61f..0c6af0a4d25 100644
--- a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h
+++ b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h
@@ -7,14 +7,35 @@
namespace proton::test {
/**
+ * Run the given function in the master thread and wait until all threads done.
+ */
+template <typename FunctionType>
+void
+runInMasterAndSyncAll(searchcorespi::index::IThreadingService &writeService, FunctionType func)
+{
+ writeService.master().execute(vespalib::makeLambdaTask(std::move(func)));
+ writeService.sync_all_executors();
+}
+
+/**
* Run the given function in the master thread and wait until done.
*/
template <typename FunctionType>
void
+runInMasterAndSync(searchcorespi::index::IThreadingService &writeService, FunctionType func)
+{
+ writeService.master().execute(vespalib::makeLambdaTask(std::move(func)));
+ writeService.master().sync();
+}
+
+/**
+ * Run the given function in the master thread.
+ */
+template <typename FunctionType>
+void
runInMaster(searchcorespi::index::IThreadingService &writeService, FunctionType func)
{
writeService.master().execute(vespalib::makeLambdaTask(std::move(func)));
- writeService.sync_all_executors();
}
}
diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp
index bafcd545e33..c077ab83a6e 100644
--- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp
+++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp
@@ -200,9 +200,9 @@ struct ReferenceAttributeTest : public ::testing::Test {
void notifyReferencedRemove(const GlobalId &gid) {
_attr->notifyReferencedRemove(gid);
}
- void setGidToLidMapperFactory(std::shared_ptr<MyGidToLidMapperFactory> factory) {
+ void setGidToLidMapperFactory(std::shared_ptr<MyGidToLidMapperFactory> factory, const std::vector<GlobalId>& removes) {
_attr->setGidToLidMapperFactory(factory);
- _attr->populateTargetLids();
+ _attr->populateTargetLids(removes);
}
uint32_t getUniqueGids() {
return getStatus().getNumUniqueValues();
@@ -257,7 +257,7 @@ TEST_F(ReferenceAttributeTest, reference_for_a_document_can_be_cleared)
TEST_F(ReferenceAttributeTest, lid_beyond_range_is_mapped_to_zero)
{
auto factory = std::make_shared<MyGidToLidMapperFactory>();
- setGidToLidMapperFactory(factory);
+ setGidToLidMapperFactory(factory, {});
ensureDocIdLimit(5);
_attr->addDocs(1);
set(5, toGid(doc2));
@@ -318,7 +318,7 @@ TEST_F(ReferenceAttributeTest, update_uses_gid_mapper_to_set_target_lid)
{
ensureDocIdLimit(6);
auto factory = std::make_shared<MyGidToLidMapperFactory>();
- setGidToLidMapperFactory(factory);
+ setGidToLidMapperFactory(factory, {});
set(1, toGid(doc1));
set(2, toGid(doc2));
set(4, toGid(doc1));
@@ -371,7 +371,7 @@ void
checkPopulateTargetLids(ReferenceAttributeTest &f)
{
auto factory = std::make_shared<MyGidToLidMapperFactory>();
- f.setGidToLidMapperFactory(factory);
+ f.setGidToLidMapperFactory(factory, {});
f.assertTargetLid(1, 10);
f.assertTargetLid(2, 17);
f.assertTargetLid(3, 10);
@@ -401,6 +401,22 @@ TEST_F(ReferenceAttributeTest, populateTargetLids_uses_gid_mapper_to_update_lid_
EXPECT_TRUE(vespalib::unlink("test.udat"));
}
+TEST_F(ReferenceAttributeTest, populateTargetLids_handles_removes)
+{
+ preparePopulateTargetLids(*this);
+ auto factory = std::make_shared<MyGidToLidMapperFactory>();
+ setGidToLidMapperFactory(factory, { toGid(doc1) });
+ assertTargetLid(1, 0);
+ assertTargetLid(2, 17);
+ assertTargetLid(3, 0);
+ assertTargetLid(4, 0);
+ assertNoTargetLid(5);
+ assertLids(0, { });
+ assertLids(10, { });
+ assertLids(17, { 2 });
+ assertLids(18, { });
+}
+
TEST_F(ReferenceAttributeTest, notifyReferencedPut_and_notifyReferencedRemove_changes_reverse_mapping)
{
preparePopulateTargetLids(*this);
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
index 342c1ea568b..eb822313d61 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
@@ -366,8 +366,8 @@ ReferenceAttribute::notifyReferencedPut(const GlobalId &gid, DocId targetLid)
commit();
}
-void
-ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid)
+bool
+ReferenceAttribute::notifyReferencedRemoveNoCommit(const GlobalId &gid)
{
EntryRef ref = _store.find(gid);
if (ref.valid()) {
@@ -377,6 +377,15 @@ ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid)
if (oldTargetLid != 0) {
_store.remove(ref);
}
+ return true;
+ }
+ return false;
+}
+
+void
+ReferenceAttribute::notifyReferencedRemove(const GlobalId &gid)
+{
+ if (notifyReferencedRemoveNoCommit(gid)) {
commit();
}
}
@@ -401,7 +410,7 @@ public:
}
void
-ReferenceAttribute::populateTargetLids()
+ReferenceAttribute::populateTargetLids(const std::vector<GlobalId>& removes)
{
if (_gidToLidMapperFactory) {
std::unique_ptr<IGidToLidMapper> mapperUP = _gidToLidMapperFactory->getMapper();
@@ -409,6 +418,9 @@ ReferenceAttribute::populateTargetLids()
TargetLidPopulator populator(*this);
mapper.foreach(populator);
}
+ for (auto& remove : removes) {
+ notifyReferencedRemoveNoCommit(remove);
+ }
commit();
}
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
index b411d6cb923..4016230ef89 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
@@ -84,8 +84,9 @@ public:
void notifyReferencedPutNoCommit(const GlobalId &gid, DocId targetLid);
void notifyReferencedPut(const GlobalId &gid, DocId targetLid);
+ bool notifyReferencedRemoveNoCommit(const GlobalId &gid);
void notifyReferencedRemove(const GlobalId &gid);
- void populateTargetLids();
+ void populateTargetLids(const std::vector<GlobalId>& removes);
void clearDocs(DocId lidLow, DocId lidLimit) override;
void onShrinkLidSpace() override;
diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
index ee9a97a4685..52a363c9888 100644
--- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
+++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
@@ -8,6 +8,7 @@
#include <vespa/searchsummary/docsummary/positionsdfw.h>
#include <vespa/searchsummary/docsummary/idocsumenvironment.h>
#include <vespa/searchsummary/docsummary/docsumstate.h>
+#include <vespa/searchsummary/test/slime_value.h>
#include <vespa/searchlib/util/rawbuf.h>
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/vespalib/data/slime/slime.h>
@@ -113,7 +114,7 @@ struct MyGetDocsumsStateCallback : GetDocsumsStateCallback {
template <typename AttrType>
void checkWritePositionField(Test &test, AttrType &attr,
- uint32_t doc_id, const string &expected) {
+ uint32_t doc_id, const vespalib::string &expect_json) {
for (AttributeVector::DocId i = 0; i < doc_id + 1; ) {
attr.addDoc(i);
if (i == 007) {
@@ -133,7 +134,7 @@ void checkWritePositionField(Test &test, AttrType &attr,
PositionsDFW::UP writer =
createPositionsDFW(attr.getName().c_str(), &attribute_man);
ASSERT_TRUE(writer.get());
- ResType res_type = RES_LONG_STRING;
+ ResType res_type = RES_JSONSTRING;
MyGetDocsumsStateCallback callback;
GetDocsumsState state(callback);
state._attributes.push_back(&attr);
@@ -142,19 +143,17 @@ void checkWritePositionField(Test &test, AttrType &attr,
vespalib::slime::SlimeInserter inserter(target);
writer->insertField(doc_id, &state, res_type, inserter);
- vespalib::Memory got = target.get().asString();
- test.EXPECT_EQUAL(expected.size(), got.size);
- test.EXPECT_EQUAL(expected, string(got.data, got.size));
+ test::SlimeValue expected(expect_json);
+ test.EXPECT_EQUAL(expected.slime, target);
}
void Test::requireThat2DPositionFieldIsWritten() {
SingleInt64ExtAttribute attr("foo");
- checkWritePositionField(*this, attr, 0x3e, "<position x=\"6\" y=\"7\" latlong=\"N0.000007;E0.000006\" />");
- checkWritePositionField(*this, attr, 007, "<position x=\"-1\" y=\"-1\" latlong=\"S0.000001;W0.000001\" />");
- checkWritePositionField(*this, attr, 0x42, "<position x=\"0\" y=\"-1\" latlong=\"S0.000001;E0.000000\" />");
- checkWritePositionField(*this, attr, 0x17, "<position x=\"-16711935\" y=\"16711935\" latlong=\"N16.711935;W16.711935\" />");
- checkWritePositionField(*this, attr, 42, "");
-
+ checkWritePositionField(*this, attr, 0x3e, "{x:6,y:7,latlong:'N0.000007;E0.000006'}");
+ checkWritePositionField(*this, attr, 007, "{x:-1,y:-1,latlong:'S0.000001;W0.000001'}");
+ checkWritePositionField(*this, attr, 0x42, "{x:0,y:-1,latlong:'S0.000001;E0.000000'}");
+ checkWritePositionField(*this, attr, 0x17, "{x:-16711935,y:16711935,latlong:'N16.711935;W16.711935'}");
+ checkWritePositionField(*this, attr, 42, "null");
}
} // namespace
diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp
index 0a421882784..5855dd0e8fd 100644
--- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp
+++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp
@@ -65,8 +65,6 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback {
strlen("long_string")));
EXPECT_TRUE(packer->AddLongData( "long_data",
strlen("long_data")));
- EXPECT_TRUE(packer->AddLongString( "xml_string",
- strlen("xml_string")));
FieldBlock jsf1("{foo:1, bar:2}");
EXPECT_TRUE(packer->AddLongData(jsf1.data(), jsf1.dataLen()));
EXPECT_TRUE(packer->AddLongString("abc", 3));
@@ -98,7 +96,6 @@ DocsumFixture::DocsumFixture()
EXPECT_TRUE(cfg->AddConfigEntry("data_field", RES_DATA));
EXPECT_TRUE(cfg->AddConfigEntry("longstring_field", RES_LONG_STRING));
EXPECT_TRUE(cfg->AddConfigEntry("longdata_field", RES_LONG_DATA));
- EXPECT_TRUE(cfg->AddConfigEntry("xmlstring_field", RES_XMLSTRING));
EXPECT_TRUE(cfg->AddConfigEntry("jsonstring_field", RES_JSONSTRING));
EXPECT_TRUE(cfg->AddConfigEntry("bad_jsonstring_field", RES_JSONSTRING));
config->CreateEnumMaps();
@@ -121,7 +118,6 @@ TEST_FF("require that docsum can be written as slime", DocsumFixture(), Slime())
EXPECT_EQUAL(f2.get()["data_field"].asData().make_string(), std::string("data"));
EXPECT_EQUAL(f2.get()["longstring_field"].asString().make_string(), std::string("long_string"));
EXPECT_EQUAL(f2.get()["longdata_field"].asData().make_string(), std::string("long_data"));
- EXPECT_EQUAL(f2.get()["xmlstring_field"].asString().make_string(), std::string("xml_string"));
EXPECT_EQUAL(f2.get()["jsonstring_field"]["foo"].asLong(), 1u);
EXPECT_EQUAL(f2.get()["jsonstring_field"]["bar"].asLong(), 2u);
EXPECT_EQUAL(f2.get()["bad_jsonstring_field"].type().getId(), 0u);
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
index 6dd952f87ef..448feedac80 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
@@ -118,7 +118,6 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type
}
break;
case RES_JSONSTRING:
- case RES_XMLSTRING:
case RES_FEATUREDATA:
case RES_LONG_STRING:
case RES_STRING: {
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
index afc1ec4b3f2..fada441f718 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
@@ -121,7 +121,6 @@ CopyDFW::insertField(uint32_t /*docid*/, GeneralResult *gres, GetDocsumsState *s
target.insertLong(valint64);
break; }
- case RES_XMLSTRING:
case RES_JSONSTRING:
case RES_FEATUREDATA:
case RES_LONG_STRING:
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
index 766ebdc51f4..666c5098ca4 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
@@ -120,7 +120,6 @@ static void convertEntry(GetDocsumsState *state,
case RES_STRING:
case RES_LONG_STRING:
case RES_FEATUREDATA:
- case RES_XMLSTRING:
entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace);
if (len != 0) {
inserter.insertString(Memory(ptr, len));
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp
index 3c01c23c965..ceb358e5859 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp
@@ -255,7 +255,6 @@ GeneralResult::unpack(const char *buf, const size_t buflen)
break;
}
- case RES_XMLSTRING:
case RES_JSONSTRING:
case RES_FEATUREDATA:
case RES_LONG_STRING: {
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
index 43a3dc2163b..ce14069a135 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
@@ -94,15 +94,10 @@ AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type
vespalib::string value = vespalib::stringify(absdist);
vespalib::Memory data(value.c_str(), value.size());
- if (type == RES_STRING ||
- type == RES_LONG_STRING ||
- type == RES_XMLSTRING)
- {
+ if (type == RES_STRING || type == RES_LONG_STRING) {
target.insertString(data);
}
- if (type == RES_LONG_DATA ||
- type == RES_DATA)
- {
+ if (type == RES_LONG_DATA || type == RES_DATA) {
target.insertData(data);
}
}
@@ -173,63 +168,21 @@ insertFromAttr(const attribute::IAttributeVector &attribute, uint32_t docid, ves
}
}
-vespalib::asciistream
-formatField(const attribute::IAttributeVector &attribute, uint32_t docid, ResType type) {
- vespalib::asciistream target;
- int32_t docx = 0;
- int32_t docy = 0;
-
- IntegerContent pos;
- pos.fill(attribute, docid);
- uint32_t numValues = pos.size();
- LOG(debug, "docid=%d, numValues=%d", docid, numValues);
-
- bool isShort = !IDocsumFieldWriter::IsBinaryCompatible(type, RES_LONG_STRING);
- for (uint32_t i = 0; i < numValues; i++) {
- int64_t docxy(pos[i]);
- vespalib::geo::ZCurve::decode(docxy, &docx, &docy);
- if (docx == 0 && docy == INT_MIN) {
- LOG(spam, "skipping empty zcurve value");
- continue;
- }
- double degrees_ns = docy;
- degrees_ns /= 1000000.0;
- double degrees_ew = docx;
- degrees_ew /= 1000000.0;
-
- target << "<position x=\"" << docx << "\" y=\"" << docy << "\"";
- target << " latlong=\"";
- target << vespalib::FloatSpec::fixed;
- if (degrees_ns < 0) {
- target << "S" << (-degrees_ns);
- } else {
- target << "N" << degrees_ns;
- }
- target << ";";
- if (degrees_ew < 0) {
- target << "W" << (-degrees_ew);
- } else {
- target << "E" << degrees_ew;
- }
- target << "\" />";
- if (isShort && target.size() > 30000) {
- target << "<overflow />";
- break;
- }
- }
- return target;
-}
+void checkExpected(ResType type) {
+ static bool alreadyWarned = false;
+ if (type == RES_JSONSTRING) return;
+ if (alreadyWarned) return;
+ alreadyWarned = true;
+ LOG(error, "Unexpected summary field type %s", ResultConfig::GetResTypeName(type));
}
+} // namespace
+
void
PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target)
{
- if (type == RES_XMLSTRING) {
- insertFromAttr(get_attribute(*dsState), docid, target);
- return;
- }
- vespalib::asciistream val(formatField(get_attribute(*dsState), docid, type));
- target.insertString(vespalib::Memory(val.c_str(), val.size()));
+ checkExpected(type);
+ insertFromAttr(get_attribute(*dsState), docid, target);
}
//--------------------------------------------------------------------------
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h
index 97435d793be..858fdea2404 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h
@@ -27,7 +27,6 @@ enum ResType {
RES_DATA,
RES_LONG_STRING,
RES_LONG_DATA,
- RES_XMLSTRING,
RES_JSONSTRING,
RES_TENSOR,
RES_FEATUREDATA
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
index 09604a7316c..d3c0caeec48 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
@@ -53,7 +53,6 @@ ResultConfig::GetResTypeName(ResType type)
case RES_DATA: return "data";
case RES_LONG_STRING: return "longstring";
case RES_LONG_DATA: return "longdata";
- case RES_XMLSTRING: return "xmlstring";
case RES_JSONSTRING: return "jsonstring";
case RES_TENSOR: return "tensor";
case RES_FEATUREDATA: return "featuredata";
@@ -126,6 +125,8 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const
Reset();
int maxclassID = 0x7fffffff; // avoid negative classids
_defaultSummaryId = cfg.defaultsummaryid;
+ _useV8geoPositions = cfg.usev8geopositions;
+
for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) {
const auto& cfg_class = cfg.classes[i];
if (cfg_class.name.empty()) {
@@ -173,7 +174,7 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const
} else if (strcmp(fieldtype, "longdata") == 0) {
rc = resClass->AddConfigEntry(fieldname, RES_LONG_DATA);
} else if (strcmp(fieldtype, "xmlstring") == 0) {
- rc = resClass->AddConfigEntry(fieldname, RES_XMLSTRING);
+ rc = resClass->AddConfigEntry(fieldname, RES_JSONSTRING);
} else if (strcmp(fieldtype, "jsonstring") == 0) {
rc = resClass->AddConfigEntry(fieldname, RES_JSONSTRING);
} else if (strcmp(fieldtype, "tensor") == 0) {
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h
index 4cc37512929..8a8bfabaaec 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h
@@ -33,6 +33,7 @@ private:
typedef vespalib::hash_map<vespalib::string, uint32_t> NameMap;
typedef vespalib::hash_map<uint32_t, ResultClass::UP> IdMap;
uint32_t _defaultSummaryId;
+ bool _useV8geoPositions;
search::util::StringEnum _fieldEnum;
IdMap _classLookup;
NameMap _nameLookup; // name -> class id
@@ -41,6 +42,7 @@ private:
void Init();
public:
+ bool useV8geoPositions() const { return _useV8geoPositions; }
class iterator {
public:
iterator(IdMap::iterator it) : _it(it) { }
@@ -123,11 +125,10 @@ public:
return (b == RES_STRING || b == RES_DATA);
case RES_LONG_STRING:
case RES_LONG_DATA:
- case RES_XMLSTRING:
case RES_FEATUREDATA:
case RES_JSONSTRING:
return (b == RES_LONG_STRING || b == RES_LONG_DATA ||
- b == RES_XMLSTRING || b == RES_FEATUREDATA || b == RES_JSONSTRING);
+ b == RES_FEATUREDATA || b == RES_JSONSTRING);
default:
return false;
}
@@ -159,9 +160,8 @@ public:
return b == RES_INT64;
case RES_STRING:
case RES_LONG_STRING:
- case RES_XMLSTRING:
case RES_JSONSTRING:
- return (b == RES_STRING || b == RES_LONG_STRING || b == RES_XMLSTRING || b == RES_JSONSTRING);
+ return (b == RES_STRING || b == RES_LONG_STRING || b == RES_JSONSTRING);
case RES_DATA:
case RES_LONG_DATA:
return (b == RES_DATA || b == RES_LONG_DATA);
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp
index 6c2c4f9c4ea..4ab06cbd41a 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.cpp
@@ -114,7 +114,6 @@ ResultPacker::AddEmpty()
case RES_INT64: return AddInt64(search::attribute::getUndefined<int64_t>());
case RES_STRING: return AddString(nullptr, 0);
case RES_DATA: return AddData(nullptr, 0);
- case RES_XMLSTRING:
case RES_JSONSTRING:
case RES_FEATUREDATA:
case RES_LONG_STRING: return AddLongString(nullptr, 0);
diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h
index 3226c16aba3..4b489d41f7d 100644
--- a/storage/src/tests/distributor/distributor_stripe_test_util.h
+++ b/storage/src/tests/distributor/distributor_stripe_test_util.h
@@ -26,7 +26,7 @@ class DocumentSelectionParser;
class ExternalOperationHandler;
class IdealStateManager;
class IdealStateMetricSet;
-class NodeSupportedFeatures;
+struct NodeSupportedFeatures;
class Operation;
class StripeBucketDBUpdater;
diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp
index 02b43a32df3..1c47170de6c 100644
--- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp
+++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp
@@ -106,12 +106,14 @@ PersistenceProviderWrapper::putAsync(const spi::Bucket& bucket, spi::Timestamp t
}
void
-PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, spi::Timestamp timestamp, const spi::DocumentId& id,
+PersistenceProviderWrapper::removeAsync(const spi::Bucket& bucket, std::vector<TimeStampAndDocumentId> ids,
spi::Context& context, spi::OperationComplete::UP onComplete)
{
- LOG_SPI("remove(" << bucket << ", " << timestamp << ", " << id << ")");
+ for (const TimeStampAndDocumentId & stampedId : ids) {
+ LOG_SPI("remove(" << bucket << ", " << stampedId.first << ", " << stampedId.second << ")");
+ }
CHECK_ERROR_ASYNC(spi::RemoveResult, FAIL_REMOVE, onComplete);
- _spi.removeAsync(bucket, timestamp, id, context, std::move(onComplete));
+ _spi.removeAsync(bucket, std::move(ids), context, std::move(onComplete));
}
void
diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h
index cfc7002a643..1552a955221 100644
--- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h
+++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h
@@ -100,7 +100,7 @@ public:
spi::BucketIdListResult listBuckets(BucketSpace bucketSpace) const override;
spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override;
void putAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentSP, spi::Context&, spi::OperationComplete::UP) override;
- void removeAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&, spi::OperationComplete::UP) override;
+ void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId> ids, spi::Context&, spi::OperationComplete::UP) override;
void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const spi::DocumentId&, spi::Context&, spi::OperationComplete::UP) override;
void updateAsync(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&, spi::OperationComplete::UP) override;
spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override;
diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp
index f3560bfa2cb..c163f6de024 100644
--- a/storage/src/tests/persistence/persistencetestutils.cpp
+++ b/storage/src/tests/persistence/persistencetestutils.cpp
@@ -130,19 +130,6 @@ PersistenceTestUtils::setupExecutor(uint32_t numThreads) {
_sequenceTaskExecutor = vespalib::SequencedTaskExecutor::create(test_executor, numThreads, 1000, vespalib::Executor::OptimizeFor::ADAPTIVE);
}
-document::Document::SP
-PersistenceTestUtils::schedulePut(
- uint32_t location,
- spi::Timestamp timestamp,
- uint32_t minSize,
- uint32_t maxSize)
-{
- document::Document::SP doc(createRandomDocumentAtLocation(location, timestamp, minSize, maxSize));
- auto msg = std::make_shared<api::PutCommand>(makeDocumentBucket(document::BucketId(16, location)), doc, timestamp);
- fsHandler().schedule(msg);
- return doc;
-}
-
StorBucketDatabase::WrappedEntry
PersistenceTestUtils::getBucket(const document::BucketId& id)
{
diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h
index dd9f5f15123..fc986c3c6f2 100644
--- a/storage/src/tests/persistence/persistencetestutils.h
+++ b/storage/src/tests/persistence/persistencetestutils.h
@@ -108,14 +108,10 @@ public:
PersistenceTestUtils();
~PersistenceTestUtils() override;
- document::Document::SP schedulePut(uint32_t location, spi::Timestamp timestamp, uint32_t minSize = 0, uint32_t maxSize = 128);
-
- void setupDisks();
void setupExecutor(uint32_t numThreads);
void TearDown() override {
if (_sequenceTaskExecutor) {
- _sequenceTaskExecutor->sync_all();
_sequenceTaskExecutor.reset();
}
_env.reset();
diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp
index 0c862f11b05..51fe4ef4bd3 100644
--- a/storage/src/tests/persistence/processalltest.cpp
+++ b/storage/src/tests/persistence/processalltest.cpp
@@ -41,11 +41,14 @@ TEST_F(ProcessAllHandlerTest, remove_location) {
AsyncHandler handler(getEnv(), getPersistenceProvider(), _bucketOwnershipNotifier, *_sequenceTaskExecutor, _bucketIdFactory);
auto tracker = handler.handleRemoveLocation(*cmd, createTracker(cmd, bucket));
+ std::shared_ptr<api::StorageMessage> msg;
+ ASSERT_TRUE(_replySender.queue.getNext(msg, 60s));
+
EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n"
"DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n",
dumpBucket(bucketId));
- auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(std::move(*tracker).stealReplySP());
+ auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(msg);
ASSERT_TRUE(reply);
EXPECT_EQ(2u, reply->documents_removed());
}
@@ -65,6 +68,9 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
auto cmd = std::make_shared<api::RemoveLocationCommand>("testdoctype1.headerval % 2 == 0", bucket);
auto tracker = handler.handleRemoveLocation(*cmd, createTracker(cmd, bucket));
+ std::shared_ptr<api::StorageMessage> msg;
+ ASSERT_TRUE(_replySender.queue.getNext(msg, 60s));
+
EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n"
"DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n"
"DocEntry(102, 1, id:mail:testdoctype1:n=4:62608.html)\n"
@@ -77,7 +83,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
"DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n",
dumpBucket(bucketId));
- auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(std::move(*tracker).stealReplySP());
+ auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(msg);
ASSERT_TRUE(reply);
EXPECT_EQ(5u, reply->documents_removed());
}
diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp
index aac2b0748c4..b5161673af3 100644
--- a/storage/src/vespa/storage/persistence/asynchandler.cpp
+++ b/storage/src/vespa/storage/persistence/asynchandler.cpp
@@ -413,20 +413,14 @@ AsyncHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, MessageTrack
std::make_shared<document::DocIdOnly>(),
processor, spi::NEWEST_DOCUMENT_ONLY,tracker->context());
- std::vector<std::future<std::unique_ptr<spi::Result>>> results;
- results.reserve(to_remove.size());
- for (auto & entry : to_remove) {
- auto catcher = std::make_unique<spi::CatchResult>();
- results.push_back(catcher->future_result());
- _spi.removeAsync(bucket, entry.first, entry.second, tracker->context(), std::move(catcher));
- }
- for (auto & future : results) {
- auto result = future.get();
- if (result->getErrorCode() != spi::Result::ErrorType::NONE) {
- throw std::runtime_error(fmt("Failed to do remove for removelocation: %s", result->getErrorMessage().c_str()));
- }
- }
- tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, to_remove.size()));
+ auto task = makeResultTask([&cmd, tracker = std::move(tracker), removed = to_remove.size()](spi::Result::UP response) {
+ tracker->checkForError(*response);
+ tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, removed));
+ tracker->sendReply();
+ });
+
+ _spi.removeAsync(bucket, std::move(to_remove), tracker->context(),
+ std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task)));
return tracker;
}
diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp
index 7d7c8b79e0c..254a26aa454 100644
--- a/storage/src/vespa/storage/persistence/mergehandler.cpp
+++ b/storage/src/vespa/storage/persistence/mergehandler.cpp
@@ -506,9 +506,10 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results
auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), std::move(docId), "put", _clock, _env._metrics.merge_handler_metrics.put_latency);
_spi.putAsync(bucket, timestamp, std::move(doc), context, std::move(complete));
} else {
- DocumentId docId(e._docName);
- auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), docId, "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency);
- _spi.removeAsync(bucket, timestamp, docId, context, std::move(complete));
+ std::vector<spi::PersistenceProvider::TimeStampAndDocumentId> ids;
+ ids.emplace_back(timestamp, e._docName);
+ auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].second, "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency);
+ _spi.removeAsync(bucket, std::move(ids), context, std::move(complete));
}
}
diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp
index 9ccd901744b..752c1175f22 100644
--- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp
+++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp
@@ -154,11 +154,11 @@ ProviderErrorWrapper::putAsync(const spi::Bucket &bucket, spi::Timestamp ts, spi
}
void
-ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, spi::Timestamp ts, const document::DocumentId &docId,
+ProviderErrorWrapper::removeAsync(const spi::Bucket &bucket, std::vector<TimeStampAndDocumentId> ids,
spi::Context & context, spi::OperationComplete::UP onComplete)
{
onComplete->addResultHandler(this);
- _impl.removeAsync(bucket, ts, docId, context, std::move(onComplete));
+ _impl.removeAsync(bucket, std::move(ids), context, std::move(onComplete));
}
void
diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h
index 14d20cf8a52..7285c405d5c 100644
--- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h
+++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h
@@ -58,7 +58,7 @@ public:
void register_error_listener(std::shared_ptr<ProviderErrorListener> listener);
void putAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentSP, spi::Context &, spi::OperationComplete::UP) override;
- void removeAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&, spi::OperationComplete::UP) override;
+ void removeAsync(const spi::Bucket&, std::vector<TimeStampAndDocumentId>, spi::Context&, spi::OperationComplete::UP) override;
void removeIfFoundAsync(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&, spi::OperationComplete::UP) override;
void updateAsync(const spi::Bucket &, spi::Timestamp, spi::DocumentUpdateSP, spi::Context &, spi::OperationComplete::UP) override;
void setActiveStateAsync(const spi::Bucket& b, spi::BucketInfo::ActiveState newState, spi::OperationComplete::UP onComplete) override;
diff --git a/vespa-feed-client/abi-spec.json b/vespa-feed-client/abi-spec.json
index 808fe152fee..7f78e81b447 100644
--- a/vespa-feed-client/abi-spec.json
+++ b/vespa-feed-client/abi-spec.json
@@ -195,6 +195,7 @@
"public"
],
"methods": [
+ "public void <init>(java.time.Duration)",
"public void <init>(java.time.Duration, java.time.Duration)",
"public void success()",
"public void failure(ai.vespa.feed.client.HttpResponse)",
diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java
index cdf55f0ba7e..3b79d47b494 100644
--- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java
+++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java
@@ -37,7 +37,7 @@ public class FeedClientBuilder {
int connectionsPerEndpoint = 4;
int maxStreamsPerConnection = 4096;
FeedClient.RetryStrategy retryStrategy = defaultRetryStrategy;
- FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Duration.ofSeconds(10), Duration.ofMinutes(10));
+ FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Duration.ofSeconds(10));
Path certificateFile;
Path privateKeyFile;
Path caCertificatesFile;
diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java
index b878840d70f..cb5e35c79a5 100644
--- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java
+++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java
@@ -31,20 +31,32 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker {
private final long graceMillis;
private final long doomMillis;
+ /**
+ * Creates a new circuit breaker with the given grace periods.
+ * @param grace the period of consecutive failures before state changes to half-open.
+ */
+ public GracePeriodCircuitBreaker(Duration grace) {
+ this(System::currentTimeMillis, grace, null);
+ }
+
+ /**
+ * Creates a new circuit breaker with the given grace periods.
+ * @param grace the period of consecutive failures before state changes to half-open.
+ * @param doom the period of consecutive failures before shutting down.
+ */
public GracePeriodCircuitBreaker(Duration grace, Duration doom) {
this(System::currentTimeMillis, grace, doom);
+ if (doom.isNegative())
+ throw new IllegalArgumentException("Doom delay must be non-negative");
}
GracePeriodCircuitBreaker(LongSupplier clock, Duration grace, Duration doom) {
if (grace.isNegative())
throw new IllegalArgumentException("Grace delay must be non-negative");
- if (doom.isNegative())
- throw new IllegalArgumentException("Doom delay must be non-negative");
-
this.clock = requireNonNull(clock);
this.graceMillis = grace.toMillis();
- this.doomMillis = doom.toMillis();
+ this.doomMillis = doom == null ? -1 : doom.toMillis();
}
@Override
@@ -74,11 +86,11 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker {
long failingMillis = clock.getAsLong() - failingSinceMillis.get();
if (failingMillis > graceMillis && halfOpen.compareAndSet(false, true))
log.log(INFO, "Circuit breaker is now half-open, as no requests have succeeded for the " +
- "last " + failingMillis + "ms. The server will be pinged to see if it recovers, " +
- "but this client will give up if no successes are observed within " + doomMillis + "ms. " +
- "First failure was '" + detail.get() + "'.");
+ "last " + failingMillis + "ms. The server will be pinged to see if it recovers" +
+ (doomMillis >= 0 ? ", but this client will give up if no successes are observed within " + doomMillis + "ms" : "") +
+ ". First failure was '" + detail.get() + "'.");
- if (failingMillis > doomMillis && open.compareAndSet(false, true))
+ if (doomMillis >= 0 && failingMillis > doomMillis && open.compareAndSet(false, true))
log.log(WARNING, "Circuit breaker is now open, after " + doomMillis + "ms of failing request, " +
"and this client will give up and abort its remaining feed operations.");
diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java
index 11b844f2c69..0f840201ca8 100644
--- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java
+++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java
@@ -208,16 +208,16 @@ class HttpRequestStrategyTest {
// Enqueue some operations to the same id, which are serialised, and then shut down while operations are in flight.
Phaser phaser = new Phaser(2);
Phaser blocker = new Phaser(2);
- AtomicReference<CompletableFuture<HttpResponse>> completion = new AtomicReference<>();
cluster.expect((req, vessel) -> {
if (req == blocking) {
- phaser.arriveAndAwaitAdvance(); // Synchronise with tst main thread, and then ...
+ phaser.arriveAndAwaitAdvance(); // Synchronise with test main thread, and then ...
blocker.arriveAndAwaitAdvance(); // ... block dispatch thread, so we get something in the queue.
throw new RuntimeException("armageddon"); // Dispatch thread should die, tearing down everything.
}
else if (req == failing) {
phaser.arriveAndAwaitAdvance(); // Let test thread enqueue more ops before failing (and retrying) this.
vessel.completeExceptionally(new IOException("failed"));
+ phaser.arriveAndAwaitAdvance(); // Ensure a retry is scheduled before test thread is allowed to continue.
}
else phaser.arriveAndAwaitAdvance(); // Don't complete from mock cluster, but require destruction to do this.
});
@@ -228,7 +228,8 @@ class HttpRequestStrategyTest {
CompletableFuture<HttpResponse> blocked = strategy.enqueue(id3, blocking);
CompletableFuture<HttpResponse> delayed = strategy.enqueue(id4, request);
phaser.arriveAndAwaitAdvance(); // inflight completes dispatch, but causes no response.
- phaser.arriveAndAwaitAdvance(); // failed completes dispatch, and a retry is enqueued.
+ phaser.arriveAndAwaitAdvance(); // failed is allowed to dispatch ...
+ phaser.arriveAndAwaitAdvance(); // ... and a retry is enqueued.
phaser.arriveAndAwaitAdvance(); // blocked starts dispatch, and hangs, blocking dispatch thread.
// Current state: inflight is "inflight to cluster", serialised1/2 are waiting completion of it;
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java
index c877daeeb2a..98b6aee1e33 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java
@@ -67,10 +67,10 @@ class EndpointResultQueue {
InflightOperation operation = inflightOperations.remove(result.getOperationId());
if (operation == null) {
if (duplicateGivesWarning) {
- log.warning("Result for ID '" + result.getOperationId() + "' received from '" + endpoint +
- "', but we have no record of a sent operation. Either something is wrong on the server side " +
- "(bad VIP usage?), or we have somehow received duplicate results, " +
- "or operation was received _after_ client-side timeout.");
+ log.info("Result for ID '" + result.getOperationId() + "' received from '" + endpoint +
+ "', but we have no record of a sent operation. This may happen if an operation is " +
+ "initiated, but also retried, due to HTTP failure. Otherwise, something is wrong on " +
+ "the server side (bad VIP usage?), or operation was received _after_ client-side timeout.");
}
return;
}
diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
index 831ec24ac5f..1f9216192fc 100644
--- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
+++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java
@@ -76,7 +76,8 @@ public class VespaCliTestRunner implements TestRunner {
void runTests(Suite suite, byte[] config) {
Process process = null;
try {
- process = testRunProcessBuilder(suite, toEndpointsConfig(config)).start();
+ TestConfig testConfig = TestConfig.fromJson(config);
+ process = testRunProcessBuilder(suite, testConfig).start();
BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
in.lines().forEach(line -> {
if (line.length() > 1 << 13)
@@ -95,14 +96,17 @@ public class VespaCliTestRunner implements TestRunner {
}
}
- ProcessBuilder testRunProcessBuilder(Suite suite, String endpointsConfig) {
+ ProcessBuilder testRunProcessBuilder(Suite suite, TestConfig config) throws IOException {
Path suitePath = getChildDirectory(artifactsPath, "tests")
.flatMap(testsPath -> getChildDirectory(testsPath, toSuiteDirectoryName(suite)))
.orElseThrow(() -> new IllegalStateException("No tests found, for suite '" + suite + "'"));
- ProcessBuilder builder = new ProcessBuilder("vespa", "test", "--endpoints", endpointsConfig);
+ ProcessBuilder builder = new ProcessBuilder("vespa", "test", suitePath.toAbsolutePath().toString(),
+ "--application", config.application().toFullString(),
+ "--endpoints", toEndpointsConfig(config),
+ "--data-plane-public-cert", artifactsPath.resolve("cert").toAbsolutePath().toString(),
+ "--data-plane-private-key", artifactsPath.resolve("key").toAbsolutePath().toString());
builder.redirectErrorStream(true);
- builder.directory(suitePath.toFile());
return builder;
}
@@ -133,8 +137,7 @@ public class VespaCliTestRunner implements TestRunner {
}
}
- static String toEndpointsConfig(byte[] testConfig) throws IOException {
- TestConfig config = TestConfig.fromJson(testConfig);
+ static String toEndpointsConfig(TestConfig config) throws IOException {
Cursor root = new Slime().setObject();
Cursor endpointsArray = root.setArray("endpoints");
config.deployments().get(config.zone()).forEach((cluster, url) -> {
diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java
index 68d44a386f8..be00f28d2c9 100644
--- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java
+++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java
@@ -1,6 +1,7 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.testrunner;
+import ai.vespa.hosted.api.TestConfig;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -19,34 +20,28 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
*/
class VespaCliTestRunnerTest {
- @Test
- void testEndpointsConfig() throws IOException {
- byte[] testConfig = ("{\n" +
- " \"application\": \"t:a:i\",\n" +
- " \"zone\": \"dev.aws-us-east-1c\",\n" +
- " \"system\": \"publiccd\",\n" +
- " \"isCI\": true,\n" +
- " \"zoneEndpoints\": {\n" +
- " \"dev.aws-us-east-1c\": {\n" +
- " \"default\": \"https://dev.endpoint:443/\"\n" +
- " },\n" +
- " \"prod.aws-us-east-1a\": {\n" +
- " \"default\": \"https://prod.endpoint:443/\"\n" +
- " }\n" +
- " },\n" +
- " \"clusters\": {\n" +
- " \"prod.aws-us-east-1c\": [\n" +
- " \"documents\"\n" +
- " ]\n" +
- " }\n" +
- "}\n").getBytes(StandardCharsets.UTF_8);
-
- assertEquals("{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}",
- VespaCliTestRunner.toEndpointsConfig(testConfig));
- }
+ static final TestConfig testConfig = TestConfig.fromJson(("{\n" +
+ " \"application\": \"t:a:i\",\n" +
+ " \"zone\": \"dev.aws-us-east-1c\",\n" +
+ " \"system\": \"publiccd\",\n" +
+ " \"isCI\": true,\n" +
+ " \"zoneEndpoints\": {\n" +
+ " \"dev.aws-us-east-1c\": {\n" +
+ " \"default\": \"https://dev.endpoint:443/\"\n" +
+ " },\n" +
+ " \"prod.aws-us-east-1a\": {\n" +
+ " \"default\": \"https://prod.endpoint:443/\"\n" +
+ " }\n" +
+ " },\n" +
+ " \"clusters\": {\n" +
+ " \"prod.aws-us-east-1c\": [\n" +
+ " \"documents\"\n" +
+ " ]\n" +
+ " }\n" +
+ "}\n").getBytes(StandardCharsets.UTF_8));
@Test
- void testSuitePathDiscovery() throws IOException {
+ void testSetup() throws IOException {
Path temp = Files.createTempDirectory("vespa-cli-test-runner-test-");
temp.toFile().deleteOnExit();
VespaCliTestRunner runner = new VespaCliTestRunner(temp);
@@ -54,14 +49,18 @@ class VespaCliTestRunnerTest {
Path tests = Files.createDirectory(temp.resolve("tests"));
assertTrue(runner.isSupported());
- IllegalStateException expected = assertThrows(IllegalStateException.class,
- () -> runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, ""));
- assertEquals("No tests found, for suite 'SYSTEM_TEST'", expected.getMessage());
+ IllegalStateException ise = assertThrows(IllegalStateException.class,
+ () -> runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, testConfig));
+ assertEquals("No tests found, for suite 'SYSTEM_TEST'", ise.getMessage());
Path systemTests = Files.createDirectory(tests.resolve("system-test"));
- ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, "config");
- assertEquals(systemTests.toFile(), builder.directory());
- assertEquals(List.of("vespa", "test", "--endpoints", "config"), builder.command());
+ ProcessBuilder builder = runner.testRunProcessBuilder(TestRunner.Suite.SYSTEM_TEST, testConfig);
+ assertEquals(List.of("vespa", "test", systemTests.toAbsolutePath().toString(),
+ "--application", "t.a.i",
+ "--endpoints", "{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}",
+ "--data-plane-public-cert", temp.resolve("cert").toAbsolutePath().toString(),
+ "--data-plane-private-key", temp.resolve("key").toAbsolutePath().toString()),
+ builder.command());
}
}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java
index 11fd97f2a1d..3faafd7b2e5 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java
@@ -1,21 +1,20 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.http.server;
-import java.util.Set;
-import java.util.logging.Logger;
-
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
import com.yahoo.documentapi.metrics.DocumentApiMetrics;
import com.yahoo.documentapi.metrics.DocumentOperationStatus;
import com.yahoo.documentapi.metrics.DocumentOperationType;
import com.yahoo.jdisc.Metric;
-import java.util.logging.Level;
import com.yahoo.messagebus.Reply;
import com.yahoo.messagebus.ReplyHandler;
import com.yahoo.messagebus.Trace;
import com.yahoo.vespa.http.client.core.ErrorCode;
import com.yahoo.vespa.http.client.core.OperationStatus;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
/**
* Catch message bus replies and make the available to a given session.
*
@@ -42,20 +41,26 @@ public class FeedReplyReader implements ReplyHandler {
final double latencyInSeconds = (System.currentTimeMillis() - context.creationTime) / 1000.0d;
metric.set(MetricNames.LATENCY, latencyInSeconds, null);
- if (reply.hasErrors()) {
- Set<Integer> errorCodes = reply.getErrorCodes();
- metricsHelper.reportFailure(DocumentOperationType.fromMessage(reply.getMessage()),
- DocumentOperationStatus.fromMessageBusErrorCodes(errorCodes));
+ DocumentOperationType type = DocumentOperationType.fromMessage(reply.getMessage());
+ boolean conditionMet = conditionMet(reply);
+ if (reply.hasErrors() && conditionMet) {
+ DocumentOperationStatus status = DocumentOperationStatus.fromMessageBusErrorCodes(reply.getErrorCodes());
+ metricsHelper.reportFailure(type, status);
metric.add(MetricNames.FAILED, 1, null);
- enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR,
- reply.getError(0).getCode() == DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED, reply.getTrace());
+ enqueue(context, reply.getError(0).getMessage(), ErrorCode.ERROR, false, reply.getTrace());
} else {
- metricsHelper.reportSuccessful(DocumentOperationType.fromMessage(reply.getMessage()), latencyInSeconds);
+ metricsHelper.reportSuccessful(type, latencyInSeconds);
metric.add(MetricNames.SUCCEEDED, 1, null);
- enqueue(context, "Document processed.", ErrorCode.OK, false, reply.getTrace());
+ if (!conditionMet)
+ metric.add(MetricNames.TEST_AND_SET_CONDITION_NOT_MET, 1, null);
+ enqueue(context, "Document processed.", ErrorCode.OK, !conditionMet, reply.getTrace());
}
}
+ private static boolean conditionMet(Reply reply) {
+ return !reply.hasErrors() || reply.getError(0).getCode() != DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED;
+ }
+
private void enqueue(ReplyContext context, String message, ErrorCode status, boolean isConditionNotMet, Trace trace) {
try {
String traceMessage = (trace != null && trace.getLevel() > 0) ? trace.toString() : "";
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
index 8cd628a83d9..4b49e3594f8 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java
@@ -18,6 +18,7 @@ public final class MetricNames {
public static final String OPERATIONS_PER_SEC = PREFIX + "ops_per_sec";
public static final String LATENCY = PREFIX + "latency";
public static final String FAILED = PREFIX + "failed";
+ public static final String TEST_AND_SET_CONDITION_NOT_MET = PREFIX + "test_and_set_condition_not_met";
public static final String PARSE_ERROR = PREFIX + "parse_error";
public static final String SUCCEEDED = PREFIX + "succeeded";
public static final String PENDING = PREFIX + "pending";
diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp
index 2e2728781f4..964978e5510 100644
--- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp
+++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp
@@ -705,6 +705,58 @@ TEST(DataStoreTest, control_static_sizes) {
EXPECT_EQ(0, bs.size());
}
+namespace {
+
+void test_free_element_to_held_buffer(bool direct, bool before_hold_buffer)
+{
+ MyStore s;
+ auto ref = s.addEntry(1);
+ EXPECT_EQ(0u, MyRef(ref).bufferId());
+ s.switch_primary_buffer();
+ EXPECT_EQ(1u, s.primary_buffer_id());
+
+ if (before_hold_buffer) {
+ if (direct) {
+ s.freeElem(ref, 1);
+ } else {
+ s.holdElem(ref, 1);
+ }
+ }
+ s.holdBuffer(0); // hold last buffer
+ if (!before_hold_buffer) {
+ if (direct) {
+ ASSERT_DEATH({ s.freeElem(ref, 1); }, "state.isOnHold\\(\\) && was_held");
+ } else {
+ ASSERT_DEATH({ s.holdElem(ref, 1); }, "state.isActive\\(\\)");
+ }
+ }
+ s.transferHoldLists(100);
+ s.trimHoldLists(101);
+}
+
+}
+
+TEST(DataStoreTest, free_to_active_then_held_buffer_is_ok)
+{
+ test_free_element_to_held_buffer(true, true);
+}
+
+TEST(DataStoreTest, hold_to_active_then_held_buffer_is_ok)
+{
+ test_free_element_to_held_buffer(false, true);
+}
+
+#ifndef NDEBUG
+TEST(DataStoreDeathTest, free_to_held_buffer_is_not_ok)
+{
+ test_free_element_to_held_buffer(true, false);
+}
+
+TEST(DataStoreDeathTest, hold_to_held_buffer_is_not_ok)
+{
+ test_free_element_to_held_buffer(false, false);
+}
+#endif
}
diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h
index d8bb0fb3d0f..be74c2a60d5 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastore.h
+++ b/vespalib/src/vespa/vespalib/datastore/datastore.h
@@ -27,6 +27,8 @@ template <typename RefT = EntryRefT<22> >
class DataStoreT : public DataStoreBase
{
private:
+ void free_elem_internal(EntryRef ref, size_t numElems, bool was_held);
+
public:
typedef RefT RefType;
@@ -49,7 +51,7 @@ public:
/**
* Free element(s).
*/
- void freeElem(EntryRef ref, size_t numElems);
+ void freeElem(EntryRef ref, size_t numElems) { free_elem_internal(ref, numElems, false); }
/**
* Hold element(s).
diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.hpp b/vespalib/src/vespa/vespalib/datastore/datastore.hpp
index 81027196a75..23dcb9222a1 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastore.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/datastore.hpp
@@ -23,7 +23,7 @@ DataStoreT<RefT>::~DataStoreT() = default;
template <typename RefT>
void
-DataStoreT<RefT>::freeElem(EntryRef ref, size_t numElems)
+DataStoreT<RefT>::free_elem_internal(EntryRef ref, size_t numElems, bool was_held)
{
RefType intRef(ref);
BufferState &state = getBufferState(intRef.bufferId());
@@ -35,9 +35,12 @@ DataStoreT<RefT>::freeElem(EntryRef ref, size_t numElems)
state.freeList().push_back(ref);
}
} else {
- assert(state.isOnHold());
+ assert(state.isOnHold() && was_held);
}
state.incDeadElems(numElems);
+ if (was_held) {
+ state.decHoldElems(numElems);
+ }
state.cleanHold(getBuffer(intRef.bufferId()),
intRef.unscaled_offset() * state.getArraySize(), numElems);
}
@@ -71,10 +74,7 @@ DataStoreT<RefT>::trimElemHoldList(generation_t usedGen)
for (; it != ite; ++it) {
if (static_cast<sgeneration_t>(it->_generation - usedGen) >= 0)
break;
- RefType intRef(it->_ref);
- BufferState &state = getBufferState(intRef.bufferId());
- freeElem(it->_ref, it->_len);
- state.decHoldElems(it->_len);
+ free_elem_internal(it->_ref, it->_len, true);
++freed;
}
if (freed != 0) {
@@ -91,10 +91,7 @@ DataStoreT<RefT>::clearElemHoldList()
ElemHold2List::iterator it(elemHold2List.begin());
ElemHold2List::iterator ite(elemHold2List.end());
for (; it != ite; ++it) {
- RefType intRef(it->_ref);
- BufferState &state = getBufferState(intRef.bufferId());
- freeElem(it->_ref, it->_len);
- state.decHoldElems(it->_len);
+ free_elem_internal(it->_ref, it->_len, true);
}
elemHold2List.clear();
}
diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
index d6dcdee0eb4..a74e41dfb3f 100644
--- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
+++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
@@ -449,7 +449,7 @@ DocsumFilter::getMappedDocsum(uint32_t id)
_packer.Init(resClass->GetClassID());
for (FieldSpecList::iterator it(_fields.begin()), end = _fields.end(); it != end; ++it) {
ResType type = it->getResultType();
- if (type == RES_JSONSTRING || type == RES_XMLSTRING) {
+ if (type == RES_JSONSTRING) {
// this really means 'structured data'
writeSlimeField(*it, doc, _packer);
} else {