aboutsummaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorHÃ¥kon Hallingstad <hakon@verizonmedia.com>2020-04-20 10:36:33 +0200
committerGitHub <noreply@github.com>2020-04-20 10:36:33 +0200
commitb105eead1fbbcefbb85bc962749f2a12fa660bbe (patch)
tree7a3c996c00b854066d32608a002335715fb98c96 /config-model
parentf61f6c701dc91e839b865f158a6da56ff166def7 (diff)
parent9ab0ef70e9ed4f422df67603f26bcb0c7918fdc4 (diff)
Merge branch 'master' into hakonhall/remove-use-bucket-space-metric-feature-flag
Diffstat (limited to 'config-model')
-rw-r--r--config-model/pom.xml4
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java1
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java86
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java21
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java34
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java61
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java6
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java89
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java10
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java40
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java3
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java23
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java12
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java31
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Index.java56
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Search.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java8
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java11
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java17
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java43
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java17
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java27
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java5
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java64
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java9
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java77
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java6
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java59
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java19
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java7
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java16
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java58
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java21
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Host.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java42
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java51
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java110
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java15
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java37
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java39
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java57
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java60
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java239
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/Container.java10
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java54
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java28
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java52
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java69
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java21
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java137
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java37
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java4
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java78
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java17
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java20
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java (renamed from config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java)11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java (renamed from config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java)10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java16
-rw-r--r--config-model/src/main/javacc/SDParser.jj113
-rwxr-xr-xconfig-model/src/main/perl/vespa-deploy2
-rw-r--r--config-model/src/main/resources/schema/admin.rnc19
-rw-r--r--config-model/src/main/resources/schema/common.rnc9
-rw-r--r--config-model/src/main/resources/schema/container.rnc9
-rw-r--r--config-model/src/main/resources/schema/containercluster.rnc6
-rw-r--r--config-model/src/main/resources/schema/content.rnc14
-rw-r--r--config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg3
-rw-r--r--config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json275
-rw-r--r--config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd6
-rw-r--r--config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json275
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg7
-rw-r--r--config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg6
-rw-r--r--config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg104
-rw-r--r--config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg141
-rw-r--r--config-model/src/test/derived/advanced/attributes.cfg4
-rw-r--r--config-model/src/test/derived/array_of_struct_attribute/attributes.cfg8
-rw-r--r--config-model/src/test/derived/attributeprefetch/attributes.cfg72
-rw-r--r--config-model/src/test/derived/attributes/attributes.cfg72
-rw-r--r--config-model/src/test/derived/complex/attributes.cfg36
-rw-r--r--config-model/src/test/derived/hnsw_index/attributes.cfg25
-rw-r--r--config-model/src/test/derived/hnsw_index/ilscripts.cfg5
-rw-r--r--config-model/src/test/derived/hnsw_index/test.sd14
-rw-r--r--config-model/src/test/derived/imported_position_field/attributes.cfg92
-rw-r--r--config-model/src/test/derived/imported_struct_fields/attributes.cfg368
-rw-r--r--config-model/src/test/derived/importedfields/attributes.cfg32
-rw-r--r--config-model/src/test/derived/indexschema/index-info.cfg14
-rw-r--r--config-model/src/test/derived/indexschema/indexschema.sd5
-rw-r--r--config-model/src/test/derived/inheritance/attributes.cfg12
-rw-r--r--config-model/src/test/derived/inheritfromparent/attributes.cfg4
-rw-r--r--config-model/src/test/derived/map_attribute/attributes.cfg12
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/attributes.cfg20
-rw-r--r--config-model/src/test/derived/music/attributes.cfg44
-rw-r--r--config-model/src/test/derived/nearestneighbor/query-profiles/default.xml1
-rw-r--r--config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml3
-rw-r--r--config-model/src/test/derived/nearestneighbor/test.sd27
-rw-r--r--config-model/src/test/derived/neuralnet/query-profiles.cfg54
-rw-r--r--config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml2
-rw-r--r--config-model/src/test/derived/newrank/attributes.cfg40
-rw-r--r--config-model/src/test/derived/predicate_attribute/attributes.cfg4
-rw-r--r--config-model/src/test/derived/predicate_attribute/index-info.cfg2
-rw-r--r--config-model/src/test/derived/prefixexactattribute/attributes.cfg8
-rw-r--r--config-model/src/test/derived/reference_fields/attributes.cfg12
-rw-r--r--config-model/src/test/derived/sorting/attributes.cfg12
-rw-r--r--config-model/src/test/derived/tensor/attributes.cfg20
-rw-r--r--config-model/src/test/derived/types/attributes.cfg52
-rw-r--r--config-model/src/test/examples/simple.sd2
-rw-r--r--config-model/src/test/integration/lightgbm/models/regression.json275
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java44
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java6
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/MockModelContext.java4
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java95
-rw-r--r--config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java15
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java8
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java66
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java (renamed from config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java)2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java (renamed from config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java)2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java3
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java2
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java26
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java40
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java20
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java46
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java45
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java88
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java79
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java35
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java55
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java25
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java8
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java26
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java244
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java199
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java20
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java26
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java151
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java174
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java24
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java101
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java79
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java9
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java12
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java12
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java1
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java165
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java8
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java3
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java30
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java69
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java12
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java54
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java25
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java15
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java26
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java39
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java33
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java3
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java8
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java46
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java3
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java8
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java32
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java (renamed from config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java)12
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java16
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java10
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java7
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java16
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java50
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java20
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java8
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java4
-rw-r--r--config-model/src/test/schema-test-files/services-hosted.xml9
-rw-r--r--config-model/src/test/schema-test-files/services.xml28
308 files changed, 5969 insertions, 2149 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml
index 25b733985f5..33f6657561c 100644
--- a/config-model/pom.xml
+++ b/config-model/pom.xml
@@ -105,6 +105,10 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
index a9677d4b34c..4cd0c1815dd 100644
--- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
@@ -133,6 +133,11 @@ public final class XmlHelper {
return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty());
}
+ public static Optional<Element> getOptionalChild(Element parent, String childName) {
+ return Optional.ofNullable(XML.getChild(parent, childName));
+
+ }
+
public static Optional<String> getOptionalChildValue(Element parent, String childName) {
Element child = XML.getChild(parent, childName);
if (child == null) return Optional.empty();
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
index 140cb3001a0..b9ad830090c 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
@@ -8,7 +8,6 @@ import java.util.Optional;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public interface ConfigDefinitionStore {
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
index 7c9e930bb4f..3fb7ba6bc3a 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -12,10 +12,11 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.model.api.ConfigDefinitionRepo;
import com.yahoo.config.model.api.ContainerEndpoint;
+import com.yahoo.config.model.api.EndpointCertificateSecrets;
import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.model.api.Model;
import com.yahoo.config.model.api.ModelContext;
-import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.api.ValidationParameters;
import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.application.provider.MockFileRegistry;
@@ -36,9 +37,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles;
import com.yahoo.vespa.model.container.search.QueryProfilesBuilder;
import com.yahoo.vespa.model.container.search.SemanticRuleBuilder;
import com.yahoo.vespa.model.container.search.SemanticRules;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
@@ -62,11 +62,12 @@ public class DeployState implements ConfigDefinitionStore {
private final DeployLogger logger;
private final FileRegistry fileRegistry;
private final DocumentModel documentModel;
- private final List<SearchDefinition> searchDefinitions;
+ private final List<NamedSchema> schemas;
private final ApplicationPackage applicationPackage;
private final Optional<ConfigDefinitionRepo> configDefinitionRepo;
private final Optional<ApplicationPackage> permanentApplicationPackage;
private final Optional<Model> previousModel;
+ private final boolean accessLoggingEnabledByDefault;
private final ModelContext.Properties properties;
private final Version vespaVersion;
private final Set<ContainerEndpoint> endpoints;
@@ -76,8 +77,10 @@ public class DeployState implements ConfigDefinitionStore {
private final ImportedMlModels importedModels;
private final ValidationOverrides validationOverrides;
private final Version wantedNodeVespaVersion;
+ private final Optional<String> wantedDockerImageRepo;
private final Instant now;
private final HostProvisioner provisioner;
+ private final Provisioned provisioned;
public static DeployState createTestState() {
return new Builder().build();
@@ -97,18 +100,21 @@ public class DeployState implements ConfigDefinitionStore {
FileRegistry fileRegistry,
DeployLogger deployLogger,
Optional<HostProvisioner> hostProvisioner,
+ Provisioned provisioned,
ModelContext.Properties properties,
Version vespaVersion,
Optional<ApplicationPackage> permanentApplicationPackage,
Optional<ConfigDefinitionRepo> configDefinitionRepo,
- java.util.Optional<Model> previousModel,
+ Optional<Model> previousModel,
Set<ContainerEndpoint> endpoints,
Collection<MlModelImporter> modelImporters,
Zone zone,
QueryProfiles queryProfiles,
SemanticRules semanticRules,
Instant now,
- Version wantedNodeVespaVersion) {
+ Version wantedNodeVespaVersion,
+ boolean accessLoggingEnabledByDefault,
+ Optional<String> wantedDockerImageRepo) {
this.logger = deployLogger;
this.fileRegistry = fileRegistry;
this.rankProfileRegistry = rankProfileRegistry;
@@ -116,8 +122,10 @@ public class DeployState implements ConfigDefinitionStore {
this.properties = properties;
this.vespaVersion = vespaVersion;
this.previousModel = previousModel;
+ this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault;
this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage));
- this.searchDefinitions = searchDocumentModel.getSearchDefinitions();
+ this.provisioned = provisioned;
+ this.schemas = searchDocumentModel.getSchemas();
this.documentModel = searchDocumentModel.getDocumentModel();
this.permanentApplicationPackage = permanentApplicationPackage;
this.configDefinitionRepo = configDefinitionRepo;
@@ -137,6 +145,7 @@ public class DeployState implements ConfigDefinitionStore {
this.wantedNodeVespaVersion = wantedNodeVespaVersion;
this.now = now;
+ this.wantedDockerImageRepo = wantedDockerImageRepo;
}
public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) {
@@ -147,6 +156,8 @@ public class DeployState implements ConfigDefinitionStore {
}
}
+ public Provisioned provisioned() { return provisioned; }
+
/** Get the global rank profile registry for this application. */
public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; }
@@ -157,9 +168,7 @@ public class DeployState implements ConfigDefinitionStore {
public final Optional<ConfigDefinition> getConfigDefinition(ConfigDefinitionKey defKey) {
if (existingConfigDefs == null) {
existingConfigDefs = new LinkedHashMap<>();
- if (configDefinitionRepo.isPresent()) {
- existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get()));
- }
+ configDefinitionRepo.ifPresent(definitionRepo -> existingConfigDefs.putAll(createLazyMapping(definitionRepo)));
existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs());
}
if ( ! existingConfigDefs.containsKey(defKey)) return Optional.empty();
@@ -205,8 +214,8 @@ public class DeployState implements ConfigDefinitionStore {
return applicationPackage;
}
- public List<SearchDefinition> getSearchDefinitions() {
- return searchDefinitions;
+ public List<NamedSchema> getSchemas() {
+ return schemas;
}
public DocumentModel getDocumentModel() {
@@ -217,6 +226,10 @@ public class DeployState implements ConfigDefinitionStore {
return logger;
}
+ public boolean getAccessLoggingEnabledByDefault() {
+ return accessLoggingEnabledByDefault;
+ }
+
public FileRegistry getFileRegistry() {
return fileRegistry;
}
@@ -253,6 +266,8 @@ public class DeployState implements ConfigDefinitionStore {
public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; }
+ public Optional<String> getWantedDockerImageRepo() { return wantedDockerImageRepo; }
+
public Instant now() { return now; }
public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); }
@@ -279,6 +294,7 @@ public class DeployState implements ConfigDefinitionStore {
private FileRegistry fileRegistry = new MockFileRegistry();
private DeployLogger logger = new BaseDeployLogger();
private Optional<HostProvisioner> hostProvisioner = Optional.empty();
+ private Provisioned provisioned = new Provisioned();
private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty();
private ModelContext.Properties properties = new TestProperties();
private Version version = new Version(1, 0, 0);
@@ -289,6 +305,8 @@ public class DeployState implements ConfigDefinitionStore {
private Zone zone = Zone.defaultZone();
private Instant now = Instant.now();
private Version wantedNodeVespaVersion = Vtag.currentVersion;
+ private boolean accessLoggingEnabledByDefault = true;
+ private Optional<String> wantedDockerImageRepo = Optional.empty();
public Builder applicationPackage(ApplicationPackage applicationPackage) {
this.applicationPackage = applicationPackage;
@@ -310,6 +328,11 @@ public class DeployState implements ConfigDefinitionStore {
return this;
}
+ public Builder provisioned(Provisioned provisioned) {
+ this.provisioned = provisioned;
+ return this;
+ }
+
public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) {
this.permanentApplicationPackage = permanentApplicationPackage;
return this;
@@ -360,6 +383,20 @@ public class DeployState implements ConfigDefinitionStore {
return this;
}
+ public Builder wantedDockerImageRepo(Optional<String> dockerImageRepo) {
+ this.wantedDockerImageRepo = dockerImageRepo;
+ return this;
+ }
+
+ /**
+ * Whether access logging is enabled for an application without an accesslog element in services.xml.
+ * True by default.
+ */
+ public Builder accessLoggingEnabledByDefault(boolean accessLoggingEnabledByDefault) {
+ this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault;
+ return this;
+ }
+
public DeployState build() {
return build(new ValidationParameters());
}
@@ -375,6 +412,7 @@ public class DeployState implements ConfigDefinitionStore {
fileRegistry,
logger,
hostProvisioner,
+ provisioned,
properties,
version,
permanentApplicationPackage,
@@ -386,7 +424,9 @@ public class DeployState implements ConfigDefinitionStore {
queryProfiles,
semanticRules,
now,
- wantedNodeVespaVersion);
+ wantedNodeVespaVersion,
+ accessLoggingEnabledByDefault,
+ wantedDockerImageRepo);
}
private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry,
@@ -399,20 +439,18 @@ public class DeployState implements ConfigDefinitionStore {
for (NamedReader reader : readers) {
try {
String readerName = reader.getName();
- String searchName = builder.importReader(reader, readerName, logger);
+ String topLevelName = builder.importReader(reader, readerName, logger);
String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX);
- names.put(searchName, sdName);
- if ( ! sdName.equals(searchName)) {
- throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " +
- "search element ('" + searchName +
+ names.put(topLevelName, sdName);
+ if ( ! sdName.equals(topLevelName)) {
+ throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " +
+ "top level element ('" + topLevelName +
"') are not equal for file '" + readerName + "'");
}
} catch (ParseException e) {
- throw new IllegalArgumentException("Could not parse search definition file '" +
- getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+ throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e);
} catch (IOException e) {
- throw new IllegalArgumentException("Could not read search definition file '" +
- getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+ throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e);
} finally {
closeIgnoreException(reader.getReader());
}
@@ -421,10 +459,6 @@ public class DeployState implements ConfigDefinitionStore {
return SearchDocumentModel.fromBuilderAndNames(builder, names);
}
- private String getSearchDefinitionRelativePath(String name) {
- return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name;
- }
-
private static String stripSuffix(String nodeName, String postfix) {
assert (nodeName.endsWith(postfix));
return nodeName.substring(0, nodeName.length() - postfix.length());
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
index cdd4f6f8e8a..9b9729dddb3 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
@@ -3,7 +3,7 @@ package com.yahoo.config.model.deploy;
import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.vespa.documentmodel.DocumentModel;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
import java.util.ArrayList;
import java.util.List;
@@ -13,16 +13,15 @@ import java.util.Map;
* Internal helper class to retrieve document model and search definitions.
*
* @author Ulf Lilleengen
- * @since 5.1
*/
public class SearchDocumentModel {
private final DocumentModel documentModel;
- private final List<SearchDefinition> searchDefinitions;
+ private final List<NamedSchema> schemas;
- public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) {
+ public SearchDocumentModel(DocumentModel documentModel, List<NamedSchema> schemas) {
this.documentModel = documentModel;
- this.searchDefinitions = searchDefinitions;
+ this.schemas = schemas;
}
@@ -30,22 +29,22 @@ public class SearchDocumentModel {
return documentModel;
}
- public List<SearchDefinition> getSearchDefinitions() {
- return searchDefinitions;
+ public List<NamedSchema> getSchemas() {
+ return schemas;
}
public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) {
- List<SearchDefinition> ret = new ArrayList<>();
+ List<NamedSchema> ret = new ArrayList<>();
for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
- ret.add(new SearchDefinition(names.get(search.getName()), search));
+ ret.add(new NamedSchema(names.get(search.getName()), search));
}
return new SearchDocumentModel(builder.getModel(), ret);
}
public static SearchDocumentModel fromBuilder(SearchBuilder builder) {
- List<SearchDefinition> ret = new ArrayList<>();
+ List<NamedSchema> ret = new ArrayList<>();
for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
- ret.add(new SearchDefinition(search.getName(), search));
+ ret.add(new NamedSchema(search.getName(), search));
}
return new SearchDocumentModel(builder.getModel(), ret);
}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 9f4d1b09f91..99225beba4f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -4,10 +4,11 @@ package com.yahoo.config.model.deploy;
import com.google.common.collect.ImmutableList;
import com.yahoo.config.model.api.ConfigServerSpec;
import com.yahoo.config.model.api.ContainerEndpoint;
-import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.TlsSecrets;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.Zone;
@@ -39,9 +40,11 @@ public class TestProperties implements ModelContext.Properties {
private boolean isFirstTimeDeployment = false;
private boolean useDedicatedNodeForLogserver = false;
private boolean useAdaptiveDispatch = false;
+ private double topKProbability = 1.0;
private double defaultTermwiseLimit = 1.0;
+ private double softStartSeconds = 0.0;
private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty();
-
+ private AthenzDomain athenzDomain;
@Override public boolean multitenant() { return multitenant; }
@Override public ApplicationId applicationId() { return applicationId; }
@@ -60,13 +63,30 @@ public class TestProperties implements ModelContext.Properties {
@Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; }
@Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); }
@Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
+
+ @Override
+ public double defaultSoftStartSeconds() {
+ return softStartSeconds;
+ }
+
+ @Override public double defaultTopKProbability() { return topKProbability; }
@Override public boolean useBucketSpaceMetric() { return true; }
+ @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); }
public TestProperties setDefaultTermwiseLimit(double limit) {
defaultTermwiseLimit = limit;
return this;
}
+ public TestProperties setTopKProbability(double probability) {
+ topKProbability = probability;
+ return this;
+ }
+ public TestProperties setSoftStartSeconds(double softStartSeconds) {
+ this.softStartSeconds = softStartSeconds;
+ return this;
+ }
+
public TestProperties setApplicationId(ApplicationId applicationId) {
this.applicationId = applicationId;
return this;
@@ -102,6 +122,16 @@ public class TestProperties implements ModelContext.Properties {
return this;
}
+ public TestProperties setZone(Zone zone) {
+ this.zone = zone;
+ return this;
+ }
+
+ public TestProperties setAthenzDomain(AthenzDomain domain) {
+ this.athenzDomain = domain;
+ return this;
+ }
+
public static class Spec implements ConfigServerSpec {
private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
index 48c21f370f4..cce5a7850a0 100644
--- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
@@ -7,7 +7,6 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.subscription.ConfigInstanceUtil;
import com.yahoo.log.LogLevel;
-import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.ConfigPayload;
import com.yahoo.vespa.config.ConfigPayloadBuilder;
@@ -21,14 +20,8 @@ import com.yahoo.vespa.model.admin.Admin;
import com.yahoo.vespa.model.admin.monitoring.Monitoring;
import com.yahoo.vespa.model.utils.FreezableMap;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.PrintStream;
import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
@@ -291,60 +284,6 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce
public AbstractConfigProducer getParent() { return parent; }
- /**
- * Writes files that need to be written. The files will usually only be
- * written when the Vespa model is generated through the deploy-application
- * script.
- *
- * TODO: Make sure all implemented ConfigProducers call createConfig()
- * instead of getConfig() when implementing this method.
- */
- public void writeFiles(File directory) throws java.io.IOException {
- if (!directory.isDirectory() && !directory.mkdirs()) {
- throw new java.io.IOException("Cannot create directory: "+ directory);
- }
- for (Method m : getClass().getMethods()) {
- try {
- ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m);
- if (builder!=null) {
- writeBuilder(directory, m, builder);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private void writeBuilder(File directory, Method m,
- ConfigInstance.Builder builder) throws IllegalAccessException,
- InvocationTargetException, InstantiationException,
- NoSuchMethodException, IOException {
- m.invoke(this, builder);
- Class<?> configInstClass = builder.getClass().getEnclosingClass();
- ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder);
- List<String> payloadList = ConfigInstance.serialize(inst);
- File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg");
- FileOutputStream out = new FileOutputStream(outfn);
- for (String s : payloadList) {
- out.write(Utf8.toBytes(s));
- out.write('\n');
- }
- }
-
- /**
- * New Builder instance if m is getConfig(SomeConfig.Builder), or null
- */
- private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws ReflectiveOperationException {
- if (!"getConfig".equals(m.getName())) return null;
- Type[] params = m.getParameterTypes();
- if (params.length!=1) return null;
- Type param = params[0];
- if (!(param instanceof Class)) return null;
- Class<?> paramClass = (Class<?>) param;
- if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null;
- return (ConfigInstance.Builder) paramClass.getDeclaredConstructor().newInstance();
- }
-
public void dump(PrintStream out) {
for (ConfigProducer c : getChildren().values()) {
out.println("id: " + c.getConfigId());
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
index f909f3864da..201b69c1aae 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -45,10 +45,16 @@ public class HostsXmlProvisioner implements HostProvisioner {
}
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) {
throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported");
}
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, ProvisionLogger logger) {
+ throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported");
+ }
+
private HostSpec host2HostSpec(Host host) {
return new HostSpec(host.hostname(), host.aliases());
}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index bfbe2eaddb3..8706bb44ded 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -4,8 +4,10 @@ package com.yahoo.config.model.provision;
import com.yahoo.collections.ListMap;
import com.yahoo.collections.Pair;
import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
@@ -49,7 +51,6 @@ public class InMemoryProvisioner implements HostProvisioner {
/** Free hosts of each resource size */
private final ListMap<NodeResources, Host> freeNodes = new ListMap<>();
- private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>();
private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>();
/** Indexes must be unique across all groups in a cluster */
@@ -58,6 +59,10 @@ public class InMemoryProvisioner implements HostProvisioner {
/** Use this index as start index for all clusters */
private final int startIndexForClusters;
+ private final boolean useMaxResources;
+
+ private Provisioned provisioned = new Provisioned();
+
/** Creates this with a number of nodes with resources 1, 3, 9, 1 */
public InMemoryProvisioner(int nodeCount) {
this(nodeCount, defaultResources);
@@ -65,27 +70,31 @@ public class InMemoryProvisioner implements HostProvisioner {
/** Creates this with a number of nodes with given resources */
public InMemoryProvisioner(int nodeCount, NodeResources resources) {
- this(Map.of(resources, createHostInstances(nodeCount)), true, 0);
+ this(Map.of(resources, createHostInstances(nodeCount)), true, false, 0);
}
/** Creates this with a set of host names of the flavor 'default' */
public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) {
- this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, 0);
+ this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, 0);
}
/** Creates this with a set of hosts of the flavor 'default' */
public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) {
- this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, 0, retiredHostNames);
+ this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, 0, retiredHostNames);
}
/** Creates this with a set of hosts of the flavor 'default' */
public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
- this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames);
+ this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames);
}
- public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, boolean failOnOutOfCapacity,
- int startIndexForClusters, String ... retiredHostNames) {
+ public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts,
+ boolean failOnOutOfCapacity,
+ boolean useMaxResources,
+ int startIndexForClusters,
+ String ... retiredHostNames) {
this.failOnOutOfCapacity = failOnOutOfCapacity;
+ this.useMaxResources = useMaxResources;
for (Map.Entry<NodeResources, Collection<Host>> hostsWithResources : hosts.entrySet())
for (Host host : hostsWithResources.getValue())
freeNodes.put(hostsWithResources.getKey(), host);
@@ -106,44 +115,52 @@ public class InMemoryProvisioner implements HostProvisioner {
@Override
public HostSpec allocateHost(String alias) {
- if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias);
List<Host> defaultHosts = freeNodes.get(defaultResources);
if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available");
Host newHost = freeNodes.removeValue(defaultResources, 0);
- HostSpec hostSpec = new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version());
- legacyMapping.put(alias, hostSpec);
- return hostSpec;
+ // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set
+ return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty());
}
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) {
- if (cluster.group().isPresent() && groups > 1)
+ return prepare(cluster, requestedCapacity.withGroups(groups), logger);
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) {
+ provisioned.add(cluster.id(), requested);
+ if (useMaxResources)
+ return prepare(cluster, requested.maxResources(), requested.isRequired(), requested.canFail());
+ else
+ return prepare(cluster, requested.minResources(), requested.isRequired(), requested.canFail());
+ }
+
+ public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) {
+ if (cluster.group().isPresent() && requested.groups() > 1)
throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
- if (requestedCapacity.nodeCount() % groups != 0)
- throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " +
- groups + " groups, but the node count is not divisible into this number of groups");
- int capacity = failOnOutOfCapacity || requestedCapacity.isRequired()
- ? requestedCapacity.nodeCount()
- : Math.min(requestedCapacity.nodeCount(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
- if (groups > capacity)
- groups = capacity;
+ int capacity = failOnOutOfCapacity || required
+ ? requested.nodes()
+ : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
+ int groups = requested.groups() > capacity ? capacity : requested.groups();
List<HostSpec> allocation = new ArrayList<>();
if (groups == 1) {
allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))),
- requestedCapacity.nodeResources(),
+ requested.nodeResources(),
capacity,
startIndexForClusters,
- requestedCapacity.canFail()));
+ canFail));
}
else {
for (int i = 0; i < groups; i++) {
allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))),
- requestedCapacity.nodeResources(),
+ requested.nodeResources(),
capacity / groups,
allocation.size(),
- requestedCapacity.canFail()));
+ canFail));
}
}
for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) {
@@ -154,15 +171,24 @@ public class InMemoryProvisioner implements HostProvisioner {
return allocation;
}
+ /** Create a new provisioned instance to record provision requests to this and returns it */
+ public Provisioned startProvisionedRecording() {
+ provisioned = new Provisioned();
+ return provisioned;
+ }
+
private HostSpec retire(HostSpec host) {
return new HostSpec(host.hostname(),
host.aliases(),
host.flavor(),
Optional.of(host.membership().get().retire()),
- host.version());
+ host.version(),
+ Optional.empty(),
+ Optional.empty(),
+ host.dockerImageRepo());
}
- private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources,
+ private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources,
int nodesInGroup, int startIndex, boolean canFail) {
List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>());
allocations.put(clusterGroup, allocation);
@@ -170,8 +196,8 @@ public class InMemoryProvisioner implements HostProvisioner {
// Check if the current allocations are compatible with the new request
for (int i = allocation.size() - 1; i >= 0; i--) {
Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources);
- if (currentResources.isEmpty() || requestedResources.isEmpty()) continue;
- if (!currentResources.get().compatibleWith(requestedResources.get())) {
+ if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue;
+ if (!currentResources.get().compatibleWith(requestedResources)) {
HostSpec removed = allocation.remove(i);
freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool
}
@@ -182,7 +208,7 @@ public class InMemoryProvisioner implements HostProvisioner {
// Find the smallest host that can fit the requested requested
Optional<NodeResources> hostResources = freeNodes.keySet().stream()
.sorted(new MemoryDiskCpu())
- .filter(resources -> requestedResources.isEmpty() || resources.satisfies(requestedResources.get()))
+ .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources))
.findFirst();
if (hostResources.isEmpty()) {
if (canFail)
@@ -195,8 +221,9 @@ public class InMemoryProvisioner implements HostProvisioner {
if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get());
ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(),
- hostResources.map(Flavor::new), Optional.of(membership),
- newHost.version(), Optional.empty(), requestedResources));
+ hostResources.map(Flavor::new), Optional.of(membership),
+ newHost.version(), Optional.empty(),
+ requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources)));
}
nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
index 180a16f3c8f..8945223447f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -30,6 +30,7 @@ public class SingleNodeProvisioner implements HostProvisioner {
host = new Host(HostName.getLocalhost());
this.hostSpec = new HostSpec(host.hostname(), host.aliases());
}
+
public SingleNodeProvisioner(Flavor flavor) {
host = new Host(HostName.getLocalhost());
this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor);
@@ -41,7 +42,14 @@ public class SingleNodeProvisioner implements HostProvisioner {
}
@Override
- public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1
+ @Deprecated // TODO: Remove after April 2020
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+ return prepare(cluster, capacity.withGroups(groups), logger);
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+ // TODO: This should fail if capacity requested is more than 1
List<HostSpec> hosts = new ArrayList<>();
hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++)));
return hosts;
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
index eb61bda83a6..10649df88e1 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
@@ -55,23 +55,23 @@ public class MockApplicationPackage implements ApplicationPackage {
private final File root;
private final String hostsS;
private final String servicesS;
- private final List<String> searchDefinitions;
- private final String searchDefinitionDir;
+ private final List<String> schemas;
+ private final String schemaDir;
private final Optional<String> deploymentSpec;
private final Optional<String> validationOverrides;
private final boolean failOnValidateXml;
private final QueryProfileRegistry queryProfileRegistry;
private final ApplicationMetaData applicationMetaData;
- protected MockApplicationPackage(File root, String hosts, String services, List<String> searchDefinitions,
- String searchDefinitionDir,
+ protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas,
+ String schemaDir,
String deploymentSpec, String validationOverrides, boolean failOnValidateXml,
String queryProfile, String queryProfileType) {
this.root = root;
this.hostsS = hosts;
this.servicesS = services;
- this.searchDefinitions = searchDefinitions;
- this.searchDefinitionDir = searchDefinitionDir;
+ this.schemas = schemas;
+ this.schemaDir = schemaDir;
this.deploymentSpec = Optional.ofNullable(deploymentSpec);
this.validationOverrides = Optional.ofNullable(validationOverrides);
this.failOnValidateXml = failOnValidateXml;
@@ -108,7 +108,7 @@ public class MockApplicationPackage implements ApplicationPackage {
@Override
public Reader getHosts() {
- if (hostsS==null) return null;
+ if (hostsS == null) return null;
return new StringReader(hostsS);
}
@@ -118,7 +118,7 @@ public class MockApplicationPackage implements ApplicationPackage {
SearchBuilder searchBuilder = new SearchBuilder(this,
new RankProfileRegistry(),
queryProfileRegistry);
- for (String sd : searchDefinitions) {
+ for (String sd : schemas) {
try {
String name = searchBuilder.importString(sd);
readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd)));
@@ -184,7 +184,7 @@ public class MockApplicationPackage implements ApplicationPackage {
@Override
public Reader getRankingExpression(String name) {
- File expressionFile = new File(searchDefinitionDir, name);
+ File expressionFile = new File(schemaDir, name);
try {
return IOUtils.createReader(expressionFile, "utf-8");
}
@@ -200,9 +200,9 @@ public class MockApplicationPackage implements ApplicationPackage {
public static ApplicationPackage fromSearchDefinitionDirectory(String dir) {
return new MockApplicationPackage.Builder()
- .withEmptyHosts()
- .withEmptyServices()
- .withSearchDefinitionDir(dir).build();
+ .withEmptyHosts()
+ .withEmptyServices()
+ .withSchemaDir(dir).build();
}
public static class Builder {
@@ -210,8 +210,8 @@ public class MockApplicationPackage implements ApplicationPackage {
private File root = new File("nonexisting");
private String hosts = null;
private String services = null;
- private List<String> searchDefinitions = Collections.emptyList();
- private String searchDefinitionDir = null;
+ private List<String> schemas = Collections.emptyList();
+ private String schemaDir = null;
private String deploymentSpec = null;
private String validationOverrides = null;
private boolean failOnValidateXml = false;
@@ -245,17 +245,17 @@ public class MockApplicationPackage implements ApplicationPackage {
}
public Builder withSearchDefinition(String searchDefinition) {
- this.searchDefinitions = Collections.singletonList(searchDefinition);
+ this.schemas = Collections.singletonList(searchDefinition);
return this;
}
- public Builder withSearchDefinitions(List<String> searchDefinition) {
- this.searchDefinitions = Collections.unmodifiableList(searchDefinition);
+ public Builder withSchemas(List<String> searchDefinition) {
+ this.schemas = Collections.unmodifiableList(searchDefinition);
return this;
}
- public Builder withSearchDefinitionDir(String searchDefinitionDir) {
- this.searchDefinitionDir = searchDefinitionDir;
+ public Builder withSchemaDir(String schemaDir) {
+ this.schemaDir = schemaDir;
return this;
}
@@ -285,7 +285,7 @@ public class MockApplicationPackage implements ApplicationPackage {
}
public ApplicationPackage build() {
- return new MockApplicationPackage(root, hosts, services, searchDefinitions, searchDefinitionDir,
+ return new MockApplicationPackage(root, hosts, services, schemas, schemaDir,
deploymentSpec, validationOverrides, failOnValidateXml,
queryProfile, queryProfileType);
}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
index 13f271ebe9d..87d6554f691 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.Set;
@@ -66,7 +67,7 @@ public class MockRoot extends AbstractConfigProducerRoot {
super(rootConfigId);
hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger());
this.deployState = deployState;
- fileDistributor = new FileDistributor(deployState.getFileRegistry(), null);
+ fileDistributor = new FileDistributor(deployState.getFileRegistry(), List.of(), deployState.isHosted());
}
public FileDistributionConfigProducer getFileDistributionConfigProducer() {
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index fc42864f1d0..41a30c4553d 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
private final StructDataType body;
private final Set<FieldSet> fieldSets = new LinkedHashSet<>();
private final Set<Name> documentReferences;
+ // Imported fields are virtual and therefore exist outside of the SD's document field definition
+ // block itself. But for features like imported fields in a non-search context (e.g. GC selections)
+ // it is necessary to know that certain identifiers refer to imported fields instead of being unknown
+ // document fields. To achieve this, we track the names of imported fields as part of the document
+ // config itself.
+ private final Set<String> importedFieldNames;
public NewDocumentType(Name name) {
this(name, emptySet());
}
- public NewDocumentType(Name name, Set<Name> documentReferences) {
+ public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) {
this(
name,
new StructDataType(name.getName() + ".header"),
new StructDataType(name.getName() + ".body"),
new FieldSets(),
- documentReferences);
+ documentReferences,
+ importedFieldNames);
+ }
+
+ public NewDocumentType(Name name, Set<Name> documentReferences) {
+ this(name, documentReferences, emptySet());
}
public NewDocumentType(Name name,
StructDataType header,
StructDataType body,
FieldSets fs,
- Set<Name> documentReferences) {
+ Set<Name> documentReferences,
+ Set<String> importedFieldNames) {
super(name.getName());
this.name = name;
this.header = header;
@@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
}
}
this.documentReferences = documentReferences;
+ this.importedFieldNames = importedFieldNames;
}
public Name getFullName() {
@@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
return documentReferences;
}
+ public Set<String> getImportedFieldNames() {
+ return importedFieldNames;
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index f0b1b427531..d3a78321106 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType;
import com.yahoo.searchdefinition.document.Attribute;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -338,7 +340,8 @@ public class DocumentModelBuilder {
sdoc.getDocumentType().contentStruct(),
sdoc.getDocumentType().getBodyType(),
sdoc.getFieldSets(),
- convertDocumentReferencesToNames(sdoc.getDocumentReferences()));
+ convertDocumentReferencesToNames(sdoc.getDocumentReferences()),
+ convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields()));
for (SDDocumentType n : sdoc.getInheritedTypes()) {
NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
NewDocumentType inherited = model.getDocumentManager().getDocumentType(name);
@@ -404,6 +407,13 @@ public class DocumentModelBuilder {
.collect(toSet());
}
+ private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) {
+ if (importedFields == null) {
+ return emptySet();
+ }
+ return Collections.unmodifiableSet(importedFields.fields().keySet());
+ }
+
private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
for (Field f : fields) {
DataType type = f.getDataType();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
index 14f8a0a9d37..0d0da71bd0f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
@@ -25,8 +25,8 @@ public class DocumentReferenceResolver {
private final Map<String, Search> searchMapping;
- public DocumentReferenceResolver(List<Search> searchDefinitions) {
- this.searchMapping = createDocumentNameToSearchMapping(searchDefinitions);
+ public DocumentReferenceResolver(List<Search> schemas) {
+ this.searchMapping = createDocumentNameToSearchMapping(schemas);
}
public void resolveReferences(SDDocumentType documentType) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
index 9ff749a994c..ade8ae21870 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
@@ -41,7 +41,7 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier {
}
if (structUsedByField.getName().equals(structType.getName())) {
//this field is using this type!!
- field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0);
+ field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0);
field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching());
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
new file mode 100644
index 00000000000..25cdd1e08cd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
@@ -0,0 +1,31 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+
+import java.util.List;
+
+/**
+ * Enumerates and emplaces a set of all imported fields into a SDDocumentType from
+ * its corresponding Search instance.
+ */
+public class ImportedFieldsEnumerator {
+
+ private final List<Search> schemas;
+
+ public ImportedFieldsEnumerator(List<Search> schemas) {
+ this.schemas = schemas;
+ }
+
+ public void enumerateImportedFields(SDDocumentType documentType) {
+ var search = this.schemas.stream()
+ .filter(s -> s.getDocument() != null)
+ .filter(s -> s.getDocument().getName().equals(documentType.getName()))
+ .findFirst();
+ if (search.isEmpty()) {
+ return; // No imported fields present.
+ }
+ search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
index e46db1d1b5f..aba6cf9a233 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
@@ -2,6 +2,7 @@
package com.yahoo.searchdefinition;
import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
import com.yahoo.searchdefinition.document.RankType;
import com.yahoo.searchdefinition.document.Stemming;
@@ -9,6 +10,9 @@ import java.io.Serializable;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashSet;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
/**
@@ -20,6 +24,8 @@ import java.util.Set;
*/
public class Index implements Cloneable, Serializable {
+ public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES }
+
public enum Type {
VESPA("vespa");
@@ -57,6 +63,10 @@ public class Index implements Cloneable, Serializable {
/** The boolean index definition, if set */
private BooleanIndexDefinition boolIndex;
+ private Optional<HnswIndexParams> hnswIndexParams = Optional.empty();
+
+ private Optional<DistanceMetric> distanceMetric = Optional.empty();
+
/** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */
private boolean interleavedFeatures = false;
@@ -115,20 +125,26 @@ public class Index implements Cloneable, Serializable {
}
@Override
- public int hashCode() {
- return name.hashCode() + ( prefix ? 17 : 0 );
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Index index = (Index) o;
+ return prefix == index.prefix &&
+ normalized == index.normalized &&
+ interleavedFeatures == index.interleavedFeatures &&
+ Objects.equals(name, index.name) &&
+ rankType == index.rankType &&
+ Objects.equals(aliases, index.aliases) &&
+ stemming == index.stemming &&
+ type == index.type &&
+ Objects.equals(boolIndex, index.boolIndex) &&
+ Objects.equals(distanceMetric, index.distanceMetric) &&
+ Objects.equals(hnswIndexParams, index.hnswIndexParams);
}
@Override
- public boolean equals(Object object) {
- if ( ! (object instanceof Index)) return false;
-
- Index other=(Index)object;
- return
- this.name.equals(other.name) &&
- this.prefix==other.prefix &&
- this.stemming==other.stemming &&
- this.normalized==other.normalized;
+ public int hashCode() {
+ return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, distanceMetric, hnswIndexParams, interleavedFeatures);
}
public String toString() {
@@ -176,6 +192,24 @@ public class Index implements Cloneable, Serializable {
boolIndex = def;
}
+ public Optional<DistanceMetric> getDistanceMetric() {
+ return distanceMetric;
+ }
+
+ public void setDistanceMetric(String value) {
+ String upper = value.toUpperCase(Locale.ENGLISH);
+ DistanceMetric dm = DistanceMetric.valueOf(upper);
+ distanceMetric = Optional.of(dm);
+ }
+
+ public Optional<HnswIndexParams> getHnswIndexParams() {
+ return hnswIndexParams;
+ }
+
+ public void setHnswIndexParams(HnswIndexParams params) {
+ hnswIndexParams = Optional.of(params);
+ }
+
public void setInterleavedFeatures(boolean value) {
interleavedFeatures = value;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index fa196cd3bbf..23eb814de81 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -788,8 +788,7 @@ public class RankProfile implements Cloneable {
type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () ->
new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() +
" with type " + field.getType().asTensorType() +
- ", but this is already defined " +
- "in another query profile with type " +
+ ", but this is already defined in another query profile with type " +
context.getType(feature.get())));
context.setType(feature.get(), type);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index f90a7e4f6cd..0ab8a2308a4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -288,7 +288,7 @@ public class Search implements ImmutableSearch {
/**
* Adds an extra field of this search definition not contained in a document
*
- * @param field to add to the searchdefinitions list of external fields.
+ * @param field to add to the schemas list of external fields
*/
public void addExtraField(SDField field) {
if (fields.containsKey(field.getName())) {
@@ -383,7 +383,7 @@ public class Search implements ImmutableSearch {
* Consolidates a set of index settings for the same index into one
*
* @param indices The list of indexes to consolidate.
- * @return The consolidated index
+ * @return the consolidated index
*/
private Index consolidateIndices(List<Index> indices) {
Index first = indices.get(0);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 949539ff99f..eb68e6af203 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -230,20 +230,22 @@ public class SearchBuilder {
sdocs.add(search.getDocument());
}
}
- SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+ var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
orderer.process();
for (SDDocumentType sdoc : orderer.getOrdered()) {
new FieldOperationApplierForStructs().process(sdoc);
new FieldOperationApplier().process(sdoc);
}
- DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList);
+ var resolver = new DocumentReferenceResolver(searchList);
sdocs.forEach(resolver::resolveReferences);
+ var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList);
+ sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields);
if (validate)
new DocumentGraphValidator().validateDocumentGraph(sdocs);
- DocumentModelBuilder builder = new DocumentModelBuilder(model);
+ var builder = new DocumentModelBuilder(model);
for (Search search : new SearchOrderer().order(searchList)) {
new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list?
process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 76dff404568..8b5f7658475 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -240,6 +240,17 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
aaB.tensortype(attribute.tensorType().get().toString());
}
aaB.imported(imported);
+ var dma = attribute.distanceMetric();
+ if (attribute.hnswIndexParams().isPresent()) {
+ var ib = new AttributesConfig.Attribute.Index.Builder();
+ var params = attribute.hnswIndexParams().get();
+ ib.hnsw.enabled(true);
+ ib.hnsw.maxlinkspernode(params.maxLinksPerNode());
+ ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert());
+ var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(dma.toString());
+ ib.hnsw.distancemetric(dm);
+ aaB.index(ib);
+ }
return aaB;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 245913d7822..fc8710fa1a1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -15,9 +15,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.derived.validation.Validation;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
import java.io.IOException;
import java.io.Writer;
+import java.util.logging.Level;
/**
* A set of all derived configuration of a search definition. Use this as a facade to individual configurations when
@@ -39,12 +41,13 @@ public class DerivedConfiguration {
private VsmSummary streamingSummary;
private IndexSchema indexSchema;
private ImportedFields importedFields;
+ private QueryProfileRegistry queryProfiles;
/**
* Creates a complete derived configuration from a search definition.
* Only used in tests.
*
- * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is
+ * @param search the search to derive a configuration from. Derived objects will be snapshots, but this argument is
* live. Which means that this object will be inconsistent when the given search definition is later
* modified.
* @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
@@ -56,11 +59,11 @@ public class DerivedConfiguration {
/**
* Creates a complete derived configuration snapshot from a search definition.
*
- * @param search The search to derive a configuration from. Derived objects will be snapshots, but this
+ * @param search the search to derive a configuration from. Derived objects will be snapshots, but this
* argument is live. Which means that this object will be inconsistent when the given
* search definition is later modified.
* @param deployLogger a {@link DeployLogger} for logging when doing operations on this
- * @param deployProperties Properties set on deploy.
+ * @param deployProperties properties set on deploy
* @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
* @param queryProfiles the query profiles of this application
*/
@@ -72,6 +75,7 @@ public class DerivedConfiguration {
ImportedMlModels importedModels) {
Validator.ensureNotNull("Search definition", search);
this.search = search;
+ this.queryProfiles = queryProfiles;
if ( ! search.isDocumentsOnly()) {
streamingFields = new VsmFields(search);
streamingSummary = new VsmSummary(search);
@@ -120,6 +124,10 @@ public class DerivedConfiguration {
exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg");
}
+ public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException {
+ exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg");
+ }
+
private static void exportCfg(ConfigInstance instance, String fileName) throws IOException {
Writer writer = null;
try {
@@ -186,4 +194,7 @@ public class DerivedConfiguration {
public ImportedFields getImportedFields() {
return importedFields;
}
+
+ public QueryProfileRegistry getQueryProfiles() { return queryProfiles; }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
index 02b2a383318..da25680ca47 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -1,16 +1,25 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
-import com.yahoo.document.*;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.document.PositionDataType;
import com.yahoo.searchdefinition.Index;
import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.Stemming;
import com.yahoo.searchdefinition.processing.ExactMatch;
import com.yahoo.searchdefinition.processing.NGramMatch;
import com.yahoo.vespa.documentmodel.SummaryField;
import com.yahoo.search.config.IndexInfoConfig;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
/**
@@ -34,8 +43,10 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
private static final String CMD_PLAIN_TOKENS = "plain-tokens";
private static final String CMD_MULTIVALUE = "multivalue";
private static final String CMD_FAST_SEARCH = "fast-search";
+ private static final String CMD_PREDICATE = "predicate";
private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds";
private static final String CMD_NUMERICAL = "numerical";
+ private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting";
private Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
private Map<String, String> aliases = new java.util.LinkedHashMap<>();
private Map<String, FieldSet> fieldSets;
@@ -90,6 +101,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
protected void derive(ImmutableSDField field, Search search, boolean inPosition) {
if (field.getDataType().equals(DataType.PREDICATE)) {
+ addIndexCommand(field, CMD_PREDICATE);
Index index = field.getIndex(field.getName());
if (index != null) {
BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
@@ -286,21 +298,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
// TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this.
private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) {
- // Explicit query commands on the field set, overrides everything.
- if (!fieldSet.queryCommands().isEmpty()) {
- for (String qc : fieldSet.queryCommands()) {
- iiB.command(
- new IndexInfoConfig.Indexinfo.Command.Builder()
- .indexname(fieldSet.getName())
- .command(qc));
- }
- return;
- }
+ for (String qc : fieldSet.queryCommands())
+ iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder().indexname(fieldSet.getName()).command(qc));
boolean anyIndexing = false;
boolean anyAttributing = false;
boolean anyLowerCasing = false;
boolean anyStemming = false;
boolean anyNormalizing = false;
+ String phraseSegmentingCommand = null;
String stemmingCommand = null;
Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching
// First a pass over the fields to read some params to decide field settings implicitly:
@@ -321,8 +326,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
if (field.getNormalizing().doRemoveAccents()) {
anyNormalizing = true;
}
- if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType)
+ if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) {
fieldSetMatching = field.getMatching();
+ }
+ Optional<String> explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst();
+ if (explicitPhraseSegmentingCommand.isPresent()) {
+ phraseSegmentingCommand = explicitPhraseSegmentingCommand.get();
+ }
}
if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) {
// We have both attributes and indexes and no explicit match setting ->
@@ -365,6 +375,11 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
new IndexInfoConfig.Indexinfo.Command.Builder()
.indexname(fieldSet.getName())
.command(CMD_NORMALIZE));
+ if (phraseSegmentingCommand != null)
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(phraseSegmentingCommand));
}
} else {
// Assume only attribute fields
@@ -400,9 +415,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
} else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) {
}
-
}
-
}
private boolean hasMultiValueField(FieldSet fieldSet) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
index 60b8ee78c7b..39a67a69575 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
@@ -5,6 +5,7 @@ import com.yahoo.document.ArrayDataType;
import com.yahoo.document.DataType;
import com.yahoo.document.Field;
import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.TensorDataType;
import com.yahoo.document.WeightedSetDataType;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
@@ -20,7 +21,7 @@ import java.util.List;
import java.util.Map;
/**
- * Deriver of indexschema config containing information of all index fields with name and data type.
+ * Deriver of indexschema config containing information of all text index fields with name and data type.
*
* @author geirst
*/
@@ -44,9 +45,14 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
super.derive(search);
}
+ private boolean isTensorField(ImmutableSDField field) {
+ return field.getDataType() instanceof TensorDataType;
+ }
+
private void deriveIndexFields(ImmutableSDField field, Search search) {
- if (!field.doesIndexing() &&
- !field.isIndexStructureField())
+ // Note: Indexes for tensor fields are NOT part of the index schema for text fields.
+ if ((!field.doesIndexing() && !field.isIndexStructureField()) ||
+ isTensorField(field))
{
return;
}
@@ -138,11 +144,10 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
return Collections.singletonList(field);
}
if (fieldType instanceof ArrayDataType) {
- boolean header = field.isHeader();
List<Field> ret = new LinkedList<>();
- Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header);
+ Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType());
for (Field flatField : flattenField(innerField)) {
- ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header));
+ ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType())));
}
return ret;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index fbcaf2a3a80..1661a80f238 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -24,6 +24,7 @@ import com.yahoo.document.datatypes.Float16FieldValue;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.document.datatypes.TensorFieldValue;
import com.yahoo.tensor.TensorType;
+import static com.yahoo.searchdefinition.Index.DistanceMetric;
import java.io.Serializable;
import java.util.LinkedHashSet;
@@ -66,6 +67,10 @@ public final class Attribute implements Cloneable, Serializable {
/** This is set if the type of this is REFERENCE */
private final Optional<StructuredDataType> referenceDocumentType;
+ private Optional<DistanceMetric> distanceMetric = Optional.empty();
+
+ private Optional<HnswIndexParams> hnswIndexParams = Optional.empty();
+
private boolean isPosition = false;
private final Sorting sorting = new Sorting();
@@ -195,6 +200,12 @@ public final class Attribute implements Cloneable, Serializable {
public Optional<TensorType> tensorType() { return tensorType; }
public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; }
+ public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN;
+ public DistanceMetric distanceMetric() {
+ return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC);
+ }
+ public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; }
+
public Sorting getSorting() { return sorting; }
public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; }
@@ -217,6 +228,8 @@ public final class Attribute implements Cloneable, Serializable {
public void setUpperBound(long upperBound) { this.upperBound = upperBound; }
public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; }
public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); }
+ public void setDistanceMetric(Optional<DistanceMetric> dm) { this.distanceMetric = dm; }
+ public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); }
public String getName() { return name; }
public Type getType() { return type; }
@@ -335,7 +348,7 @@ public final class Attribute implements Cloneable, Serializable {
public int hashCode() {
return Objects.hash(
name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent,
- isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType);
+ isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams);
}
@Override
@@ -349,8 +362,8 @@ public final class Attribute implements Cloneable, Serializable {
/** Returns whether these attributes describes the same entity, even if they have different names */
public boolean isCompatible(Attribute other) {
- if ( ! this.type.equals(other.type)) return false;
- if ( ! this.collectionType.equals(other.collectionType)) return false;
+ if (! this.type.equals(other.type)) return false;
+ if (! this.collectionType.equals(other.collectionType)) return false;
if (this.isPrefetch() != other.isPrefetch()) return false;
if (this.removeIfZero != other.removeIfZero) return false;
if (this.createIfNonExistent != other.createIfNonExistent) return false;
@@ -359,9 +372,11 @@ public final class Attribute implements Cloneable, Serializable {
// if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now
if (this.fastSearch != other.fastSearch) return false;
if (this.huge != other.huge) return false;
- if ( ! this.sorting.equals(other.sorting)) return false;
- if (!this.tensorType.equals(other.tensorType)) return false;
- if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false;
+ if (! this.sorting.equals(other.sorting)) return false;
+ if (! Objects.equals(tensorType, other.tensorType)) return false;
+ if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false;
+ if (! Objects.equals(distanceMetric, other.distanceMetric)) return false;
+ if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false;
return true;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
index 55dedc4a1d7..22424286ef9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
@@ -25,10 +25,7 @@ public class FieldSet {
public FieldSet addFieldName(String field) { fieldNames.add(field); return this; }
public Set<String> getFieldNames() { return fieldNames; }
public Set<ImmutableSDField> fields() { return fields; }
-
- public Set<String> queryCommands() {
- return queryCommands;
- }
+ public Set<String> queryCommands() { return queryCommands; }
public void setMatching(Matching matching) {
this.matching = matching;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java
new file mode 100644
index 00000000000..2f084d3e513
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java
@@ -0,0 +1,64 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.util.Locale;
+import java.util.Optional;
+
+/**
+ * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search.
+ *
+ * @author geirst
+ */
+public class HnswIndexParams {
+
+ public static final int DEFAULT_MAX_LINKS_PER_NODE = 16;
+ public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200;
+
+ private final Optional<Integer> maxLinksPerNode;
+ private final Optional<Integer> neighborsToExploreAtInsert;
+
+ public static class Builder {
+ private Optional<Integer> maxLinksPerNode = Optional.empty();
+ private Optional<Integer> neighborsToExploreAtInsert = Optional.empty();
+
+ public void setMaxLinksPerNode(int value) {
+ maxLinksPerNode = Optional.of(value);
+ }
+ public void setNeighborsToExploreAtInsert(int value) {
+ neighborsToExploreAtInsert = Optional.of(value);
+ }
+ public HnswIndexParams build() {
+ return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert);
+ }
+ }
+
+ public HnswIndexParams() {
+ this.maxLinksPerNode = Optional.empty();
+ this.neighborsToExploreAtInsert = Optional.empty();
+ }
+
+ public HnswIndexParams(Optional<Integer> maxLinksPerNode,
+ Optional<Integer> neighborsToExploreAtInsert) {
+ this.maxLinksPerNode = maxLinksPerNode;
+ this.neighborsToExploreAtInsert = neighborsToExploreAtInsert;
+ }
+
+ /**
+ * Creates a new instance where values from the given parameter instance are used where they are present,
+ * otherwise we use values from this.
+ */
+ public HnswIndexParams overrideFrom(Optional<HnswIndexParams> other) {
+ if (! other.isPresent()) return this;
+ HnswIndexParams rhs = other.get();
+ return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode),
+ rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert));
+ }
+
+ public int maxLinksPerNode() {
+ return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE);
+ }
+
+ public int neighborsToExploreAtInsert() {
+ return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index 414a605c621..b3f98cc6f26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable {
private FieldSets fieldSets;
// Document references
private Optional<DocumentReferences> documentReferences = Optional.empty();
+ private TemporaryImportedFields temporaryImportedFields;
static {
VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
@@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable {
type.docType = docType.clone();
type.inheritedTypes.putAll(inheritedTypes);
type.structType = structType;
+ // TODO this isn't complete; should it be..?!
return type;
}
@@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable {
this.documentReferences = Optional.of(documentReferences);
}
+ public TemporaryImportedFields getTemporaryImportedFields() {
+ return temporaryImportedFields;
+ }
+
+ public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) {
+ this.temporaryImportedFields = temporaryImportedFields;
+ }
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index c657d29033a..30ce142d503 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -38,9 +38,8 @@ import java.util.TreeMap;
/**
* The field class represents a document field. It is used in
- * the Document class to get and set fields. Each SDField has
- * a name, a numeric ID, a data type, and a boolean that says whether it's
- * a header field. The numeric ID is used when the fields are stored
+ * the Document class to get and set fields. Each SDField has a name, a numeric ID,
+ * a data type. The numeric ID is used when the fields are stored
* in serialized form.
*
* @author bratseth
@@ -92,7 +91,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
private NormalizeLevel normalizing = new NormalizeLevel();
/** Extra query commands of this field */
- private List<String> queryCommands=new java.util.ArrayList<>(0);
+ private List<String> queryCommands = new java.util.ArrayList<>(0);
/** Summary fields defined in this field */
private Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(0);
@@ -120,15 +119,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
* Creates a new field. This method is only used to create reserved fields
* @param name The name of the field
* @param dataType The datatype of the field
- * @param isHeader Whether this is a "header" field or a "content" field (true = "header").
*/
- protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) {
- super(name, id, dataType, isHeader);
- populate(populate, repo, name, dataType, isHeader);
+ protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) {
+ super(name, id, dataType);
+ populate(populate, repo, name, dataType);
}
- public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) {
- this(repo, name, id, dataType, isHeader, true);
+ public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
+ this(repo, name, id, dataType, true);
}
/**
@@ -136,41 +134,35 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
@param name The name of the field
@param dataType The datatype of the field
- @param isHeader Whether this is a "header" field or a "content" field
- (true = "header").
*/
- public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) {
- super(name,dataType,isHeader);
- populate(populate, repo, name, dataType, isHeader);
+ public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) {
+ super(name,dataType);
+ populate(populate, repo, name, dataType);
}
- private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) {
- populate(populate,repo, name, dataType, isHeader, null, 0);
+ private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) {
+ populate(populate,repo, name, dataType, null, 0);
}
- private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching, int recursion) {
+ private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) {
if (populate || (dataType instanceof MapDataType)) {
- populateWithStructFields(repo, name, dataType, isHeader, recursion);
+ populateWithStructFields(repo, name, dataType, recursion);
populateWithStructMatching(repo, name, dataType, fieldMatching);
}
}
- public SDField(String name, DataType dataType, boolean isHeader) {
- this(null, name, dataType, isHeader, true);
- }
/**
* Creates a new field.
*
* @param name The name of the field
* @param dataType The datatype of the field
- * @param isHeader Whether this is a "header" field or a "content" field (true = "header").
* @param owner the owning document (used to check for id collisions)
*/
- protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) {
- super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) {
+ super(name, dataType, owner == null ? null : owner.getDocumentType());
this.ownerDocType=owner;
- populate(populate, repo, name, dataType, isHeader);
+ populate(populate, repo, name, dataType);
}
/**
@@ -178,27 +170,25 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
*
* @param name The name of the field
* @param dataType The datatype of the field
- * @param isHeader Whether this is a "header" field or a "content" field (true = "header").
* @param owner The owning document (used to check for id collisions)
* @param fieldMatching The matching object to set for the field
*/
- protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner,
+ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner,
Matching fieldMatching, boolean populate, int recursion) {
- super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+ super(name, dataType, owner == null ? null : owner.getDocumentType());
this.ownerDocType=owner;
if (fieldMatching != null)
this.setMatching(fieldMatching);
- populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion);
+ populate(populate, repo, name, dataType, fieldMatching, recursion);
}
/**
- * Constructor for <b>header</b> fields
*
* @param name The name of the field
* @param dataType The datatype of the field
*/
public SDField(SDDocumentType repo, String name, DataType dataType) {
- this(repo, name,dataType,true, true);
+ this(repo, name,dataType, true);
}
public SDField(String name, DataType dataType) {
this(null, name,dataType);
@@ -277,7 +267,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
}
}
- public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) {
+ public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) {
DataType dt = getFirstStructOrMapRecursive();
if (dt == null) {
return;
@@ -286,11 +276,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
MapDataType mdt = (MapDataType) dataType;
SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(),
- isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+ getOwnerDocType(), new Matching(), true, recursion + 1);
structFields.put("key", keyField);
SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(),
- isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+ getOwnerDocType(), new Matching(), true, recursion + 1);
structFields.put("value", valueField);
} else {
if (recursion >= 10) {
@@ -306,7 +296,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
}
for (Field field : subType.fieldSet()) {
SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(),
- isHeader, subType, new Matching(), true, recursion + 1);
+ subType, new Matching(), true, recursion + 1);
structFields.put(field.getName(), subField);
}
}
@@ -759,20 +749,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
return queryCommands.contains(name);
}
- /**
- * A list of query commands
- *
- * @return a list of strings with query commands.
- */
+ /** Returns a list of query commands */
@Override
- public List<String> getQueryCommands() {
- return queryCommands;
- }
+ public List<String> getQueryCommands() { return queryCommands; }
- /**
- * The document that this field was declared in, or null
- *
- */
+ /** Returns the document that this field was declared in, or null */
private SDDocumentType getOwnerDocType() {
return ownerDocType;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
index 886bf777d3a..04d11792379 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
@@ -8,12 +8,12 @@ import com.yahoo.document.DataType;
*/
public class TemporarySDField extends SDField {
- public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) {
- super(owner, name, dataType, isHeader, owner, false);
+ public TemporarySDField(String name, DataType dataType, SDDocumentType owner) {
+ super(owner, name, dataType, owner, false);
}
public TemporarySDField(String name, DataType dataType) {
- super(null, name, dataType, true, false);
+ super(null, name, dataType, false);
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
index 6fdf448a39b..a6707ec7ac0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
@@ -27,6 +27,7 @@ public class ExpressionTransforms {
ImmutableList.of(new TensorFlowFeatureConverter(),
new OnnxFeatureConverter(),
new XgboostFeatureConverter(),
+ new LightGBMFeatureConverter(),
new ConstantDereferencer(),
new ConstantTensorTransformer(),
new FunctionInliner(),
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java
new file mode 100644
index 00000000000..5bde627dc0a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java
@@ -0,0 +1,59 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.expressiontransforms;
+
+import com.yahoo.path.Path;
+import com.yahoo.searchlib.rankingexpression.rule.Arguments;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+import com.yahoo.vespa.model.ml.ConvertedModel;
+import com.yahoo.vespa.model.ml.FeatureArguments;
+
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Replaces instances of the lightgbm(model-path) pseudofeature with the
+ * native Vespa ranking expression implementing the same computation.
+ *
+ * @author lesters
+ */
+public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> {
+
+ /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */
+ private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>();
+
+ @Override
+ public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) {
+ if (node instanceof ReferenceNode)
+ return transformFeature((ReferenceNode) node, context);
+ else if (node instanceof CompositeNode)
+ return super.transformChildren((CompositeNode) node, context);
+ else
+ return node;
+ }
+
+ private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) {
+ if ( ! feature.getName().equals("lightgbm")) return feature;
+
+ try {
+ FeatureArguments arguments = asFeatureArguments(feature.getArguments());
+ ConvertedModel convertedModel =
+ convertedLightGBMModels.computeIfAbsent(arguments.path(),
+ path -> ConvertedModel.fromSourceOrStore(path, true, context));
+ return convertedModel.expression(arguments, context);
+ } catch (IllegalArgumentException | UncheckedIOException e) {
+ throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e);
+ }
+ }
+
+ private FeatureArguments asFeatureArguments(Arguments arguments) {
+ if (arguments.size() != 1)
+ throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " +
+ "the LightGBM model file under [application]/models");
+ return new FeatureArguments(arguments);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
index 09e8b3ccdfa..ab9148992e0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
@@ -11,7 +11,7 @@ public interface FieldOperationContainer {
/** Adds an operation */
void addOperation(FieldOperation op);
- /** Apply all operations. Operations must be sorted in thjeir natural order before applying each operation. */
+ /** Apply all operations. Operations must be sorted in their natural order before applying each operation. */
void applyOperations(SDField field);
String getName();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
index 39f543c7db3..0c1f443dee3 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
@@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.fieldoperation;
import com.yahoo.searchdefinition.Index;
import com.yahoo.searchdefinition.Index.Type;
import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.document.Stemming;
@@ -31,6 +32,9 @@ public class IndexOperation implements FieldOperation {
private OptionalDouble densePostingListThreshold = OptionalDouble.empty();
private Optional<Boolean> enableBm25 = Optional.empty();
+ private Optional<String> distanceMetric = Optional.empty();
+ private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty();
+
public String getIndexName() {
return indexName;
}
@@ -91,6 +95,12 @@ public class IndexOperation implements FieldOperation {
if (enableBm25.isPresent()) {
index.setInterleavedFeatures(enableBm25.get());
}
+ if (distanceMetric.isPresent()) {
+ index.setDistanceMetric(distanceMetric.get());
+ }
+ if (hnswIndexParams.isPresent()) {
+ index.setHnswIndexParams(hnswIndexParams.get().build());
+ }
}
public Type getType() {
@@ -116,8 +126,17 @@ public class IndexOperation implements FieldOperation {
public void setDensePostingListThreshold(double densePostingListThreshold) {
this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold);
}
+
public void setEnableBm25(boolean value) {
enableBm25 = Optional.of(value);
}
+ public void setDistanceMetric(String value) {
+ this.distanceMetric = Optional.of(value);
+ }
+
+ public void setHnswIndexParams(HnswIndexParams.Builder params) {
+ this.hnswIndexParams = Optional.of(params);
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
index a0d47d7fa81..0a29fae04bf 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
@@ -4,9 +4,10 @@ package com.yahoo.searchdefinition.fieldoperation;
import com.yahoo.searchdefinition.document.SDField;
/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @author Einar M R Rosenvinge
*/
public class IndexingRewriteOperation implements FieldOperation {
- public void apply(SDField field) {
- }
+
+ public void apply(SDField field) { }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
index d1eb18c4916..0ffd13927b4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -71,7 +71,7 @@ public class AddExtraFieldsToDocument extends Processor {
if (docField == null) {
ImmutableSDField existingField = search.getField(field.getName());
if (existingField == null) {
- SDField newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true);
+ SDField newField = new SDField(document, field.getName(), field.getDataType(), true);
newField.setIsExtraField(true);
document.addField(newField);
} else if (!existingField.isImportedField()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index d6c334ee80b..6d3de23238d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor {
}
private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference,
- ImmutableSDField targetField, boolean validate) {
+ ImmutableSDField targetField, boolean validate) {
TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()),
reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName()));
ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference);
@@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor {
}
private void validateTargetField(TemporaryImportedField importedField,
- ImmutableSDField targetField, DocumentReference reference) {
+ ImmutableSDField targetField, DocumentReference reference) {
if (!targetField.doesAttributing()) {
fail(importedField, targetFieldAsString(targetField.getName(), reference) +
": Is not an attribute field. Only attribute fields supported");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
index 9a9e9bbba5f..89b8889b4ae 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
@@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType;
import com.yahoo.tensor.evaluation.TypeContext;
import com.yahoo.vespa.model.container.search.QueryProfiles;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.logging.Level;
/**
@@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor {
public void process(boolean validate, boolean documentsOnly) {
if (documentsOnly) return;
+ Set<Reference> warnedAbout = new HashSet<>();
for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) {
try {
- resolveTypesIn(profile, validate);
+ resolveTypesIn(profile, validate, warnedAbout);
}
catch (IllegalArgumentException e) {
throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e);
@@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor {
*
* @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types
*/
- private void resolveTypesIn(RankProfile profile, boolean validate) {
+ private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) {
MapEvaluationTypeContext context = profile.typeContext(queryProfiles);
for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) {
ExpressionFunction expressionFunction = function.getValue().function();
@@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor {
profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context));
ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context);
- if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) {
- deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " +
+ if ( context.tensorsAreUsed() &&
+ ! context.queryFeaturesNotDeclared().isEmpty() &&
+ ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) {
+ deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() +
+ "' are not declared in query profile " +
"types and will be interpreted as scalars, not tensors: " +
context.queryFeaturesNotDeclared());
+ warnedAbout.addAll(context.queryFeaturesNotDeclared());
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index 8e54d7c00d6..c97ee2bd935 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType;
import com.yahoo.document.TensorDataType;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.vespa.model.container.search.QueryProfiles;
@@ -23,34 +24,71 @@ public class TensorFieldProcessor extends Processor {
@Override
public void process(boolean validate, boolean documentsOnly) {
- if ( ! validate) return;
-
- for (SDField field : search.allConcreteFields()) {
+ for (var field : search.allConcreteFields()) {
if ( field.getDataType() instanceof TensorDataType ) {
- validateIndexingScripsForTensorField(field);
- validateAttributeSettingForTensorField(field);
+ if (validate) {
+ validateIndexingScripsForTensorField(field);
+ validateAttributeSettingForTensorField(field);
+ }
+ processIndexSettingsForTensorField(field, validate);
}
else if (field.getDataType() instanceof CollectionDataType){
- validateDataTypeForCollectionField(field);
+ if (validate) {
+ validateDataTypeForCollectionField(field);
+ }
}
}
}
private void validateIndexingScripsForTensorField(SDField field) {
- if (field.doesIndexing()) {
- fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field.");
+ if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) {
+ fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " +
+ "Currently, only tensors with 1 indexed dimension supports that.");
+ }
+ }
+
+ private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) {
+ var type = ((TensorDataType)field.getDataType()).getTensorType();
+ // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search).
+ if ((type.dimensions().size() == 1) &&
+ type.dimensions().get(0).isIndexed()) {
+ return true;
}
+ return false;
+ }
+
+ private String tensorTypeToString(ImmutableSDField field) {
+ return ((TensorDataType)field.getDataType()).getTensorType().toString();
}
private void validateAttributeSettingForTensorField(SDField field) {
if (field.doesAttributing()) {
- Attribute attribute = field.getAttributes().get(field.getName());
+ var attribute = field.getAttributes().get(field.getName());
if (attribute != null && attribute.isFastSearch()) {
fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'.");
}
}
}
+ private void processIndexSettingsForTensorField(SDField field, boolean validate) {
+ if (!field.doesIndexing()) {
+ return;
+ }
+ if (isTensorTypeThatSupportsHnswIndex(field)) {
+ if (validate && !field.doesAttributing()) {
+ fail(search, field, "A tensor that has an index must also be an attribute.");
+ }
+ var index = field.getIndex(field.getName());
+ // TODO: Calculate default params based on tensor dimension size
+ var params = new HnswIndexParams();
+ if (index != null) {
+ params = params.overrideFrom(index.getHnswIndexParams());
+ field.getAttribute().setDistanceMetric(index.getDistanceMetric());
+ }
+ field.getAttribute().setHnswIndexParams(params);
+ }
+ }
+
private void validateDataTypeForCollectionField(SDField field) {
if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType)
fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
index d0a0bbfb748..d6398bc348c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -61,7 +61,7 @@ public class UriHack extends Processor {
String partName = uriName + "." + suffix;
// I wonder if this is explicit in qrs or implicit in backend?
// search.addFieldSetItem(uriName, partName);
- SDField partField = new SDField(partName, generatedType, true);
+ SDField partField = new SDField(partName, generatedType);
partField.setIndexStructureField(uriField.doesIndexing());
partField.setRankType(uriField.getRankType());
partField.setStemming(Stemming.NONE);
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 02d500931d7..e0d015cc8b2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -62,6 +62,7 @@ public class DocumentManager {
}
}
}
+
private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
for (AnnotationType type : types) {
DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
@@ -110,6 +111,7 @@ public class DocumentManager {
doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
}
buildConfig(dt.getFieldSets(), doc);
+ buildImportedFieldsConfig(dt.getImportedFieldNames(), doc);
} else if (type instanceof TemporaryStructuredDataType) {
//Ignored
} else if (type instanceof StructDataType) {
@@ -164,4 +166,12 @@ public class DocumentManager {
doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
}
+ private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) {
+ for (String fieldName : fieldNames) {
+ var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder();
+ ib.name(fieldName);
+ builder.importedfield(ib);
+ }
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index d4228b52746..e6bf826dccc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -58,6 +58,7 @@ public class DocumentTypes {
buildConfig(annotation, atb);
}
buildConfig(documentType.getFieldSets(), db);
+ buildImportedFieldsConfig(documentType.getImportedFieldNames(), db);
builder.documenttype(db);
}
@@ -120,6 +121,14 @@ public class DocumentTypes {
}
}
+ private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) {
+ for (String fieldName : fieldNames) {
+ var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder();
+ ib.name(fieldName);
+ builder.importedfield(ib);
+ }
+ }
+
private void buildConfig(StructDataType type,
DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
DocumenttypesConfig.Documenttype.Builder documentBuilder,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index 72c475fb091..75b33e184d9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -461,6 +461,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
public FileReference sendFile(String relativePath) {
return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost());
}
+
public FileReference sendUri(String uri) {
return getRoot().getFileDistributor().sendUriToHost(uri, getHost());
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
index bf86bc4a453..48cd378a9a6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
@@ -1,16 +1,14 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-import java.util.Map;
-
import com.yahoo.config.ConfigInstance;
import com.yahoo.config.ConfigInstance.Builder;
import com.yahoo.config.model.producer.UserConfigRepo;
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Map;
+
/**
* Interface that should be implemented by all config producing modules
* in the vespa model.
@@ -35,17 +33,6 @@ public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer
List<Service> getDescendantServices();
/**
- * Writes files that need to be written. The files will usually
- * only be written when the Vespa model is generated through the
- * deploy-application script.
- * This is primarily intended for debugging.
- *
- * @param directory directory to write files to
- * @throws java.io.IOException if writing fails
- */
- void writeFiles(File directory) throws IOException;
-
- /**
* Dump the three of config producers to the specified stream.
*
* @param out The stream to print to, e.g. System.out
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
index a952d63526b..db469c2091e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
@@ -1,12 +1,11 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model;
-import java.io.File;
-import java.util.Objects;
-
import com.yahoo.cloud.config.SentinelConfig;
import com.yahoo.config.model.producer.AbstractConfigProducer;
+import java.util.Objects;
+
/**
* A physical host, running a set of services.
* The identity of a host is its hostname. Hosts are comparable on their host name.
@@ -67,10 +66,6 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?>
}
@Override
- public void writeFiles(File directory) {
- }
-
- @Override
public void getConfig(SentinelConfig.Builder builder) {
// TODO (MAJOR_RELEASE): This shouldn't really be here, but we need to make sure users can upgrade if we change sentinel to use hosts/<hostname>/sentinel instead of hosts/<hostname>
// as config id. We should probably wait for a major release
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
index eda562bea5a..557a61ec211 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
@@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.ProvisionLogger;
import java.net.UnknownHostException;
-import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -34,13 +33,11 @@ public class HostSystem extends AbstractConfigProducer<Host> {
private static Logger log = Logger.getLogger(HostSystem.class.getName());
- private Map<String,String> hostnames = new LinkedHashMap<>();
-
private final Map<String, HostResource> hostname2host = new LinkedHashMap<>();
private final HostProvisioner provisioner;
private final DeployLogger deployLogger;
- public HostSystem(AbstractConfigProducer parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) {
+ public HostSystem(AbstractConfigProducer<?> parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) {
super(parent, name);
this.provisioner = provisioner;
this.deployLogger = deployLogger;
@@ -49,7 +46,8 @@ public class HostSystem extends AbstractConfigProducer<Host> {
void checkName(String hostname) {
// Give a warning if the host does not exist
try {
- Object address = java.net.InetAddress.getByName(hostname);
+ @SuppressWarnings("unused")
+ Object ignore = java.net.InetAddress.getByName(hostname);
} catch (UnknownHostException e) {
deployLogger.log(Level.WARNING, "Unable to lookup IP address of host: " + hostname);
}
@@ -78,36 +76,10 @@ public class HostSystem extends AbstractConfigProducer<Host> {
return hostname2host.get(name);
}
- /**
- * Returns the canonical name of a given host. This will cache names for faster lookup.
- *
- * @param hostname the hostname to retrieve the canonical hostname for.
- * @return The canonical hostname, or null if unable to resolve.
- * @throws UnknownHostException if the hostname cannot be resolved
- */
- public String getCanonicalHostname(String hostname) throws UnknownHostException {
- if ( ! hostnames.containsKey(hostname)) {
- hostnames.put(hostname, lookupCanonicalHostname(hostname));
- }
- return hostnames.get(hostname);
- }
-
- /**
- * Static helper method that looks up the canonical name of a given host.
- *
- * @param hostname the hostname to retrieve the canonical hostname for.
- * @return The canonical hostname, or null if unable to resolve.
- * @throws UnknownHostException if the hostname cannot be resolved
- */
- // public - This is used by amenders outside this repo
- public static String lookupCanonicalHostname(String hostname) throws UnknownHostException {
- return java.net.InetAddress.getByName(hostname).getCanonicalHostName();
- }
-
@Override
public String toString() {
return "hosts [" + hostname2host.values().stream()
- .map(host -> host.getHostname())
+ .map(HostResource::getHostname)
.collect(Collectors.joining(", ")) +
"]";
}
@@ -139,8 +111,8 @@ public class HostSystem extends AbstractConfigProducer<Host> {
}
}
- public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) {
- List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger));
+ public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) {
+ List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger));
// TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership.
Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>();
for (HostSpec spec : allocatedHosts) {
@@ -169,7 +141,7 @@ public class HostSystem extends AbstractConfigProducer<Host> {
}
Set<HostSpec> getHostSpecs() {
- return getHosts().stream().map(host -> host.spec()).collect(Collectors.toCollection(LinkedHashSet::new));
+ return getHosts().stream().map(HostResource::spec).collect(Collectors.toCollection(LinkedHashSet::new));
}
/** A provision logger which forwards to a deploy logger */
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 2cfd2e23d08..6f2123f248c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -23,11 +23,13 @@ import com.yahoo.config.model.api.FileDistribution;
import com.yahoo.config.model.api.HostInfo;
import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
import com.yahoo.config.model.producer.UserConfigRepo;
import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.log.LogLevel;
import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
@@ -61,7 +63,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil;
import com.yahoo.yolean.Exceptions;
import org.xml.sax.SAXException;
-import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Constructor;
@@ -123,6 +124,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
private final FileDistributor fileDistributor;
+ private final Provisioned provisioned;
+
/** Creates a Vespa Model from internal model types only */
public VespaModel(ApplicationPackage app) throws IOException, SAXException {
this(app, new NullConfigModelRegistry());
@@ -163,6 +166,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
configModelRegistry = new VespaConfigModelRegistry(configModelRegistry);
VespaModelBuilder builder = new VespaDomBuilder();
this.applicationPackage = deployState.getApplicationPackage();
+ this.provisioned = deployState.provisioned();
root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this);
createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(),
@@ -205,7 +209,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
/** Creates a mutable model with no services instantiated */
public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException {
- return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry(), null));
+ return new VespaModel(new NullConfigModelRegistry(), deployState, false,
+ new FileDistributor(deployState.getFileRegistry(), List.of(), deployState.isHosted()));
}
private void validateWrapExceptions() {
@@ -564,23 +569,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
id2producer.put(configId, descendant);
}
- /**
- * Writes MODEL.cfg files for all config producers.
- *
- * @param baseDirectory dir to write files to
- */
- public void writeFiles(File baseDirectory) throws IOException {
- super.writeFiles(baseDirectory);
- for (ConfigProducer cp : id2producer.values()) {
- try {
- File destination = new File(baseDirectory, cp.getConfigId().replace("/", File.separator));
- cp.writeFiles(destination);
- } catch (IOException e) {
- throw new IOException(cp.getConfigId() + ": " + e.getMessage());
- }
- }
- }
-
public Clients getClients() {
return configModelRepo.getClients();
}
@@ -629,11 +617,23 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
return Collections.unmodifiableMap(id2producer);
}
- /**
- * Returns this root's model repository
- */
+ /** Returns this root's model repository */
public ConfigModelRepo configModelRepo() {
return configModelRepo;
}
+ /** If provisioning through the node repo, returns the provision requests issued during build of this */
+ public Provisioned provisioned() { return provisioned; }
+
+ /** Returns the id of all clusters in this */
+ public Set<ClusterSpec.Id> allClusters() {
+ return hostSystem().getHosts().stream()
+ .map(HostResource::spec)
+ .filter(spec -> spec.membership().isPresent())
+ .map(spec -> spec.membership().get().cluster().id())
+ .collect(Collectors.toSet());
+ }
+
+
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
index 54807697da4..631f4dab1a7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
@@ -146,11 +146,13 @@ public class VespaModelFactory implements ModelFactory {
.properties(modelContext.properties())
.vespaVersion(version())
.modelHostProvisioner(createHostProvisioner(modelContext))
+ .provisioned(modelContext.provisioned())
.endpoints(modelContext.properties().endpoints())
.modelImporters(modelImporters)
.zone(zone)
.now(clock.instant())
- .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion());
+ .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion())
+ .wantedDockerImageRepo(modelContext.wantedDockerImageRepository());
modelContext.previousModel().ifPresent(builder::previousModel);
return builder.build(validationParameters);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
index 5714d41ef67..24e88d7ef7d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
@@ -4,13 +4,14 @@ package com.yahoo.vespa.model.admin;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.search.config.QrStartConfig;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.component.Handler;
/**
* @author hmusum
*/
-public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> implements ThreadpoolConfig.Producer {
+public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> {
public LogserverContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) {
super(parent, name, name, deployState);
@@ -27,6 +28,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain
builder.maxthreads(10);
}
+ @Override
+ public void getConfig(QrStartConfig.Builder builder) {
+ super.getConfig(builder);
+ builder.jvm.heapsize(384);
+ }
+
protected boolean messageBusEnabled() { return false; }
private void addLogHandler() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
index 2de4e5f5950..41d9df414ea 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
@@ -11,8 +11,7 @@ import com.yahoo.vespa.model.container.ContainerCluster;
*
* @author gjoranv
*/
-public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> implements
- ThreadpoolConfig.Producer
+public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer>
{
public ClusterControllerContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) {
super(parent, subId, name, deployState);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
index fd924eb2a0f..fccacc3210d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
@@ -2,6 +2,8 @@
package com.yahoo.vespa.model.admin.metricsproxy;
+import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler;
+import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig;
import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
@@ -20,6 +22,7 @@ import java.util.Map;
import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.createMetricsHandler;
/**
* Container running a metrics proxy.
@@ -28,9 +31,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus
*/
public class MetricsProxyContainer extends Container implements
NodeDimensionsConfig.Producer,
+ NodeInfoConfig.Producer,
RpcConnectorConfig.Producer,
VespaServicesConfig.Producer
{
+ public static final int BASEPORT = 19092;
final boolean isHostedVespa;
@@ -46,6 +51,7 @@ public class MetricsProxyContainer extends Container implements
addMetricsProxyComponent(NodeDimensions.class);
addMetricsProxyComponent(RpcConnector.class);
addMetricsProxyComponent(VespaServices.class);
+ addHandler(createMetricsHandler(MetricsV2Handler.class, MetricsV2Handler.V2_PATH));
}
@Override
@@ -53,8 +59,6 @@ public class MetricsProxyContainer extends Container implements
return METRICS_PROXY_CONTAINER;
}
- static public int BASEPORT = 19092;
-
@Override
public int getWantedPort() {
return BASEPORT;
@@ -121,7 +125,21 @@ public class MetricsProxyContainer extends Container implements
}
}
- private void addMetricsProxyComponent(Class<?> componentClass) {
+ @Override
+ public void getConfig(NodeInfoConfig.Builder builder) {
+ builder.role(getNodeRole())
+ .hostname(getHostName());
+ }
+
+ private String getNodeRole() {
+ String hostConfigId = getHost().getConfigId();
+ if (! isHostedVespa) return hostConfigId;
+ return getHostResource().spec().membership()
+ .map(ClusterMembership::stringValue)
+ .orElse(hostConfigId);
+ }
+
+ private void addMetricsProxyComponent(Class<?> componentClass) {
addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index fcc6c3279de..20f2bfe6636 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -20,6 +20,9 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
import ai.vespa.metricsproxy.rpc.RpcServer;
import ai.vespa.metricsproxy.service.ConfigSentinelClient;
import ai.vespa.metricsproxy.service.SystemPollerProvider;
+import ai.vespa.metricsproxy.telegraf.Telegraf;
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import ai.vespa.metricsproxy.telegraf.TelegrafRegistry;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
@@ -67,7 +70,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
ApplicationDimensionsConfig.Producer,
ConsumersConfig.Producer,
MonitoringConfig.Producer,
- ThreadpoolConfig.Producer,
+ TelegrafConfig.Producer,
MetricsNodesConfig.Producer
{
public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName());
@@ -116,14 +119,30 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH);
addMetricsProxyComponent(ApplicationMetricsRetriever.class);
+
+ addTelegrafComponents();
}
private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) {
+ Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath);
+ addComponent(metricsHandler);
+ }
+
+ static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) {
Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>(
new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null));
metricsHandler.addServerBindings("http://*" + bindingPath,
"http://*" + bindingPath + "/*");
- addComponent(metricsHandler);
+ return metricsHandler;
+ }
+
+ private void addTelegrafComponents() {
+ getAdmin().ifPresent(admin -> {
+ if (admin.getUserMetrics().usesExternalMetricSystems()) {
+ addMetricsProxyComponent(Telegraf.class);
+ addMetricsProxyComponent(TelegrafRegistry.class);
+ }
+ });
}
@Override
@@ -156,6 +175,32 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
}
@Override
+ public void getConfig(TelegrafConfig.Builder builder) {
+ builder.isHostedVespa(isHostedVespa());
+
+ var userConsumers = getUserMetricsConsumers();
+ for (var consumer : userConsumers.values()) {
+ for (var cloudWatch : consumer.cloudWatches()) {
+ var cloudWatchBuilder = new TelegrafConfig.CloudWatch.Builder();
+ cloudWatchBuilder
+ .region(cloudWatch.region())
+ .namespace(cloudWatch.namespace())
+ .consumer(cloudWatch.consumer());
+
+ cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder
+ .accessKeyName(hostedAuth.accessKeyName)
+ .secretKeyName(hostedAuth.secretKeyName));
+
+ cloudWatch.sharedCredentials().ifPresent(sharedCredentials -> {
+ cloudWatchBuilder.file(sharedCredentials.file);
+ sharedCredentials.profile.ifPresent(cloudWatchBuilder::profile);
+ });
+ builder.cloudWatch(cloudWatchBuilder);
+ }
+ }
+ }
+
+ @Override
public void getConfig(ThreadpoolConfig.Builder builder) {
builder.maxthreads(10);
}
@@ -198,7 +243,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
Optional.of(monitoring.getInterval()) : Optional.empty();
}
- private void addMetricsProxyComponent(Class<?> componentClass) {
+ private void addMetricsProxyComponent(Class<?> componentClass) {
addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
new file mode 100644
index 00000000000..5351c3fb3a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
@@ -0,0 +1,59 @@
+package com.yahoo.vespa.model.admin.monitoring;
+
+import java.util.Optional;
+
+/**
+ * Helper object for CloudWatch configuration.
+ *
+ * @author gjoranv
+ */
+public class CloudWatch {
+ private final String region;
+ private final String namespace;
+ private final MetricsConsumer consumer;
+
+ private HostedAuth hostedAuth;
+ private SharedCredentials sharedCredentials;
+
+ public CloudWatch(String region, String namespace, MetricsConsumer consumer) {
+ this.region = region;
+ this.namespace = namespace;
+ this.consumer = consumer;
+ }
+
+ public String region() { return region; }
+ public String namespace() { return namespace; }
+ public String consumer() { return consumer.getId(); }
+
+ public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); }
+ public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); }
+
+ public void setHostedAuth(String accessKeyName, String secretKeyName) {
+ hostedAuth = new HostedAuth(accessKeyName, secretKeyName);
+ }
+
+ public void setSharedCredentials(String file, Optional<String> profile) {
+ sharedCredentials = new SharedCredentials(file, profile);
+ }
+
+ public static class HostedAuth {
+ public final String accessKeyName;
+ public final String secretKeyName;
+
+ HostedAuth(String accessKeyName, String secretKeyName) {
+ this.accessKeyName = accessKeyName;
+ this.secretKeyName = secretKeyName;
+ }
+ }
+
+ public static class SharedCredentials {
+ public final String file;
+ public final Optional<String> profile;
+
+ SharedCredentials(String file, Optional<String> profile) {
+ this.file = file;
+ this.profile = profile;
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java
index 19a62f47e39..c80cebe3d5b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java
@@ -20,10 +20,12 @@ import static java.util.Collections.singleton;
*/
public class DefaultPublicMetrics {
+ public static final String DEFAULT_METRIC_SET_ID = "default";
+
public static MetricSet defaultPublicMetricSet = createMetricSet();
private static MetricSet createMetricSet() {
- return new MetricSet("public",
+ return new MetricSet(DEFAULT_METRIC_SET_ID,
getAllMetrics(),
singleton(defaultVespaMetricSet));
}
@@ -84,6 +86,7 @@ public class DefaultPublicMetrics {
metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate"));
metrics.add(new Metric("content.proton.documentdb.matching.docs_reranked.rate"));
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
index 529ed6ecf67..a8fbcf50b02 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
@@ -2,9 +2,13 @@
package com.yahoo.vespa.model.admin.monitoring;
import javax.annotation.concurrent.Immutable;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
+import static java.util.Collections.unmodifiableList;
+
/**
* Represents an arbitrary metric consumer
*
@@ -13,12 +17,15 @@ import java.util.Objects;
*/
@Immutable
public class MetricsConsumer {
+
private final String id;
private final MetricSet metricSet;
+ private final List<CloudWatch> cloudWatches = new ArrayList<>();
+
/**
- * @param id The consumer
- * @param metricSet The metrics for this consumer
+ * @param id the consumer
+ * @param metricSet the metrics for this consumer
*/
public MetricsConsumer(String id, MetricSet metricSet) {
this.id = Objects.requireNonNull(id, "A consumer must have a non-null id.");;
@@ -38,4 +45,12 @@ public class MetricsConsumer {
return metricSet.getMetrics();
}
+ public void addCloudWatch(CloudWatch cloudWatch) {
+ cloudWatches.add(cloudWatch);
+ }
+
+ public List<CloudWatch> cloudWatches() {
+ return unmodifiableList(cloudWatches);
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
index 58b77ee1297..c05cad89852 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
@@ -9,6 +9,7 @@ import java.util.Set;
* @author gjoranv
*/
public class SystemMetrics {
+
public static final String CPU_UTIL = "cpu.util";
public static final String CPU_SYS_UTIL = "cpu.sys.util";
public static final String CPU_THROTTLED_TIME = "cpu.throttled_time.rate";
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 57d938f8a71..141794256b8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -72,6 +72,9 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.server.network.tls-connections-broken"));
metrics.add(new Metric("vds.server.network.failed-tls-config-reloads"));
+ // C++ Fnet metrics
+ metrics.add(new Metric("vds.server.fnet.num-connections"));
+
return metrics;
}
@@ -126,6 +129,8 @@ public class VespaMetricSet {
metrics.add(new Metric("serverActiveThreads.count"));
metrics.add(new Metric("serverActiveThreads.last"));
+ metrics.add(new Metric("jdisc.thread_pool.unhandled_exceptions.rate"));
+
metrics.add(new Metric("httpapi_latency.max"));
metrics.add(new Metric("httpapi_latency.sum"));
metrics.add(new Metric("httpapi_latency.count"));
@@ -179,6 +184,15 @@ public class VespaMetricSet {
metrics.add(new Metric("jdisc.http.request.content_size.sum"));
metrics.add(new Metric("jdisc.http.request.content_size.count"));
metrics.add(new Metric("jdisc.http.request.content_size.average")); // TODO: Remove in Vespa 8
+
+ metrics.add(new Metric("jdisc.http.ssl.handshake.failure.missing_client_cert.rate"));
+ metrics.add(new Metric("jdisc.http.ssl.handshake.failure.expired_client_cert.rate"));
+ metrics.add(new Metric("jdisc.http.ssl.handshake.failure.invalid_client_cert.rate"));
+ metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_protocols.rate"));
+ metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_ciphers.rate"));
+ metrics.add(new Metric("jdisc.http.ssl.handshake.failure.unknown.rate"));
+
+ metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate"));
return metrics;
}
@@ -288,6 +302,14 @@ public class VespaMetricSet {
return metrics;
}
+ private static void addSearchNodeExecutorMetrics(Set<Metric> metrics, String prefix) {
+ metrics.add(new Metric(prefix + ".queuesize.max"));
+ metrics.add(new Metric(prefix + ".queuesize.sum"));
+ metrics.add(new Metric(prefix + ".queuesize.count"));
+ metrics.add(new Metric(prefix + ".maxpending.last")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric(prefix + ".accepted.rate"));
+ }
+
private static Set<Metric> getSearchNodeMetrics() {
Set<Metric> metrics = new LinkedHashSet<>();
@@ -332,18 +354,12 @@ public class VespaMetricSet {
metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count"));
// Executors shared between all document dbs
- metrics.add(new Metric("content.proton.executor.proton.maxpending.last"));
- metrics.add(new Metric("content.proton.executor.proton.accepted.rate"));
- metrics.add(new Metric("content.proton.executor.flush.maxpending.last"));
- metrics.add(new Metric("content.proton.executor.flush.accepted.rate"));
- metrics.add(new Metric("content.proton.executor.match.maxpending.last"));
- metrics.add(new Metric("content.proton.executor.match.accepted.rate"));
- metrics.add(new Metric("content.proton.executor.docsum.maxpending.last"));
- metrics.add(new Metric("content.proton.executor.docsum.accepted.rate"));
- metrics.add(new Metric("content.proton.executor.shared.maxpending.last"));
- metrics.add(new Metric("content.proton.executor.shared.accepted.rate"));
- metrics.add(new Metric("content.proton.executor.warmup.maxpending.last"));
- metrics.add(new Metric("content.proton.executor.warmup.accepted.rate"));
+ addSearchNodeExecutorMetrics(metrics, "content.proton.executor.proton");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.executor.flush");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.executor.match");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.executor.docsum");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.executor.shared");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.executor.warmup");
// jobs
metrics.add(new Metric("content.proton.documentdb.job.total.average"));
@@ -357,18 +373,12 @@ public class VespaMetricSet {
metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average"));
// Threading service (per document db)
- metrics.add(new Metric("content.proton.documentdb.threading_service.master.maxpending.last"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.master.accepted.rate"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.index.maxpending.last"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.index.accepted.rate"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.summary.maxpending.last"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.summary.accepted.rate"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.maxpending.last"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.accepted.rate"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.maxpending.last"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.accepted.rate"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.maxpending.last"));
- metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.accepted.rate"));
+ addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.master");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.summary");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_inverter");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_writer");
+ addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.attribute_field_writer");
// lid space
metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average"));
@@ -453,10 +463,13 @@ public class VespaMetricSet {
metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum"));
metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count"));
metrics.add(new Metric("content.proton.documentdb.matching.query_latency.average")); // TODO: Remove in Vespa 8
- metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max"));
- metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum"));
- metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count"));
+ metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); // TODO: Remove in Vespa 8
metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max"));
+ metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum"));
+ metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count"));
metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate"));
@@ -468,10 +481,13 @@ public class VespaMetricSet {
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove in Vespa 8
- metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max"));
- metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum"));
- metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count"));
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); // TODO: Remove in Vespa 8
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max"));
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum"));
+ metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum"));
metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count"));
@@ -513,13 +529,24 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.sum"));
metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.count"));
metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.max"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.sum"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.count"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.max"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.sum"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.count"));
metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max"));
metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum"));
metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count"));
metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8
- metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average"));
+ metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate"));
metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate"));
+ metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate"));
metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max"));
metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum"));
metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count"));
@@ -528,19 +555,27 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum"));
metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count"));
metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8
-
+
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count"));
@@ -560,13 +595,13 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8
metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate"));
-
//Distributor
metrics.add(new Metric("vds.idealstate.buckets_rechecking.average"));
metrics.add(new Metric("vds.idealstate.idealstate_diff.average"));
@@ -589,6 +624,8 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate"));
metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate"));
metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average"));
+ metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.count"));
+ metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.rate"));
metrics.add(new Metric("vds.distributor.puts.sum.latency.max"));
metrics.add(new Metric("vds.distributor.puts.sum.latency.sum"));
@@ -596,18 +633,24 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8
metrics.add(new Metric("vds.distributor.puts.sum.ok.rate"));
metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate"));
+ metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate"));
metrics.add(new Metric("vds.distributor.removes.sum.latency.max"));
metrics.add(new Metric("vds.distributor.removes.sum.latency.sum"));
metrics.add(new Metric("vds.distributor.removes.sum.latency.count"));
metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8
metrics.add(new Metric("vds.distributor.removes.sum.ok.rate"));
metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate"));
+ metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate"));
metrics.add(new Metric("vds.distributor.updates.sum.latency.max"));
metrics.add(new Metric("vds.distributor.updates.sum.latency.sum"));
metrics.add(new Metric("vds.distributor.updates.sum.latency.count"));
metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8
metrics.add(new Metric("vds.distributor.updates.sum.ok.rate"));
metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate"));
+ metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate"));
metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate"));
metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate"));
metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate"));
@@ -617,6 +660,7 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8
metrics.add(new Metric("vds.distributor.gets.sum.ok.rate"));
metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate"));
metrics.add(new Metric("vds.distributor.visitor.sum.latency.max"));
metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum"));
metrics.add(new Metric("vds.distributor.visitor.sum.latency.count"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
index 9b0d9dbfadc..1f81f16a80b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
@@ -28,4 +28,13 @@ public class Metrics {
return consumers.keySet().stream()
.anyMatch(existing -> existing.equalsIgnoreCase(id));
}
+
+ /**
+ * Returns true if any of the consumers have specified external metric systems.
+ */
+ public boolean usesExternalMetricSystems() {
+ return consumers.values().stream()
+ .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty());
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
new file mode 100644
index 00000000000..314ef9cc5a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
@@ -0,0 +1,40 @@
+package com.yahoo.vespa.model.admin.monitoring.builder.xml;
+
+import com.yahoo.vespa.model.admin.monitoring.CloudWatch;
+import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
+import org.w3c.dom.Element;
+
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute;
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild;
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchBuilder {
+
+ private static final String REGION_ATTRIBUTE = "region";
+ private static final String NAMESPACE_ATTRIBUTE = "namespace";
+ private static final String CREDENTIALS_ELEMENT = "credentials";
+ private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name";
+ private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name";
+ private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials";
+ private static final String PROFILE_ATTRIBUTE = "profile";
+ private static final String FILE_ATTRIBUTE = "file";
+
+ public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) {
+ CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE),
+ cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE),
+ consumer);
+
+ getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT)
+ .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE),
+ elem.getAttribute(SECRET_KEY_ATTRIBUTE)));
+
+ getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT)
+ .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(FILE_ATTRIBUTE),
+ getOptionalAttribute(elem, PROFILE_ATTRIBUTE)));
+
+ return cloudWatch;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
index b13fa4917e4..b686288868f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
@@ -42,7 +42,11 @@ public class MetricsBuilder {
throwIfIllegalConsumerId(metrics, consumerId);
MetricSet metricSet = buildMetricSet(consumerId, consumerElement);
- metrics.addConsumer(new MetricsConsumer(consumerId, metricSet));
+ var consumer = new MetricsConsumer(consumerId, metricSet);
+ for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) {
+ consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer));
+ }
+ metrics.addConsumer(consumer);
}
return metrics;
}
@@ -58,11 +62,11 @@ public class MetricsBuilder {
private MetricSet buildMetricSet(String consumerId, Element consumerElement) {
List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream()
- .map(metricElement -> metricFromElement(metricElement))
+ .map(MetricsBuilder::metricFromElement)
.collect(Collectors.toCollection(LinkedList::new));
List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream()
- .map(metricSetElement -> availableMetricSets.get(metricSetElement.getAttribute(ID_ATTRIBUTE)))
+ .map(metricSetElement -> getMetricSetOrThrow(metricSetElement.getAttribute(ID_ATTRIBUTE)))
.collect(Collectors.toCollection(LinkedList::new));
metricSets.add(defaultVespaMetricSet);
@@ -75,6 +79,11 @@ public class MetricsBuilder {
return "user-metrics-" + consumerName;
}
+ private MetricSet getMetricSetOrThrow(String id) {
+ if (! availableMetricSets.containsKey(id)) throw new IllegalArgumentException("No such metric-set: " + id);
+ return availableMetricSets.get(id);
+ }
+
private void throwIfIllegalConsumerId(Metrics metrics, String consumerId) {
if (consumerId.equalsIgnoreCase(VESPA_CONSUMER_ID) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE)
throw new IllegalArgumentException("'Vespa' is not allowed as metrics consumer id (case is ignored.)");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java
new file mode 100644
index 00000000000..631ab0e2640
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java
@@ -0,0 +1,48 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.collections.CollectionUtil.mkString;
+import static com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator.needsAccessControlValidation;
+import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection;
+
+/**
+ * @author gjoranv
+ */
+public class AwsAccessControlValidator extends Validator {
+
+ // NOTE: must be the same as the name in the declaration of the AWS cloud in hosted.
+ static final String AWS_CLOUD_NAME = "aws";
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+
+ if (! needsAccessControlValidation(model, deployState)) return;
+ if(! deployState.zone().cloud().equals(CloudName.from(AWS_CLOUD_NAME))) return;
+
+ List<String> offendingClusters = new ArrayList<>();
+ for (var cluster : model.getContainerClusters().values()) {
+ var http = cluster.getHttp();
+ if (http == null
+ || ! http.getAccessControl().isPresent()
+ || ! http.getAccessControl().get().writeEnabled
+ || ! http.getAccessControl().get().readEnabled)
+
+ if (hasHandlerThatNeedsProtection(cluster) || ! cluster.getAllServlets().isEmpty())
+ offendingClusters.add(cluster.getName());
+ }
+ if (! offendingClusters.isEmpty())
+ deployState.validationOverrides()
+ .invalid(ValidationId.accessControl,
+ "Access-control must be enabled for read/write operations to container clusters in AWS production zones: " +
+ mkString(offendingClusters, "[", ", ", "]"), deployState.now());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java
new file mode 100644
index 00000000000..462ac39fa84
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java
@@ -0,0 +1,37 @@
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
+
+import java.util.List;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ if (!deployState.isHosted()) return;
+ if (deployState.zone().system().isPublic()) return;
+ if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return;
+
+ var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream()
+ .filter(consumer -> !consumer.cloudWatches().isEmpty())
+ .collect(toList());
+
+ if (! offendingConsumers.isEmpty()) {
+ throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " +
+ "be removed for consumers: " + consumerIds(offendingConsumers));
+ }
+ }
+
+ private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) {
+ return offendingConsumers.stream().map(MetricsConsumer::getId).collect(toList());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
index f9762ce58fa..43c1a88b0a1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
@@ -33,7 +33,7 @@ public class ComplexAttributeFieldsValidator extends Validator {
continue;
}
SearchCluster searchCluster = (SearchCluster) cluster;
- for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) {
+ for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) {
validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch());
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java
index ac38336a405..d5bea2a2959 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation;
import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.InstanceName;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ContainerModel;
@@ -12,8 +13,7 @@ import java.util.List;
import java.util.Optional;
/**
- * Validates that deployment spec (deployment.xml) has valid values (for now
- * only global-service-id is validated)
+ * Validate deployment spec (deployment.xml).
*
* @author hmusum
* @author bratseth
@@ -30,11 +30,20 @@ public class DeploymentSpecValidator extends Validator {
List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class);
for (DeploymentInstanceSpec instance : deploymentSpec.instances()) {
instance.globalServiceId().ifPresent(globalServiceId -> {
- if ( containers.stream().noneMatch(container -> container.getCluster().getName().equals(globalServiceId)))
- throw new IllegalArgumentException("The global-service-id in " + instance + ", '" + globalServiceId +
- "' specified in deployment.xml does not match any container cluster id");
+ requireClusterId(containers, instance.name(), "Attribute 'globalServiceId'", globalServiceId);
+ });
+ instance.endpoints().forEach(endpoint -> {
+ requireClusterId(containers, instance.name(), "Endpoint '" + endpoint.endpointId() + "'",
+ endpoint.containerId());
});
}
}
+ private static void requireClusterId(List<ContainerModel> containers, InstanceName instanceName, String context,
+ String id) {
+ if (containers.stream().noneMatch(container -> container.getCluster().getName().equals(id)))
+ throw new IllegalArgumentException(context + " in instance " + instanceName + ": '" + id +
+ "' specified in deployment.xml does not match any container cluster ID");
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
index f00ad0f0dbb..4b8bbd4ff08 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
@@ -12,7 +12,7 @@ public class EndpointCertificateSecretsValidator extends Validator {
@Override
public void validate(VespaModel model, DeployState deployState) {
if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) {
- throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet");
+ throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm());
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
index 7f8ff6edd85..b6f7ab4ff62 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -33,24 +33,29 @@ import java.util.logging.Logger;
/**
* Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done
- * by running through the binary 'vespa-verify-ranksetup'
+ * by running the binary 'vespa-verify-ranksetup-bin'
*
* @author vegardh
*/
public class RankSetupValidator extends Validator {
private static final Logger log = Logger.getLogger(RankSetupValidator.class.getName());
- private final boolean force;
+ private static final String binaryName = "vespa-verify-ranksetup-bin ";
- public RankSetupValidator(boolean force) {
- this.force = force;
+ private final boolean ignoreValidationErrors;
+
+ public RankSetupValidator(boolean ignoreValidationErrors) {
+ this.ignoreValidationErrors = ignoreValidationErrors;
}
@Override
public void validate(VespaModel model, DeployState deployState) {
File cfgDir = null;
try {
- cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile();
+ cfgDir = Files.createTempDirectory("verify-ranksetup." +
+ deployState.getProperties().applicationId().toFullString() +
+ ".")
+ .toFile();
for (AbstractSearchCluster cluster : model.getSearchClusters()) {
// Skipping rank expression checking for streaming clusters, not implemented yet
@@ -100,29 +105,29 @@ public class RankSetupValidator extends Validator {
IOUtils.recursiveDeleteDir(dir);
}
- private void writeConfigs(String dir, AbstractConfigProducer producer) throws IOException {
+ private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException {
RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder();
- RankProfilesConfig.Producer.class.cast(producer).getConfig(rpcb);
+ ((RankProfilesConfig.Producer) producer).getConfig(rpcb);
RankProfilesConfig rpc = new RankProfilesConfig(rpcb);
writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc);
IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder();
- IndexschemaConfig.Producer.class.cast(producer).getConfig(iscb);
+ ((IndexschemaConfig.Producer) producer).getConfig(iscb);
IndexschemaConfig isc = new IndexschemaConfig(iscb);
writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc);
AttributesConfig.Builder acb = new AttributesConfig.Builder();
- AttributesConfig.Producer.class.cast(producer).getConfig(acb);
+ ((AttributesConfig.Producer) producer).getConfig(acb);
AttributesConfig ac = new AttributesConfig(acb);
writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac);
RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder();
- RankingConstantsConfig.Producer.class.cast(producer).getConfig(rccb);
+ ((RankingConstantsConfig.Producer) producer).getConfig(rccb);
RankingConstantsConfig rcc = new RankingConstantsConfig(rccb);
writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc);
ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder();
- ImportedFieldsConfig.Producer.class.cast(producer).getConfig(ifcb);
+ ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb);
ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb);
writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc);
}
@@ -132,8 +137,8 @@ public class RankSetupValidator extends Validator {
}
private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) {
- String job = "vespa-verify-ranksetup-bin " + configId;
- ProcessExecuter executer = new ProcessExecuter();
+ String job = String.format("%s %s", binaryName, configId);
+ ProcessExecuter executer = new ProcessExecuter(true);
try {
Pair<Integer, String> ret = executer.exec(job);
if (ret.getFirst() != 0) {
@@ -147,27 +152,28 @@ public class RankSetupValidator extends Validator {
}
private void validateWarn(Exception e, DeployLogger deployLogger) {
- String msg = "Unable to execute 'vespa-verify-ranksetup', validation of rank expressions will only take place when you start Vespa: " +
+ String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " +
Exceptions.toMessageString(e);
deployLogger.log(LogLevel.WARNING, msg);
}
private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) {
- String errMsg = "For search cluster '" + sc.getClusterName() + "', search definition '" + sdName + "': error in rank setup. Details:\n";
+ StringBuilder errMsg = new StringBuilder("For search cluster '").append(sc.getClusterName()).append("', ")
+ .append("search definition '").append(sdName).append("': error in rank setup. Details:\n");
for (String line : output.split("\n")) {
// Remove debug lines from start script
if (line.startsWith("debug\t")) continue;
try {
- LogMessage logmsg = LogMessage.parseNativeFormat(line);
- errMsg = errMsg + logmsg.getLevel() + ": " + logmsg.getPayload() + "\n";
+ LogMessage logMessage = LogMessage.parseNativeFormat(line);
+ errMsg.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n");
} catch (InvalidLogFormatException e) {
- errMsg = errMsg + line + "\n";
+ errMsg.append(line).append("\n");
}
}
- if (force) {
- deployLogger.log(LogLevel.WARNING, errMsg + "(Continuing because of force.)");
+ if (ignoreValidationErrors) {
+ deployLogger.log(LogLevel.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString());
} else {
- throw new IllegalArgumentException(errMsg);
+ throw new IllegalArgumentException(errMsg.toString());
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
index 907418ea9f0..9568ea5c27c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
@@ -9,7 +9,7 @@ import com.yahoo.path.Path;
import com.yahoo.searchdefinition.RankingConstant;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
import java.io.FileNotFoundException;
@@ -47,7 +47,7 @@ public class RankingConstantsValidator extends Validator {
ApplicationPackage applicationPackage = deployState.getApplicationPackage();
ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):");
- for (SearchDefinition sd : deployState.getSearchDefinitions()) {
+ for (NamedSchema sd : deployState.getSchemas()) {
for (RankingConstant rc : sd.getSearch().rankingConstants().asMap().values()) {
try {
validateRankingConstant(rc, applicationPackage);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index 85ba75639eb..031ce0dbdd4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -15,7 +15,7 @@ import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.search.AbstractSearchCluster;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
import java.util.List;
@@ -34,7 +34,7 @@ public class SearchDataTypeValidator extends Validator {
if (cluster.isStreaming()) {
continue;
}
- for (AbstractSearchCluster.SearchDefinitionSpec spec : cluster.getLocalSDS()) {
+ for (AbstractSearchCluster.SchemaSpec spec : cluster.getLocalSDS()) {
SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument();
if (docType == null) {
continue;
@@ -44,7 +44,7 @@ public class SearchDataTypeValidator extends Validator {
}
}
- private void validateDocument(AbstractSearchCluster cluster, SearchDefinition def, SDDocumentType doc) {
+ private void validateDocument(AbstractSearchCluster cluster, NamedSchema def, SDDocumentType doc) {
for (SDDocumentType child : doc.getTypes()) {
validateDocument(cluster, def, child);
}
@@ -84,7 +84,7 @@ public class SearchDataTypeValidator extends Validator {
}
}
- private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) {
+ private void disallowIndexingOfMaps(AbstractSearchCluster cluster, NamedSchema def, Field field) {
DataType fieldType = field.getDataType();
if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) {
throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " +
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index 1e4a45428b8..22dd0289390 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -59,6 +59,8 @@ public class Validation {
new SecretStoreValidator().validate(model, deployState);
new EndpointCertificateSecretsValidator().validate(model, deployState);
new AccessControlFilterValidator().validate(model, deployState);
+ new CloudWatchValidator().validate(model, deployState);
+ new AwsAccessControlValidator().validate(model, deployState);
List<ConfigChangeAction> result = Collections.emptyList();
if (deployState.getProperties().isFirstTimeDeployment()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
index d14fe91a53b..162f6798462 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
@@ -2,6 +2,8 @@
package com.yahoo.vespa.model.application.validation.change;
import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
@@ -21,35 +23,32 @@ public class ClusterSizeReductionValidator implements ChangeValidator {
@Override
public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) {
- for (ContainerCluster currentCluster : current.getContainerClusters().values()) {
- ContainerCluster nextCluster = next.getContainerClusters().get(currentCluster.getName());
- if (nextCluster == null) continue;
- validate(currentCluster.getContainers().size(),
- nextCluster.getContainers().size(),
- currentCluster.getName(),
+ for (var clusterId : current.allClusters()) {
+ Capacity currentCapacity = current.provisioned().all().get(clusterId);
+ Capacity nextCapacity = next.provisioned().all().get(clusterId);
+ if (currentCapacity == null || nextCapacity == null) continue;
+ validate(currentCapacity,
+ nextCapacity,
+ clusterId,
overrides,
now);
}
-
- for (ContentCluster currentCluster : current.getContentClusters().values()) {
- ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getName());
- if (nextCluster == null) continue;
- validate(currentCluster.getSearch().getSearchNodes().size(),
- nextCluster.getSearch().getSearchNodes().size(),
- currentCluster.getName(),
- overrides,
- now);
- }
-
return Collections.emptyList();
}
- private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides, Instant now) {
+ private void validate(Capacity current,
+ Capacity next,
+ ClusterSpec.Id clusterId,
+ ValidationOverrides overrides,
+ Instant now) {
+ int currentSize = current.minResources().nodes();
+ int nextSize = next.minResources().nodes();
// don't allow more than 50% reduction, but always allow to reduce size with 1
if ( nextSize < ((double)currentSize) * 0.5 && nextSize != currentSize - 1)
overrides.invalid(ValidationId.clusterSizeReduction,
- "Size reduction in '" + clusterName + "' is too large. Current size: " + currentSize +
- ", new size: " + nextSize + ". New size must be at least 50% of the current size",
+ "Size reduction in '" + clusterId.value() + "' is too large: " +
+ "New min size must be at least 50% of the current min size. " +
+ "Current size: " + currentSize + ", new size: " + nextSize,
now);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
index 8fdcf249bbc..5343a322382 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
@@ -5,6 +5,7 @@ import com.yahoo.collections.Pair;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.vespa.model.HostResource;
@@ -15,6 +16,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -27,35 +29,43 @@ public class ResourcesReductionValidator implements ChangeValidator {
@Override
public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) {
- var currentRequestedResourcesByClusterId = getRequestedResourcesByClusterId(current);
- var nextRequestedResourcesByClusterId = getRequestedResourcesByClusterId(next);
-
- for (var clusterTypeAndId : currentRequestedResourcesByClusterId.keySet()) {
- if (!nextRequestedResourcesByClusterId.containsKey(clusterTypeAndId)) continue;
- validate(currentRequestedResourcesByClusterId.get(clusterTypeAndId),
- nextRequestedResourcesByClusterId.get(clusterTypeAndId),
- clusterTypeAndId.getSecond(),
- overrides,
- now);
+ for (var clusterId : current.allClusters()) {
+ Capacity currentCapacity = current.provisioned().all().get(clusterId);
+ Capacity nextCapacity = next.provisioned().all().get(clusterId);
+ if (currentCapacity == null || nextCapacity == null) continue;
+ validate(currentCapacity, nextCapacity, clusterId, overrides, now);
}
return List.of();
}
- private void validate(NodeResources currentResources, NodeResources nextResources, ClusterSpec.Id clusterId,
- ValidationOverrides overrides, Instant now) {
+ private void validate(Capacity current,
+ Capacity next,
+ ClusterSpec.Id clusterId,
+ ValidationOverrides overrides,
+ Instant now) {
+ if (current.minResources().nodeResources() == NodeResources.unspecified) return;
+ if (next.minResources().nodeResources() == NodeResources.unspecified) return;
+
List<String> illegalChanges = Stream.of(
- validateResource("vCPU", currentResources.vcpu(), nextResources.vcpu()),
- validateResource("memory GB", currentResources.memoryGb(), nextResources.memoryGb()),
- validateResource("disk GB", currentResources.diskGb(), nextResources.diskGb()))
+ validateResource("vCPU",
+ current.minResources().nodeResources().vcpu(),
+ next.minResources().nodeResources().vcpu()),
+ validateResource("memory GB",
+ current.minResources().nodeResources().memoryGb(),
+ next.minResources().nodeResources().memoryGb()),
+ validateResource("disk GB",
+ current.minResources().nodeResources().diskGb(),
+ next.minResources().nodeResources().diskGb()))
.flatMap(Optional::stream)
.collect(Collectors.toList());
if (illegalChanges.isEmpty()) return;
overrides.invalid(ValidationId.resourcesReduction,
- "Resource reduction in '" + clusterId.value() + "' is too large. " +
- String.join(" ", illegalChanges) + " New resources must be at least 50% of the current resources",
- now);
+ "Resource reduction in '" + clusterId.value() + "' is too large. " +
+ String.join(" ", illegalChanges) +
+ " New min resources must be at least 50% of the current min resources",
+ now);
}
private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) {
@@ -64,15 +74,4 @@ public class ResourcesReductionValidator implements ChangeValidator {
return Optional.of(String.format(Locale.ENGLISH ,"Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue));
}
- private static Map<Pair<ClusterSpec.Type, ClusterSpec.Id>, NodeResources> getRequestedResourcesByClusterId(VespaModel vespaModel) {
- return vespaModel.hostSystem().getHosts().stream()
- .map(HostResource::spec)
- .filter(spec -> spec.membership().isPresent() && spec.requestedResources().isPresent())
- .filter(spec -> !spec.membership().get().retired())
- .collect(Collectors.toMap(
- spec -> new Pair<>(spec.membership().get().cluster().type(), spec.membership().get().cluster().id()),
- spec -> spec.requestedResources().get(),
- (e1, e2) -> e1));
- }
-
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java
index 97153e42ee5..df5fe15ca82 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java
@@ -7,16 +7,16 @@ import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.application.validation.Validator;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ContainerCluster;
-import com.yahoo.vespa.model.container.ApplicationContainerCluster;
-import com.yahoo.vespa.model.container.component.Handler;
import java.util.ArrayList;
import java.util.List;
import static com.yahoo.collections.CollectionUtil.mkString;
-import static com.yahoo.vespa.model.container.http.AccessControl.isBuiltinGetOnly;
+import static com.yahoo.config.provision.InstanceName.defaultName;
+import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection;
/**
* Validates that hosted applications in prod zones have write protection enabled.
@@ -28,10 +28,7 @@ public class AccessControlOnFirstDeploymentValidator extends Validator {
@Override
public void validate(VespaModel model, DeployState deployState) {
- if (! deployState.isHosted()) return;
- if (! deployState.zone().environment().isProduction()) return;
- if (deployState.zone().system().isPublic()) return;
- if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return;
+ if (! needsAccessControlValidation(model, deployState)) return;
List<String> offendingClusters = new ArrayList<>();
for (ContainerCluster<? extends Container> c : model.getContainerClusters().values()) {
@@ -44,23 +41,19 @@ public class AccessControlOnFirstDeploymentValidator extends Validator {
if (hasHandlerThatNeedsProtection(cluster) || ! cluster.getAllServlets().isEmpty())
offendingClusters.add(cluster.getName());
}
- if (! offendingClusters.isEmpty()
- && deployState.getApplicationPackage().getApplicationId().instance().equals(InstanceName.defaultName()))
+ if (! offendingClusters.isEmpty())
deployState.validationOverrides().invalid(ValidationId.accessControl,
"Access-control must be enabled for write operations to container clusters in production zones: " +
- mkString(offendingClusters, "[", ", ", "]."), deployState.now());
- }
-
- private boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) {
- return cluster.getHandlers().stream().anyMatch(this::handlerNeedsProtection);
+ mkString(offendingClusters, "[", ", ", "]"), deployState.now());
}
- private boolean handlerNeedsProtection(Handler<?> handler) {
- return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler);
- }
+ public static boolean needsAccessControlValidation(VespaModel model, DeployState deployState) {
+ if (! deployState.isHosted()) return false;
+ if (! deployState.zone().environment().isProduction()) return false;
+ if (deployState.zone().system().isPublic()) return false;
+ if (! deployState.getApplicationPackage().getApplicationId().instance().equals(defaultName())) return false;
+ if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return false;
- private boolean hasNonMbusBinding(Handler<?> handler) {
- return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus"));
+ return true;
}
-
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index a146e2cd7e7..d2f06da992c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -71,7 +71,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted());
Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets)
.buildMetrics(XML.getChild(adminElement, "metrics"));
- FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent);
+ FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, deployState.isHosted());
Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted());
admin.setApplicationType(applicationType);
@@ -81,8 +81,8 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
return admin;
}
- private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent) {
- return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs);
+ private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent, boolean isHosted) {
+ return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs, isHosted);
}
protected abstract void doBuildAdmin(DeployState deployState, Admin admin, Element adminE);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
index 61d0cd7cd1e..804a5442608 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -65,12 +65,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
createSlobroks(deployLogger, admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification));
}
else {
- createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.count(), 2));
+ createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2));
}
}
private void assignLogserver(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) {
- if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server");
+ if (nodesSpecification.minResources().nodes() > 1) throw new IllegalArgumentException("You can only request a single log server");
if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications
if (nodesSpecification.isDedicated()) {
Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification);
@@ -79,7 +79,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts);
createContainerOnLogserverHost(deployState, admin, logserver.getHostResource());
} else if (containerModels.iterator().hasNext()) {
- List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false);
+ List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false);
if (hosts.isEmpty()) return; // No log server can be created (and none is needed)
createLogserver(deployState.getDeployLogger(), admin, hosts);
@@ -91,8 +91,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
private NodesSpecification createNodesSpecificationForLogserver() {
DeployState deployState = context.getDeployState();
if (deployState.getProperties().useDedicatedNodeForLogserver() &&
- context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT &&
- deployState.isHosted())
+ context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT &&
+ deployState.isHosted())
return NodesSpecification.dedicated(1, context);
else
return NodesSpecification.nonDedicated(1, context);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
index f8f1f88e339..11fab0ada29 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.text.XML;
import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.Handler;
import org.w3c.dom.Element;
@@ -14,9 +15,13 @@ import org.w3c.dom.Element;
*/
public class DomClientProviderBuilder extends DomHandlerBuilder {
+ public DomClientProviderBuilder(ApplicationContainerCluster cluster) {
+ super(cluster);
+ }
+
@Override
- protected Handler doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element clientElement) {
- Handler<? super Component<?, ?>> client = getHandler(clientElement);
+ protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element clientElement) {
+ Handler<? super Component<?, ?>> client = createHandler(clientElement);
for (Element binding : XML.getChildren(clientElement, "binding"))
client.addClientBindings(XML.getValue(binding));
@@ -24,7 +29,7 @@ public class DomClientProviderBuilder extends DomHandlerBuilder {
for (Element serverBinding : XML.getChildren(clientElement, "serverBinding"))
client.addServerBindings(XML.getValue(serverBinding));
- DomComponentBuilder.addChildren(deployState, ancestor, clientElement, client);
+ DomComponentBuilder.addChildren(deployState, parent, clientElement, client);
return client;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
index 558c4428d15..ac6d089cf24 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
@@ -1,38 +1,86 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.builder.xml.dom;
+import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.text.XML;
-import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.Handler;
import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
import org.w3c.dom.Element;
+import java.util.Set;
+
+import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1;
+import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2;
+import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1;
+import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_2;
+import static com.yahoo.vespa.model.container.ContainerCluster.VIP_HANDLER_BINDING;
+import static java.util.logging.Level.INFO;
+
/**
* @author gjoranv
*/
public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> {
+ private static final Set<String> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1,
+ METRICS_V2_HANDLER_BINDING_2,
+ STATE_HANDLER_BINDING_1,
+ STATE_HANDLER_BINDING_2,
+ VIP_HANDLER_BINDING);
+ private final ApplicationContainerCluster cluster;
+
+ public DomHandlerBuilder(ApplicationContainerCluster cluster) {
+ this.cluster = cluster;
+ }
+
@Override
- protected Handler doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element handlerElement) {
- Handler<? super Component<?, ?>> handler = getHandler(handlerElement);
+ protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element handlerElement) {
+ Handler<? super Component<?, ?>> handler = createHandler(handlerElement);
for (Element binding : XML.getChildren(handlerElement, "binding"))
- handler.addServerBindings(XML.getValue(binding));
+ addServerBinding(handler, XML.getValue(binding), deployState.getDeployLogger());
for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding"))
handler.addClientBindings(XML.getValue(clientBinding));
- DomComponentBuilder.addChildren(deployState, ancestor, handlerElement, handler);
+ DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler);
return handler;
}
- protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) {
+ Handler<? super Component<?, ?>> createHandler(Element handlerElement) {
BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement);
return new Handler<>(new ComponentModel(bundleSpec));
}
+
+ private void addServerBinding(Handler<? super Component<?, ?>> handler, String binding, DeployLogger log) {
+ throwIfBindingIsReserved(binding, handler);
+ handler.addServerBindings(binding);
+ removeExistingServerBinding(binding, handler, log);
+ }
+
+ private void throwIfBindingIsReserved(String binding, Handler<?> newHandler) {
+ for (var reserved : reservedBindings) {
+ if (binding.equals(reserved)) {
+ throw new IllegalArgumentException("Binding '" + binding + "' is a reserved Vespa binding and " +
+ "cannot be used by handler: " + newHandler.getComponentId());
+ }
+ }
+ }
+
+ private void removeExistingServerBinding(String binding, Handler<?> newHandler, DeployLogger log) {
+ for (var handler : cluster.getHandlers()) {
+ if (handler.getServerBindings().contains(binding)) {
+ handler.removeServerBinding(binding);
+ log.log(INFO, "Binding '" + binding + "' was already in use by handler '" +
+ handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId());
+ }
+ }
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
index d34a11abdf4..80c95ad6b59 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
@@ -172,7 +172,12 @@ public class ModelElement {
/** Returns the content of the attribute with the given name, or null if none */
public String stringAttribute(String name) {
- if ( ! xml.hasAttribute(name)) return null;
+ return stringAttribute(name, null);
+ }
+
+ /** Returns the content of the attribute with the given name, or the default value if none */
+ public String stringAttribute(String name, String defaultValue) {
+ if ( ! xml.hasAttribute(name)) return defaultValue;
return xml.getAttribute(name);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index 2c88f965f1f..ad6eebe1ca5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -1,11 +1,13 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.builder.xml.dom;
+import com.yahoo.collections.Pair;
import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.ConfigModelContext;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.text.XML;
@@ -15,9 +17,11 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
+import java.util.function.Function;
+import java.util.regex.Pattern;
/**
* A common utility class to represent a requirement for nodes during model building.
@@ -27,11 +31,9 @@ import java.util.Set;
*/
public class NodesSpecification {
- private final boolean dedicated;
-
- private final int count;
+ private final ClusterResources min, max;
- private final int groups;
+ private final boolean dedicated;
/** The Vespa version we want the nodes to run */
private Version version;
@@ -43,50 +45,76 @@ public class NodesSpecification {
private final boolean required;
private final boolean canFail;
-
- private final boolean exclusive;
-
- /** Whether this requires running container and content processes co-located on the same node. */
- private final boolean combined;
- /** The resources each node should have, or empty to use the default */
- private final Optional<NodeResources> resources;
-
- /** The identifier of the custom docker image layer to use (not supported yet) */
- private final Optional<String> dockerImage;
+ private final boolean exclusive;
- private NodesSpecification(boolean dedicated, int count, int groups, Version version,
- boolean required, boolean canFail, boolean exclusive, boolean combined,
- Optional<NodeResources> resources, Optional<String> dockerImage) {
+ /** The repo part of a docker image (without tag), optional */
+ private final Optional<String> dockerImageRepo;
+
+ /** The ID of the cluster referencing this node specification, if any */
+ private final Optional<String> combinedId;
+
+ private NodesSpecification(ClusterResources min,
+ ClusterResources max,
+ boolean dedicated, Version version,
+ boolean required, boolean canFail, boolean exclusive,
+ Optional<String> dockerImageRepo,
+ Optional<String> combinedId) {
+ if (max.smallerThan(min))
+ throw new IllegalArgumentException("Min resources must be larger or equal to max resources, but " +
+ max + " is smaller than " + min);
+
+ // Non-scaled resources must be equal
+ if ( ! min.nodeResources().justNonNumbers().equals(max.nodeResources().justNonNumbers()))
+ throw new IllegalArgumentException("Min and max resources must have the same non-numeric settings, but " +
+ "min is " + min + " and max " + max);
+ if (min.nodeResources().bandwidthGbps() != max.nodeResources().bandwidthGbps())
+ throw new IllegalArgumentException("Min and max resources must have the same bandwith, but " +
+ "min is " + min + " and max " + max);
+
+ this.min = min;
+ this.max = max;
this.dedicated = dedicated;
- this.count = count;
- this.groups = groups;
this.version = version;
this.required = required;
this.canFail = canFail;
this.exclusive = exclusive;
- this.resources = resources;
- this.dockerImage = dockerImage;
- this.combined = combined;
+ this.dockerImageRepo = dockerImageRepo;
+ this.combinedId = combinedId;
}
- private NodesSpecification(boolean dedicated, boolean canFail, boolean combined, Version version, ModelElement nodesElement) {
- this(dedicated,
- nodesElement.integerAttribute("count", 1),
- nodesElement.integerAttribute("groups", 1),
- version,
- nodesElement.booleanAttribute("required", false),
- canFail,
- nodesElement.booleanAttribute("exclusive", false),
- combined,
- getResources(nodesElement),
- Optional.ofNullable(nodesElement.stringAttribute("docker-image")));
+ private static NodesSpecification create(boolean dedicated, boolean canFail, Version version,
+ ModelElement nodesElement, Optional<String> dockerImageRepo) {
+ var resolvedElement = resolveElement(nodesElement);
+ var combinedId = findCombinedId(nodesElement, resolvedElement);
+ var resources = toResources(resolvedElement);
+ return new NodesSpecification(resources.getFirst(),
+ resources.getSecond(),
+ dedicated,
+ version,
+ resolvedElement.booleanAttribute("required", false),
+ canFail,
+ resolvedElement.booleanAttribute("exclusive", false),
+ dockerImageToUse(resolvedElement, dockerImageRepo),
+ combinedId);
}
- private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) {
- var resolvedElement = resolveElement(nodesElement);
- boolean combined = resolvedElement != nodesElement || isReferencedByOtherElement(nodesElement);
- return new NodesSpecification(dedicated, canFail, combined, version, resolvedElement);
+ private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) {
+ Pair<Integer, Integer> nodes = toRange(nodesElement.stringAttribute("count"), 1, Integer::parseInt);
+ Pair<Integer, Integer> groups = toRange(nodesElement.stringAttribute("groups"), 1, Integer::parseInt);
+ var min = new ClusterResources(nodes.getFirst(), groups.getFirst(), nodeResources(nodesElement).getFirst());
+ var max = new ClusterResources(nodes.getSecond(), groups.getSecond(), nodeResources(nodesElement).getSecond());
+ return new Pair<>(min, max);
+ }
+
+ /** Returns the ID of the cluster referencing this node specification, if any */
+ private static Optional<String> findCombinedId(ModelElement nodesElement, ModelElement resolvedElement) {
+ if (resolvedElement != nodesElement) {
+ // Specification for a container cluster referencing nodes in a content cluster
+ return containerIdOf(nodesElement);
+ }
+ // Specification for a content cluster that is referenced by a container cluster
+ return containerIdReferencing(nodesElement);
}
/** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */
@@ -94,7 +122,8 @@ public class NodesSpecification {
return create(true,
! context.getDeployState().getProperties().isBootstrap(),
context.getDeployState().getWantedNodeVespaVersion(),
- nodesElement);
+ nodesElement,
+ context.getDeployState().getWantedDockerImageRepo());
}
/**
@@ -110,7 +139,7 @@ public class NodesSpecification {
}
/**
- * Returns a requirement for undedicated or dedicated nodes taken from the <code>nodes</code> element
+ * Returns a requirement for non-dedicated or dedicated nodes taken from the <code>nodes</code> element
* contained in the given parent element, or empty if the parent element is null, or the nodes elements
* is not present.
*/
@@ -121,37 +150,42 @@ public class NodesSpecification {
if (nodesElement == null) return Optional.empty();
return Optional.of(create(nodesElement.booleanAttribute("dedicated", false),
! context.getDeployState().getProperties().isBootstrap(),
- context.getDeployState().getWantedNodeVespaVersion(), nodesElement));
+ context.getDeployState().getWantedNodeVespaVersion(),
+ nodesElement,
+ context.getDeployState().getWantedDockerImageRepo()));
}
- /** Returns a requirement from <code>count</code> nondedicated nodes in one group */
+ /**
+ * Returns a requirement from <code>count</code> non-dedicated nodes in one group
+ */
public static NodesSpecification nonDedicated(int count, ConfigModelContext context) {
- return new NodesSpecification(false,
- count,
- 1,
+ return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
+ new ClusterResources(count, 1, NodeResources.unspecified),
+ false,
context.getDeployState().getWantedNodeVespaVersion(),
false,
! context.getDeployState().getProperties().isBootstrap(),
false,
- false,
- Optional.empty(),
+ context.getDeployState().getWantedDockerImageRepo(),
Optional.empty());
}
/** Returns a requirement from <code>count</code> dedicated nodes in one group */
public static NodesSpecification dedicated(int count, ConfigModelContext context) {
- return new NodesSpecification(true,
- count,
- 1,
+ return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
+ new ClusterResources(count, 1, NodeResources.unspecified),
+ true,
context.getDeployState().getWantedNodeVespaVersion(),
false,
! context.getDeployState().getProperties().isBootstrap(),
false,
- false,
- Optional.empty(),
+ context.getDeployState().getWantedDockerImageRepo(),
Optional.empty());
}
+ public ClusterResources minResources() { return min; }
+ public ClusterResources maxResources() { return max; }
+
/**
* Returns whether this requires dedicated nodes.
* Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible.
@@ -165,42 +199,48 @@ public class NodesSpecification {
*/
public boolean isExclusive() { return exclusive; }
- /** Returns the number of nodes required */
- public int count() { return count; }
-
- /** Returns the number of host groups this specifies. Default is 1 */
- public int groups() { return groups; }
-
public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem,
ClusterSpec.Type clusterType,
ClusterSpec.Id clusterId,
DeployLogger logger) {
- if (combined)
+ if (combinedId.isPresent())
clusterType = ClusterSpec.Type.combined;
- ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive);
- return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger);
+ ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId)
+ .vespaVersion(version)
+ .exclusive(exclusive)
+ .combinedId(combinedId.map(ClusterSpec.Id::from))
+ .dockerImageRepo(dockerImageRepo)
+ .build();
+ return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail), logger);
}
- private static Optional<NodeResources> getResources(ModelElement nodesElement) {
+ private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) {
ModelElement resources = nodesElement.child("resources");
if (resources != null) {
- return Optional.of(new NodeResources(resources.requiredDoubleAttribute("vcpu"),
- parseGbAmount(resources.requiredStringAttribute("memory"), "B"),
- parseGbAmount(resources.requiredStringAttribute("disk"), "B"),
- Optional.ofNullable(resources.stringAttribute("bandwidth"))
- .map(b -> parseGbAmount(b, "BPS"))
- .orElse(0.3),
- parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")),
- parseOptionalStorageType(resources.stringAttribute("storage-type"))));
+ return nodeResourcesFromResorcesElement(resources);
}
else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback
- return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor")));
+ var flavorResources = NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"));
+ return new Pair<>(flavorResources, flavorResources);
}
- else { // Get the default
- return Optional.empty();
+ else {
+ return new Pair<>(NodeResources.unspecified, NodeResources.unspecified);
}
}
+ private static Pair<NodeResources, NodeResources> nodeResourcesFromResorcesElement(ModelElement element) {
+ Pair<Double, Double> vcpu = toRange(element.requiredStringAttribute("vcpu"), .0, Double::parseDouble);
+ Pair<Double, Double> memory = toRange(element.requiredStringAttribute("memory"), .0, s -> parseGbAmount(s, "B"));
+ Pair<Double, Double> disk = toRange(element.requiredStringAttribute("disk"), .0, s -> parseGbAmount(s, "B"));
+ Pair<Double, Double> bandwith = toRange(element.stringAttribute("bandwith"), .3, s -> parseGbAmount(s, "BPS"));
+ NodeResources.DiskSpeed diskSpeed = parseOptionalDiskSpeed(element.stringAttribute("disk-speed"));
+ NodeResources.StorageType storageType = parseOptionalStorageType(element.stringAttribute("storage-type"));
+
+ var min = new NodeResources(vcpu.getFirst(), memory.getFirst(), disk.getFirst(), bandwith.getFirst(), diskSpeed, storageType);
+ var max = new NodeResources(vcpu.getSecond(), memory.getSecond(), disk.getSecond(), bandwith.getSecond(), diskSpeed, storageType);
+ return new Pair<>(min, max);
+ }
+
private static double parseGbAmount(String byteAmount, String unit) {
byteAmount = byteAmount.strip();
byteAmount = byteAmount.toUpperCase();
@@ -280,24 +320,35 @@ public class NodesSpecification {
return new ModelElement(referencedNodesElement);
}
- /** Returns whether the given nodesElement is referenced by any other nodes element */
- private static boolean isReferencedByOtherElement(ModelElement nodesElement) {
+ /** Returns the ID of the parent container element of nodesElement, if any */
+ private static Optional<String> containerIdOf(ModelElement nodesElement) {
+ var element = nodesElement.getXml();
+ for (var containerTag : List.of("container", "jdisc")) {
+ var container = findParentByTag(containerTag, element);
+ if (container.isEmpty()) continue;
+ return container.map(el -> el.getAttribute("id"));
+ }
+ return Optional.empty();
+ }
+
+ /** Returns the ID of the container element referencing nodesElement, if any */
+ private static Optional<String> containerIdReferencing(ModelElement nodesElement) {
var element = nodesElement.getXml();
var services = findParentByTag("services", element);
- if (services.isEmpty()) return false;
+ if (services.isEmpty()) return Optional.empty();
var content = findParentByTag("content", element);
- if (content.isEmpty()) return false;
+ if (content.isEmpty()) return Optional.empty();
var contentClusterId = content.get().getAttribute("id");
- if (contentClusterId.isEmpty()) return false;
+ if (contentClusterId.isEmpty()) return Optional.empty();
for (var rootChild : XML.getChildren(services.get())) {
if ( ! ContainerModelBuilder.isContainerTag(rootChild)) continue;
var nodes = XML.getChild(rootChild, "nodes");
if (nodes == null) continue;
if (!contentClusterId.equals(nodes.getAttribute("of"))) continue;
- return true;
+ return Optional.of(rootChild.getAttribute("id"));
}
- return false;
+ return Optional.empty();
}
private static Optional<Element> findChildById(Element parent, String id) {
@@ -319,11 +370,33 @@ public class NodesSpecification {
return new IllegalArgumentException("referenced service '" + referenceId + "' is not defined");
}
+ private static Optional<String> dockerImageToUse(ModelElement nodesElement, Optional<String> dockerImage) {
+ String dockerImageFromElement = nodesElement.stringAttribute("docker-image");
+ return dockerImageFromElement == null ? dockerImage : Optional.of(dockerImageFromElement);
+ }
+
+ /** Parses a value ("value") or value range ("[min-value, max-value]") */
+ private static <T> Pair<T, T> toRange(String s, T defaultValue, Function<String, T> valueParser) {
+ try {
+ if (s == null) return new Pair<>(defaultValue, defaultValue);
+ s = s.trim();
+ if (s.startsWith("[") && s.endsWith("]")) {
+ String[] numbers = s.substring(1, s.length() - 1).split(",");
+ if (numbers.length != 2) throw new IllegalArgumentException();
+ return new Pair<>(valueParser.apply(numbers[0].trim()), valueParser.apply(numbers[1].trim()));
+ } else {
+ return new Pair<>(valueParser.apply(s), valueParser.apply(s));
+ }
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Expected a number or range on the form [min, max], but got '" + s + "'", e);
+ }
+ }
+
@Override
public String toString() {
- return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" +
- (resources.isPresent() ? " with resources " + resources.get() : "") +
- (groups > 1 ? " in " + groups + " groups" : "");
+ return "specification of " + (dedicated ? "dedicated " : "") +
+ (min.equals(max) ? min : "min " + min + " max " + max);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
index 97b78e1b9b1..c9caca1831f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
@@ -141,8 +141,7 @@ public class VespaDomBuilder extends VespaModelBuilder {
}
private void initializeService(AbstractService t, DeployState deployState,
- HostSystem hostSystem, Element producerSpec)
- {
+ HostSystem hostSystem, Element producerSpec) {
initializeProducer(t, deployState, producerSpec);
if (producerSpec != null) {
if (producerSpec.hasAttribute(JVM_OPTIONS)) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index efd00528d54..63e6af03c44 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -1,6 +1,7 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container;
+import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler;
import com.yahoo.component.ComponentId;
import com.yahoo.component.ComponentSpecification;
import com.yahoo.config.FileReference;
@@ -9,6 +10,9 @@ import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.container.BundlesConfig;
import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.container.handler.metrics.MetricsProxyApiConfig;
+import com.yahoo.container.handler.metrics.MetricsV2Handler;
import com.yahoo.container.jdisc.ContainerMbusConfig;
import com.yahoo.container.jdisc.messagebus.MbusServerProvider;
import com.yahoo.jdisc.http.ServletPathsConfig;
@@ -17,8 +21,10 @@ import com.yahoo.search.config.QrStartConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.ConfigProducerGroup;
+import com.yahoo.vespa.model.container.component.Handler;
import com.yahoo.vespa.model.container.component.Servlet;
import com.yahoo.vespa.model.container.jersey.Jersey2Servlet;
import com.yahoo.vespa.model.container.jersey.RestApi;
@@ -45,7 +51,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
RankProfilesConfig.Producer,
RankingConstantsConfig.Producer,
ServletPathsConfig.Producer,
- ContainerMbusConfig.Producer {
+ ContainerMbusConfig.Producer,
+ MetricsProxyApiConfig.Producer {
+
+ public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName();
+ public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH;
+ public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*";
private final Set<FileReference> applicationBundles = new LinkedHashSet<>();
@@ -58,6 +69,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
private MbusParams mbusParams;
private boolean messageBusEnabled = true;
+ private final double softStartSeconds;
private Integer memoryPercentage = null;
@@ -72,7 +84,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider");
addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider");
addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider");
+ addMetricsV2Handler();
addTestrunnerComponentsIfTester(deployState);
+ softStartSeconds = deployState.getProperties().defaultSoftStartSeconds();
}
@Override
@@ -99,6 +113,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
}
}
+ public void addMetricsV2Handler() {
+ Handler<AbstractConfigProducer<?>> handler = new Handler<>(
+ new ComponentModel(METRICS_V2_HANDLER_CLASS, null, null, null));
+ handler.addServerBindings(METRICS_V2_HANDLER_BINDING_1, METRICS_V2_HANDLER_BINDING_2);
+ addComponent(handler);
+ }
+
private void addTestrunnerComponentsIfTester(DeployState deployState) {
if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester())
addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar")));
@@ -188,10 +209,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
}
@Override
+ public void getConfig(MetricsProxyApiConfig.Builder builder) {
+ builder.metricsPort(MetricsProxyContainer.BASEPORT)
+ .metricsApiPath(ApplicationMetricsHandler.VALUES_PATH);
+ }
+
+ @Override
public void getConfig(QrStartConfig.Builder builder) {
super.getConfig(builder);
builder.jvm.verbosegc(true)
.availableProcessors(0)
+ .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m
.minHeapsize(1536)
.heapsize(1536);
if (getMemoryPercentage().isPresent()) {
@@ -223,6 +251,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
null))));
}
+ @Override
+ public void getConfig(ThreadpoolConfig.Builder builder) {
+ builder.softStartSeconds(softStartSeconds);
+ }
+
public static class MbusParams {
// the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%)
final Double maxConcurrentFactor;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 7e2d6680827..31c8724d634 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -140,7 +140,7 @@ public abstract class Container extends AbstractService implements
if (http == null) {
return defaultHttpServer;
} else {
- return http.getHttpServer();
+ return http.getHttpServer().orElse(null);
}
}
@@ -228,10 +228,10 @@ public abstract class Container extends AbstractService implements
// XXX unused - remove:
from.allocatePort("http/1");
portsMeta.on(offset++).tag("http").tag("external");
- } else if (getHttp().getHttpServer() == null) {
+ } else if (getHttp().getHttpServer().isEmpty()) {
// no http server ports
} else {
- for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) {
+ for (ConnectorFactory connectorFactory : getHttp().getHttpServer().get().getConnectorFactories()) {
int port = getPort(connectorFactory);
String name = "http/" + connectorFactory.getName();
from.requirePort(port, name);
@@ -280,7 +280,7 @@ public abstract class Container extends AbstractService implements
final Http http = getHttp();
if (http != null) {
// TODO: allow the user to specify health port manually
- if (http.getHttpServer() == null) {
+ if (http.getHttpServer().isEmpty()) {
return -1;
} else {
return getRelativePort(0);
@@ -303,7 +303,7 @@ public abstract class Container extends AbstractService implements
.slobrokId(serviceSlobrokId())).
filedistributor(filedistributorConfig());
if (clusterName != null) {
- builder.discriminator(clusterName+"."+name);
+ builder.discriminator(clusterName + "." + name);
} else {
builder.discriminator(name);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 6fa446bf365..e2b1f97a6eb 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -19,6 +19,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification;
import com.yahoo.container.core.ApplicationMetadataConfig;
import com.yahoo.container.core.document.ContainerDocumentConfig;
import com.yahoo.container.handler.ThreadPoolProvider;
+import com.yahoo.container.handler.ThreadpoolConfig;
import com.yahoo.container.jdisc.JdiscBindingsConfig;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
import com.yahoo.container.jdisc.state.StateHandler;
@@ -99,7 +100,9 @@ public abstract class ContainerCluster<CONTAINER extends Container>
DocprocConfig.Producer,
ClusterInfoConfig.Producer,
RoutingProviderConfig.Producer,
- ConfigserverConfig.Producer {
+ ConfigserverConfig.Producer,
+ ThreadpoolConfig.Producer
+{
/**
* URI prefix used for internal, usually programmatic, APIs. URIs using this
@@ -111,15 +114,20 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler";
public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName();
- public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler";
public static final String LOG_HANDLER_CLASS = com.yahoo.container.handler.LogHandler.class.getName();
public static final String DEFAULT_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.DefaultLinguisticsProvider";
public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1";
public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15";
+ public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler";
+ public static final String STATE_HANDLER_BINDING_1 = "http://*" + StateHandler.STATE_API_ROOT;
+ public static final String STATE_HANDLER_BINDING_2 = STATE_HANDLER_BINDING_1 + "/*";
+
public static final String ROOT_HANDLER_PATH = "/";
public static final String ROOT_HANDLER_BINDING = "http://*" + ROOT_HANDLER_PATH;
+ public static final String VIP_HANDLER_BINDING = "http://*/status.html";
+
private final String name;
protected List<CONTAINER> containers = new ArrayList<>();
@@ -200,15 +208,11 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public void addMetricStateHandler() {
Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>(
new ComponentModel(STATE_HANDLER_CLASS, null, null, null));
- stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT,
- "http://*" + StateHandler.STATE_API_ROOT + "/*");
+ stateHandler.addServerBindings(STATE_HANDLER_BINDING_1, STATE_HANDLER_BINDING_2);
addComponent(stateHandler);
}
public void addDefaultRootHandler() {
- if (hasHandlerWithBinding(ROOT_HANDLER_BINDING))
- return;
-
Handler<AbstractConfigProducer<?>> handler = new Handler<>(
new ComponentModel(BundleInstantiationSpecification.getFromStrings(
BINDINGS_OVERVIEW_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc
@@ -216,15 +220,6 @@ public abstract class ContainerCluster<CONTAINER extends Container>
addComponent(handler);
}
- private boolean hasHandlerWithBinding(String binding) {
- Collection<Handler<?>> handlers = getHandlers();
- for (Handler handler : handlers) {
- if (handler.getServerBindings().contains(binding))
- return true;
- }
- return false;
- }
-
public void addApplicationStatusHandler() {
Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>(
new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(
@@ -235,7 +230,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public void addVipHandler() {
Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS);
- vipHandler.addServerBindings("http://*/status.html");
+ vipHandler.addServerBindings(VIP_HANDLER_BINDING);
addComponent(vipHandler);
}
@@ -484,6 +479,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
builder.jvm
.verbosegc(false)
.availableProcessors(2)
+ .compressedClassSpaceSize(32)
.minHeapsize(32)
.heapsize(512)
.heapSizeAsPercentageOfPhysicalMemory(0)
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
index c671749cff0..a466dabe984 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
@@ -4,7 +4,12 @@ package com.yahoo.vespa.model.container.component;
import com.yahoo.component.ComponentId;
import com.yahoo.config.model.producer.AbstractConfigProducer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
/**
* A group of config producers that have a component id.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
index ee61b34987a..3d9a1b2e665 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
@@ -10,6 +10,7 @@ import com.yahoo.osgi.provider.model.ComponentModel;
* @author Tony Vaagenes
*/
public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer {
+
public static final String CLASS = "com.yahoo.container.handler.VipStatusHandler";
private final String fileName;
@@ -26,4 +27,5 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf
builder.accessdisk(true).
statusfile(fileName);
}
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
index e07c3216850..82484e07773 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
@@ -8,7 +8,9 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
/**
* Models a jdisc RequestHandler (including ClientProvider).
@@ -21,7 +23,7 @@ import java.util.List;
*/
public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> {
- private List<String> serverBindings = new ArrayList<>();
+ private Set<String> serverBindings = new LinkedHashSet<>();
private List<String> clientBindings = new ArrayList<>();
public Handler(ComponentModel model) {
@@ -40,12 +42,16 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<
serverBindings.addAll(Arrays.asList(bindings));
}
+ public void removeServerBinding(String binding) {
+ serverBindings.remove(binding);
+ }
+
public void addClientBindings(String... bindings) {
clientBindings.addAll(Arrays.asList(bindings));
}
- public final List<String> getServerBindings() {
- return Collections.unmodifiableList(serverBindings);
+ public final Set<String> getServerBindings() {
+ return Collections.unmodifiableSet(serverBindings);
}
public final List<String> getClientBindings() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
index f3758def2b1..470b82496a3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.Zone;
import com.yahoo.container.StatisticsConfig;
+import com.yahoo.container.core.VipStatusConfig;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
import com.yahoo.net.HostName;
import com.yahoo.vespa.defaults.Defaults;
@@ -29,7 +30,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
ZookeeperServerConfig.Producer,
ConfigserverConfig.Producer,
StatisticsConfig.Producer,
- HealthMonitorConfig.Producer {
+ HealthMonitorConfig.Producer,
+ VipStatusConfig.Producer {
private final CloudConfigOptions options;
private ContainerCluster containerCluster;
@@ -116,8 +118,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
}
builder.serverId(HostName.getLocalhost());
- if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) {
- builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort());
+ if (!containerCluster.getHttp().getHttpServer().get().getConnectorFactories().isEmpty()) {
+ builder.httpport(containerCluster.getHttp().getHttpServer().get().getConnectorFactories().get(0).getListenPort());
}
if (options.useVespaVersionInRequest().isPresent()) {
builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get());
@@ -178,4 +180,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
builder.snapshot_interval(60.0);
}
+ @Override
+ public void getConfig(VipStatusConfig.Builder builder) {
+ builder.initiallyInRotation(false);
+ }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
index d3ba2718d71..dd48e65c340 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
@@ -1,10 +1,10 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.http;
-import com.google.common.collect.ImmutableList;
import com.yahoo.component.ComponentId;
import com.yahoo.component.ComponentSpecification;
import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
import com.yahoo.vespa.model.container.component.Handler;
@@ -15,7 +15,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -24,23 +23,23 @@ import java.util.stream.Stream;
* Helper class for http access control.
*
* @author gjoranv
+ * @author bjorncs
*/
public final class AccessControl {
public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain");
- private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of(
+ public static final List<String> UNPROTECTED_HANDLERS = List.of(
FileStatusHandlerComponent.CLASS,
ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS,
ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS,
ContainerCluster.STATE_HANDLER_CLASS,
- ContainerCluster.LOG_HANDLER_CLASS
+ ContainerCluster.LOG_HANDLER_CLASS,
+ ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS
);
public static final class Builder {
private String domain;
- private String applicationId;
- private Optional<String> vespaDomain = Optional.empty();
private boolean readEnabled = false;
private boolean writeEnabled = true;
private final Set<String> excludeBindings = new LinkedHashSet<>();
@@ -48,9 +47,8 @@ public final class AccessControl {
private Collection<Servlet> servlets = Collections.emptyList();
private final DeployLogger logger;
- public Builder(String domain, String applicationId, DeployLogger logger) {
+ public Builder(String domain, DeployLogger logger) {
this.domain = domain;
- this.applicationId = applicationId;
this.logger = logger;
}
@@ -69,52 +67,37 @@ public final class AccessControl {
return this;
}
- public Builder vespaDomain(String vespaDomain) {
- this.vespaDomain = Optional.ofNullable(vespaDomain);
- return this;
- }
-
- public Builder setHandlers(Collection<Handler<?>> handlers) {
- this.handlers = handlers;
- return this;
- }
-
- public Builder setServlets(Collection<Servlet> servlets) {
- this.servlets = servlets;
+ public Builder setHandlers(ApplicationContainerCluster cluster) {
+ this.handlers = cluster.getHandlers();
+ this.servlets = cluster.getAllServlets();
return this;
}
public AccessControl build() {
- return new AccessControl(domain, applicationId, writeEnabled, readEnabled,
- excludeBindings, vespaDomain, servlets, handlers, logger);
+ return new AccessControl(domain, writeEnabled, readEnabled,
+ excludeBindings, servlets, handlers, logger);
}
}
public final String domain;
- public final String applicationId;
public final boolean readEnabled;
public final boolean writeEnabled;
- public final Optional<String> vespaDomain;
private final Set<String> excludedBindings;
private final Collection<Handler<?>> handlers;
private final Collection<Servlet> servlets;
private final DeployLogger logger;
private AccessControl(String domain,
- String applicationId,
boolean writeEnabled,
boolean readEnabled,
Set<String> excludedBindings,
- Optional<String> vespaDomain,
Collection<Servlet> servlets,
Collection<Handler<?>> handlers,
DeployLogger logger) {
this.domain = domain;
- this.applicationId = applicationId;
this.readEnabled = readEnabled;
this.writeEnabled = writeEnabled;
this.excludedBindings = Collections.unmodifiableSet(excludedBindings);
- this.vespaDomain = vespaDomain;
this.handlers = handlers;
this.servlets = servlets;
this.logger = logger;
@@ -125,6 +108,10 @@ public final class AccessControl {
.collect(Collectors.toCollection(ArrayList::new));
}
+ public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) {
+ return cluster.getHandlers().stream().anyMatch(AccessControl::handlerNeedsProtection);
+ }
+
private Stream<Binding> getHandlerBindings() {
return handlers.stream()
.filter(this::shouldHandlerBeProtected)
@@ -144,7 +131,7 @@ public final class AccessControl {
&& handler.getServerBindings().stream().noneMatch(excludedBindings::contains);
}
- public static boolean isBuiltinGetOnly(Handler<?> handler) {
+ private static boolean isBuiltinGetOnly(Handler<?> handler) {
return UNPROTECTED_HANDLERS.contains(handler.getClassId().getName());
}
@@ -159,4 +146,13 @@ public final class AccessControl {
private static Stream<String> servletBindings(Servlet servlet) {
return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath);
}
+
+ private static boolean handlerNeedsProtection(Handler<?> handler) {
+ return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler);
+ }
+
+ private static boolean hasNonMbusBinding(Handler<?> handler) {
+ return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus"));
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
index 400ddf80cf9..0fcf7b2d06c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
@@ -8,46 +8,39 @@ import com.yahoo.vespa.model.container.component.chain.Chain;
import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
/**
* Represents the http servers and filters of a container cluster.
*
* @author Tony Vaagenes
+ * @author bjorncs
*/
public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer {
- private FilterChains filterChains;
- private JettyHttpServer httpServer;
- private List<Binding> bindings;
- private final Optional<AccessControl> accessControl;
+ private final FilterChains filterChains;
+ private final List<Binding> bindings = new CopyOnWriteArrayList<>();
+ private volatile JettyHttpServer httpServer;
+ private volatile AccessControl accessControl;
- public Http(List<Binding> bindings) {
- this(bindings, null);
+ public Http(FilterChains chains) {
+ super("http");
+ this.filterChains = chains;
}
- public Http(List<Binding> bindings, AccessControl accessControl) {
- super( "http");
- this.bindings = Collections.unmodifiableList(bindings);
- this.accessControl = Optional.ofNullable(accessControl);
- }
-
- public void setFilterChains(FilterChains filterChains) {
- this.filterChains = filterChains;
- }
-
- public void setBindings(List<Binding> bindings) {
- this.bindings = Collections.unmodifiableList(bindings);
+ public void setAccessControl(AccessControl accessControl) {
+ if (this.accessControl != null) throw new IllegalStateException("Access control already assigned");
+ this.accessControl = accessControl;
}
public FilterChains getFilterChains() {
return filterChains;
}
- public JettyHttpServer getHttpServer() {
- return httpServer;
+ public Optional<JettyHttpServer> getHttpServer() {
+ return Optional.ofNullable(httpServer);
}
public void setHttpServer(JettyHttpServer newServer) {
@@ -76,25 +69,21 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl
}
public Optional<AccessControl> getAccessControl() {
- return accessControl;
+ return Optional.ofNullable(accessControl);
}
@Override
public void getConfig(ServerConfig.Builder builder) {
for (Binding binding : bindings) {
builder.filter(new ServerConfig.Filter.Builder()
- .id(binding.filterId().stringValue())
- .binding(binding.binding()));
+ .id(binding.filterId().stringValue())
+ .binding(binding.binding()));
}
}
@Override
public void validate() {
- validate(bindings);
- }
-
- void validate(Collection<Binding> bindings) {
- if (bindings.isEmpty()) return;
+ if (((Collection<Binding>) bindings).isEmpty()) return;
if (filterChains == null)
throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings");
@@ -107,5 +96,4 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl
throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding());
}
}
-
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
index 4f84a01ff94..4a331718985 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
@@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.vespa.model.container.component.SimpleComponent;
+import java.util.List;
import java.util.Optional;
import static com.yahoo.component.ComponentSpecification.fromString;
@@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString;
* Configure SSL using file references
*
* @author mortent
+ * @author bjorncs
*/
public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer {
public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@";
@@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C
private final String certificatePath;
private final String caCertificatePath;
private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication;
+ private final List<String> cipherSuites;
+ private final List<String> protocolVersions;
- public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) {
+ public ConfiguredFilebasedSslProvider(String servername,
+ String privateKeyPath,
+ String certificatePath,
+ String caCertificatePath,
+ String clientAuthentication,
+ List<String> cipherSuites,
+ List<String> protocolVersions) {
super(new ComponentModel(
new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername),
fromString(COMPONENT_CLASS),
@@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C
this.certificatePath = certificatePath;
this.caCertificatePath = caCertificatePath;
this.clientAuthentication = mapToConfigEnum(clientAuthentication);
+ this.cipherSuites = cipherSuites;
+ this.protocolVersions = protocolVersions;
}
@Override
public void getConfig(ConnectorConfig.Builder builder) {
- builder.ssl.enabled(true);
- builder.ssl.privateKeyFile(privateKeyPath);
- builder.ssl.certificateFile(certificatePath);
- builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse(""));
- builder.ssl.clientAuth(clientAuthentication);
+ builder.ssl(
+ new ConnectorConfig.Ssl.Builder()
+ .enabled(true)
+ .privateKeyFile(privateKeyPath)
+ .certificateFile(certificatePath)
+ .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse(""))
+ .clientAuth(clientAuthentication)
+ .enabledCipherSuites(cipherSuites)
+ .enabledProtocols(protocolVersions));
}
public SimpleComponent getComponent() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
index 12db3b87243..fb8e9dffbbb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
@@ -7,6 +7,7 @@ import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth;
import com.yahoo.vespa.model.container.component.SimpleComponent;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
+import java.time.Duration;
import java.util.List;
/**
@@ -17,42 +18,49 @@ import java.util.List;
public class HostedSslConnectorFactory extends ConnectorFactory {
private static final List<String> INSECURE_WHITELISTED_PATHS = List.of("/status.html");
+ private static final String DEFAULT_HOSTED_TRUSTSTORE = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem";
private final boolean enforceClientAuth;
+ private final String proxyProtocol;
/**
- * Create connector factory that uses a certificate provided by the config-model / configserver.
+ * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore.
*/
- public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) {
- return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificates*/null), false);
+ // TODO Enforce client authentication
+ public static HostedSslConnectorFactory withProvidedCertificate(String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets) {
+ return new HostedSslConnectorFactory(proxyProtocol,
+ createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null), false);
}
/**
* Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application.
*/
- public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
- return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, tlsCaCertificates), true);
+ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(
+ String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
+ return new HostedSslConnectorFactory(proxyProtocol,
+ createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates), true);
}
/**
* Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration).
*/
- public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName) {
- return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true);
+ public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String proxyProtocol, String serverName) {
+ return new HostedSslConnectorFactory(proxyProtocol, new DefaultSslProvider(serverName), true);
}
- private HostedSslConnectorFactory(SimpleComponent sslProviderComponent, boolean enforceClientAuth) {
+ private HostedSslConnectorFactory(String proxyProtocol, SimpleComponent sslProviderComponent, boolean enforceClientAuth) {
super("tls4443", 4443, sslProviderComponent);
+ this.proxyProtocol = proxyProtocol;
this.enforceClientAuth = enforceClientAuth;
}
private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider(
- String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
+ String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificatesPath, String tlsCaCertificates) {
return new ConfiguredDirectSslProvider(
serverName,
endpointCertificateSecrets.key(),
endpointCertificateSecrets.certificate(),
- /*caCertificatePath*/null,
+ tlsCaCertificatesPath,
tlsCaCertificates,
ClientAuth.Enum.WANT_AUTH);
}
@@ -60,9 +68,27 @@ public class HostedSslConnectorFactory extends ConnectorFactory {
@Override
public void getConfig(ConnectorConfig.Builder connectorBuilder) {
super.getConfig(connectorBuilder);
- connectorBuilder.tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder()
- .pathWhitelist(INSECURE_WHITELISTED_PATHS)
- .enable(enforceClientAuth));
+ connectorBuilder
+ .tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder()
+ .pathWhitelist(INSECURE_WHITELISTED_PATHS)
+ .enable(enforceClientAuth))
+ .proxyProtocol(configureProxyProtocol())
+ .idleTimeout(Duration.ofMinutes(3).toSeconds())
+ .maxConnectionLife(Duration.ofMinutes(10).toSeconds());
+ }
+
+ private ConnectorConfig.ProxyProtocol.Builder configureProxyProtocol() {
+ ConnectorConfig.ProxyProtocol.Builder proxyProtocolBuilder = new ConnectorConfig.ProxyProtocol.Builder();
+ switch (proxyProtocol) {
+ case "https-only":
+ return proxyProtocolBuilder.enabled(false).mixedMode(false);
+ case "https+proxy-protocol":
+ return proxyProtocolBuilder.enabled(true).mixedMode(true);
+ case "proxy-protocol-only":
+ return proxyProtocolBuilder.enabled(true).mixedMode(false);
+ default:
+ throw new IllegalArgumentException("Unknown proxy-protocol settings: " + proxyProtocol);
+ }
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
index b37caf22216..bfde9b9add1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
@@ -6,19 +6,18 @@ import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.builder.xml.XmlHelper;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.text.XML;
import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
-import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ApplicationContainerCluster;
+import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.component.chain.Chain;
import com.yahoo.vespa.model.container.http.AccessControl;
+import com.yahoo.vespa.model.container.http.Binding;
import com.yahoo.vespa.model.container.http.FilterChains;
import com.yahoo.vespa.model.container.http.Http;
-import com.yahoo.vespa.model.container.http.Binding;
import org.w3c.dom.Element;
import java.util.ArrayList;
@@ -55,24 +54,18 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
filterChains = new FilterChainsBuilder().newChainsInstance(ancestor);
}
- Http http = new Http(bindings, accessControl);
- http.setFilterChains(filterChains);
-
- buildHttpServers(deployState, ancestor, http, spec);
-
+ Http http = new Http(filterChains);
+ http.getBindings().addAll(bindings);
+ http.setAccessControl(accessControl);
+ http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec));
return http;
}
private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) {
- String application = XmlHelper.getOptionalChildValue(accessControlElem, "application")
- .orElse(getDeployedApplicationId(deployState, ancestor).value());
-
- AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), application, deployState.getDeployLogger());
+ AthenzDomain domain = getAccessControlDomain(deployState, accessControlElem);
+ AccessControl.Builder builder = new AccessControl.Builder(domain.value(), deployState.getDeployLogger());
- getContainerCluster(ancestor).ifPresent(cluster -> {
- builder.setHandlers(cluster.getHandlers());
- builder.setServlets(cluster.getAllServlets());
- });
+ getContainerCluster(ancestor).ifPresent(builder::setHandlers);
XmlHelper.getOptionalAttribute(accessControlElem, "read").ifPresent(
readAttr -> builder.readEnabled(Boolean.valueOf(readAttr)));
@@ -85,17 +78,29 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
.map(XML::getValue)
.forEach(builder::excludeBinding);
}
- XmlHelper.getOptionalChildValue(accessControlElem, "vespa-domain").ifPresent(builder::vespaDomain);
return builder.build();
}
- /**
- * Returns the id of the deployed application, or the default value if not explicitly set (self-hosted).
- */
- private static ApplicationName getDeployedApplicationId(DeployState deployState, AbstractConfigProducer ancestor) {
- return getContainerCluster(ancestor)
- .map(cluster -> deployState.getProperties().applicationId().application())
- .orElse(ApplicationId.defaultId().application());
+ // TODO Fail if domain is not provided through deploy properties
+ private static AthenzDomain getAccessControlDomain(DeployState deployState, Element accessControlElem) {
+ AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null);
+ AthenzDomain explicitDomain = XmlHelper.getOptionalAttribute(accessControlElem, "domain")
+ .map(AthenzDomain::from)
+ .orElse(null);
+ if (tenantDomain == null) {
+ if (explicitDomain == null) {
+ throw new IllegalStateException("No Athenz domain provided for 'access-control'");
+ }
+ deployState.getDeployLogger().log(Level.WARNING, "Athenz tenant is not provided by deploy call. This will soon be handled as failure.");
+ }
+ if (explicitDomain != null) {
+ if (tenantDomain != null && !explicitDomain.equals(tenantDomain)) {
+ throw new IllegalArgumentException(
+ String.format("Domain in access-control ('%s') does not match tenant domain ('%s')", explicitDomain.value(), tenantDomain.value()));
+ }
+ deployState.getDeployLogger().log(Level.WARNING, "Domain in 'access-control' is deprecated and will be removed soon");
+ }
+ return tenantDomain != null ? tenantDomain : explicitDomain;
}
private static Optional<ApplicationContainerCluster> getContainerCluster(AbstractConfigProducer configProducer) {
@@ -125,10 +130,6 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
return result;
}
- private void buildHttpServers(DeployState deployState, AbstractConfigProducer ancestor, Http http, Element spec) {
- http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec));
- }
-
static int readPort(ModelElement spec, boolean isHosted, DeployLogger logger) {
Integer port = spec.integerAttribute("port");
if (port == null)
@@ -139,13 +140,9 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
int legalPortInHostedVespa = Container.BASEPORT;
if (isHosted && port != legalPortInHostedVespa && ! spec.booleanAttribute("required", false)) {
- // TODO: After January 2020:
- // - Set required='true' for the http server on port 4443 in the tester services.xml in InternalStepRunner
- // - Enable 2 currently ignored tests in this module
- // - throw IllegalArgumentException here instead of warning
- logger.log(Level.WARNING, "Illegal port " + port + " in http server '" +
- spec.stringAttribute("id") + "'" +
- ": Port must be set to " + legalPortInHostedVespa);
+ throw new IllegalArgumentException("Illegal port " + port + " in http server '" +
+ spec.stringAttribute("id") + "'" +
+ ": Port must be set to " + legalPortInHostedVespa);
}
return port;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
index db831a1ec2f..562026ab4dd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
@@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
import com.yahoo.vespa.model.container.component.SimpleComponent;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
-import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider;
import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider;
+import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider;
import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider;
import org.w3c.dom.Element;
+import java.util.Arrays;
+import java.util.List;
import java.util.Optional;
+import static java.util.stream.Collectors.toList;
+
/**
* @author Einar M R Rosenvinge
* @author mortent
@@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file"));
Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file");
Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication");
+ List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites");
+ List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols");
return new ConfiguredFilebasedSslProvider(
serverName,
privateKeyFile,
certificateFile,
caCertificateFile.orElse(null),
- clientAuthentication.orElse(null));
+ clientAuthentication.orElse(null),
+ cipherSuites,
+ protocols);
} else if (sslProviderConfigurator != null) {
String className = sslProviderConfigurator.getAttribute("class");
String bundle = sslProviderConfigurator.getAttribute("bundle");
@@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
}
}
+ private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) {
+ return XmlHelper.getOptionalChildValue(sslElement, listElementName)
+ .map(element ->
+ Arrays.stream(element.split(","))
+ .filter(listEntry -> !listEntry.isBlank())
+ .map(String::trim)
+ .collect(toList()))
+ .orElse(List.of());
+ }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index e19d81e7fb2..4c4b1ca7f82 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -58,7 +58,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) {
for (AbstractSearchCluster searchCluster : searchClusters) {
if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue;
- owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster));
+ var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster);
+ owningCluster.addComponent(dispatcher);
}
}
@@ -130,7 +131,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
AbstractSearchCluster sys = findClusterWithId(searchClusters, i);
QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder().
name(sys.getClusterName());
- for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) {
+ for (AbstractSearchCluster.SchemaSpec spec : sys.getLocalSDS()) {
scB.searchdef(spec.getSearchDefinition().getName());
}
scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId()));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
index 704188e80e8..284aa3b46c0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
@@ -1,6 +1,7 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.search;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.vespa.config.search.DispatchConfig;
import com.yahoo.vespa.model.container.component.Component;
@@ -13,7 +14,7 @@ import com.yahoo.vespa.model.search.IndexedSearchCluster;
*
* @author bratseth
*/
-public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel>
+public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel>
implements DispatchConfig.Producer {
private final IndexedSearchCluster indexedSearchCluster;
@@ -21,14 +22,17 @@ public class DispatcherComponent extends Component<DispatcherComponent, Componen
public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) {
super(toComponentModel(indexedSearchCluster));
this.indexedSearchCluster = indexedSearchCluster;
+ String clusterName = indexedSearchCluster.getClusterName();
+ var rpcResoucePool = new RpcResourcePoolComponent(clusterName);
+ inject(rpcResoucePool);
+ addComponent(rpcResoucePool);
}
private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) {
String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher
return new ComponentModel(dispatcherComponentId,
- "com.yahoo.search.dispatch.Dispatcher",
- BundleMapper.searchAndDocprocBundle,
- null);
+ com.yahoo.search.dispatch.Dispatcher.class.getName(),
+ BundleMapper.searchAndDocprocBundle);
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
index 0abb0803405..0a9618e7b08 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
@@ -232,8 +232,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
return propB;
}
- private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(
- String fullName, Object value) {
+ private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(String fullName, Object value) {
QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder();
if (value instanceof SubstituteString)
value=value.toString(); // Send only types understood by configBuilder downwards
@@ -251,7 +250,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
qtB.matchaspath(true);
for (QueryProfileType inherited : profileType.inherited())
qtB.inherit(inherited.getId().stringValue());
- List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values());
+ List<FieldDescription> fields = new ArrayList<>(profileType.declaredFields().values());
Collections.sort(fields);
for (FieldDescription field : fields)
qtB.field(createConfig(field));
@@ -260,22 +259,20 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) {
QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder();
- fB.
- name(field.getName()).
- type(field.getType().stringValue());
+ fB.name(field.getName()).type(field.getType().stringValue());
if ( ! field.isOverridable())
fB.overridable(false);
if (field.isMandatory())
fB.mandatory(true);
- String aliases=toSpaceSeparatedString(field.getAliases());
- if (!aliases.isEmpty())
+ String aliases = toSpaceSeparatedString(field.getAliases());
+ if ( ! aliases.isEmpty())
fB.alias(aliases);
return fB;
}
public String toSpaceSeparatedString(List<String> list) {
- StringBuilder b=new StringBuilder();
- for (Iterator<String> i=list.iterator(); i.hasNext(); ) {
+ StringBuilder b = new StringBuilder();
+ for (Iterator<String> i = list.iterator(); i.hasNext(); ) {
b.append(i.next());
if (i.hasNext())
b.append(" ");
@@ -290,10 +287,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
}
}
- /**
- * The config produced by this
- * @return query profiles config
- */
+ /** Returns the config produced by this */
public QueryProfilesConfig getConfig() {
QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder();
getConfig(qB);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java
new file mode 100644
index 00000000000..2689c2ce71b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java
@@ -0,0 +1,18 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.xml.BundleMapper;
+
+public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> {
+
+ public RpcResourcePoolComponent(String clusterName) {
+ super(toComponentModel(clusterName));
+ }
+
+ private static ComponentModel toComponentModel(String clusterName) {
+ String componentId = "rpcresourcepool." + clusterName;
+ return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
index e05b2d27e09..4ecc666a9f2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
@@ -118,7 +118,7 @@ public class LocalProvider extends Provider implements
public List<String> getDocumentTypes() {
List<String> documentTypes = new ArrayList<>();
- for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) {
+ for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) {
documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName());
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
index 2828fcf09d0..19d1b6546a6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
@@ -9,6 +9,8 @@ import org.w3c.dom.Element;
import java.util.Arrays;
import java.util.List;
+import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS;
+
/**
* This object builds a bundle instantiation spec from an XML element.
*
@@ -36,7 +38,7 @@ public class BundleInstantiationSpecificationBuilder {
private static void validate(BundleInstantiationSpecification instSpec) {
List<String> forbiddenClasses = Arrays.asList(
- "com.yahoo.search.handler.SearchHandler",
+ SEARCH_HANDLER_CLASS,
"com.yahoo.processing.handler.ProcessingHandler");
for (String forbiddenClass: forbiddenClasses) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index aef2697a5dd..4bd9f5fa8b0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -19,12 +19,15 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
import com.yahoo.config.model.builder.xml.ConfigModelId;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.Zone;
@@ -57,9 +60,11 @@ import com.yahoo.vespa.model.container.SecretStore;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.chain.Chain;
import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.http.AccessControl;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
import com.yahoo.vespa.model.container.http.FilterChains;
import com.yahoo.vespa.model.container.http.Http;
@@ -88,6 +93,7 @@ import java.util.function.Consumer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID;
import static java.util.logging.Level.WARNING;
/**
@@ -106,6 +112,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
private static final String DEPRECATED_CONTAINER_TAG = "jdisc";
private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables";
+ static final String SEARCH_HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName();
+ static final String SEARCH_HANDLER_BINDING = "http://*/search/*";
+
public enum Networking { disable, enable }
private ApplicationPackage app;
@@ -175,7 +184,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
DocumentFactoryBuilder.buildDocumentFactories(cluster, spec);
addConfiguredComponents(deployState, cluster, spec);
addSecretStore(cluster, spec);
- addHandlers(deployState, cluster, spec);
addRestApis(deployState, spec, cluster);
addServlets(deployState, spec, cluster);
@@ -188,8 +196,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
cluster.addDefaultHandlersExceptStatus();
addStatusHandlers(cluster, context.getDeployState().isHosted());
+ addUserHandlers(deployState, cluster, spec);
- addHttp(deployState, spec, cluster, context.getApplicationType(), deployState.getProperties().applicationId().instance().isTester());
+ addHttp(deployState, spec, cluster, context);
addAccessLogs(deployState, cluster, spec);
addRoutingAliases(cluster, spec, deployState.zone().environment());
@@ -287,7 +296,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
private void addClientProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) {
for (Element clientSpec: XML.getChildren(spec, "client")) {
- cluster.addComponent(new DomClientProviderBuilder().build(deployState, cluster, clientSpec));
+ cluster.addComponent(new DomClientProviderBuilder(cluster).build(deployState, cluster, clientSpec));
}
}
@@ -302,7 +311,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent);
}
- if (accessLogElements.isEmpty() && cluster.getSearch() != null)
+ if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault())
cluster.addDefaultSearchAccessLog();
}
@@ -311,21 +320,23 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
}
- private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ApplicationType applicationType, boolean isTesterApplication) {
+ private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) {
Element httpElement = XML.getChild(spec, "http");
if (httpElement != null) {
cluster.setHttp(buildHttp(deployState, cluster, httpElement));
}
- if (deployState.isHosted() && applicationType == ApplicationType.DEFAULT && !isTesterApplication) {
+ if (isHostedTenantApplication(context)) {
+ addHostedImplicitHttpIfNotPresent(cluster);
+ addHostedImplicitAccessControlIfNotPresent(deployState, cluster);
addAdditionalHostedConnector(deployState, cluster);
}
}
private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster) {
- addImplicitHttpIfNotPresent(cluster);
- JettyHttpServer server = cluster.getHttp().getHttpServer();
+ JettyHttpServer server = cluster.getHttp().getHttpServer().get();
String serverName = server.getComponentId().getName();
+ String proxyProtocol = deployState.getProperties().proxyProtocol();
// If the deployment contains certificate/private key reference, setup TLS port
if (deployState.endpointCertificateSecrets().isPresent()) {
boolean authorizeClient = deployState.zone().system().isPublic();
@@ -334,27 +345,47 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
}
EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get();
HostedSslConnectorFactory connectorFactory = authorizeClient
- ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get())
- : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets);
+ ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(proxyProtocol, serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get())
+ : HostedSslConnectorFactory.withProvidedCertificate(proxyProtocol, serverName, endpointCertificateSecrets);
server.addConnector(connectorFactory);
} else {
- server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName));
+ server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(proxyProtocol, serverName));
}
}
- private static void addImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) {
+ private static boolean isHostedTenantApplication(ConfigModelContext context) {
+ var deployState = context.getDeployState();
+ boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester();
+ return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication;
+ }
+
+ private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) {
if(cluster.getHttp() == null) {
- Http http = new Http(Collections.emptyList());
- http.setFilterChains(new FilterChains(cluster));
- cluster.setHttp(http);
+ cluster.setHttp(new Http(new FilterChains(cluster)));
}
- if(cluster.getHttp().getHttpServer() == null) {
+ if(cluster.getHttp().getHttpServer().isEmpty()) {
JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"));
cluster.getHttp().setHttpServer(defaultHttpServer);
defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort()));
}
}
+ private void addHostedImplicitAccessControlIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) {
+ Http http = cluster.getHttp();
+ if (http.getAccessControl().isPresent()) return; // access control added explicitly
+ AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null);
+ if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure.
+ AccessControl accessControl =
+ new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger())
+ .setHandlers(cluster)
+ .readEnabled(false)
+ .writeEnabled(false)
+ .build();
+ http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID)));
+ http.setAccessControl(accessControl);
+ http.getBindings().addAll(accessControl.getBindings());
+ }
+
private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) {
Http http = new HttpBuilder().build(deployState, cluster, httpElement);
@@ -441,10 +472,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
containerSearch.setPageTemplates(PageTemplates.create(applicationPackage));
}
- private void addHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) {
+ private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) {
for (Element component: XML.getChildren(spec, "handler")) {
cluster.addComponent(
- new DomHandlerBuilder().build(deployState, cluster, component));
+ new DomHandlerBuilder(cluster).build(deployState, cluster, component));
}
}
@@ -532,14 +563,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted()));
}
+ /**
+ * Add nodes to cluster according to the given containerElement.
+ *
+ * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different set
+ * of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be changed
+ * simultaneously for all active config models.
+ */
private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) {
Element nodesElement = XML.getChild(containerElement, "nodes");
- if (nodesElement == null) { // default single node on localhost
- ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa());
- HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context);
- node.setHostResource(host);
- node.initService(context.getDeployLogger());
- cluster.addContainers(Collections.singleton(node));
+ if (nodesElement == null) {
+ cluster.addContainers(allocateWithoutNodesTag(cluster, context));
} else {
List<ApplicationContainer> nodes = createNodes(cluster, nodesElement, context);
@@ -615,30 +649,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
" must be an integer percentage ending by the '%' sign");
}
}
-
- /** Creates a single host when there is no nodes tag */
- private HostResource allocateSingleNodeHost(ApplicationContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) {
+
+ /** Allocate a container cluster without a nodes tag */
+ private List<ApplicationContainer> allocateWithoutNodesTag(ApplicationContainerCluster cluster, ConfigModelContext context) {
DeployState deployState = context.getDeployState();
HostSystem hostSystem = cluster.hostSystem();
if (deployState.isHosted()) {
- Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context);
- if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node
- return singleContentHost.get();
- }
- else { // request 1 node
- ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
- ClusterSpec.Id.from(cluster.getName()),
- deployState.getWantedNodeVespaVersion(),
- false);
- Capacity capacity = Capacity.fromCount(1,
- Optional.empty(),
- false,
- ! deployState.getProperties().isBootstrap());
- return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next();
- }
- } else {
- return hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC);
+ // request just enough nodes to satisfy environment capacity requirement
+ ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
+ ClusterSpec.Id.from(cluster.getName()))
+ .vespaVersion(deployState.getWantedNodeVespaVersion())
+ .dockerImageRepo(deployState.getWantedDockerImageRepo())
+ .build();
+ int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1;
+ Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified),
+ false,
+ !deployState.getProperties().isBootstrap());
+ var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log);
+ return createNodesFromHosts(log, hosts, cluster);
}
+ return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context);
+ }
+
+ private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) {
+ ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa());
+ node.setHostResource(host);
+ node.initService(context.getDeployLogger());
+ return List.of(node);
}
private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) {
@@ -652,13 +689,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
private List<ApplicationContainer> createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) {
NodeType type = NodeType.valueOf(nodesElement.getAttribute("type"));
- ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
- ClusterSpec.Id.from(cluster.getName()),
- context.getDeployState().getWantedNodeVespaVersion(),
- false);
+ ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()))
+ .vespaVersion(context.getDeployState().getWantedNodeVespaVersion())
+ .dockerImageRepo(context.getDeployState().getWantedDockerImageRepo())
+ .build();
Map<HostResource, ClusterMembership> hosts =
cluster.getRoot().hostSystem().allocateHosts(clusterSpec,
- Capacity.fromRequiredNodeType(type), 1, log);
+ Capacity.fromRequiredNodeType(type), log);
return createNodesFromHosts(context.getDeployLogger(), hosts, cluster);
}
@@ -766,7 +803,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(cluster.getSearch().getChains(),
"com.yahoo.search.handler.SearchHandler");
- String[] defaultBindings = {"http://*/search/*"};
+ String[] defaultBindings = {SEARCH_HANDLER_BINDING};
for (String binding: serverBindings(searchElement, defaultBindings)) {
searchHandler.addServerBindings(binding);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 75161859068..fcaba66ef69 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -16,8 +16,8 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
import com.yahoo.vespa.model.search.NodeSpec;
import com.yahoo.vespa.model.search.SearchCluster;
-import com.yahoo.vespa.model.search.SearchDefinition;
-import com.yahoo.vespa.model.search.SearchDefinitionXMLHandler;
+import com.yahoo.vespa.model.search.NamedSchema;
+import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler;
import com.yahoo.vespa.model.search.SearchNode;
import com.yahoo.vespa.model.search.StreamingSearchCluster;
import com.yahoo.vespa.model.search.TransactionLogServer;
@@ -52,6 +52,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
private final String clusterName;
private final Map<String, NewDocumentType> documentDefinitions;
private final Set<NewDocumentType> globallyDistributedDocuments;
+ private Double visibilityDelay = 0.0;
/** The search nodes of this if it does not have an indexed cluster */
private List<SearchNode> nonIndexed = new ArrayList<>();
@@ -135,21 +136,21 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem,
String clusterName, ContentSearchCluster search) {
- List<ModelElement> indexedDefs = getIndexedSearchDefinitions(clusterElem);
+ List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem);
if (!indexedDefs.isEmpty()) {
IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState);
isc.setRoutingSelector(clusterElem.childAsString("documents.selection"));
Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
if (visibilityDelay != null) {
- isc.setVisibilityDelay(visibilityDelay);
+ search.setVisibilityDelay(visibilityDelay);
}
search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs);
}
}
- private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) {
+ private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) {
List<ModelElement> indexedDefs = new ArrayList<>();
ModelElement docElem = clusterElem.child("documents");
if (docElem == null) {
@@ -179,29 +180,36 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
this.flushOnShutdown = flushOnShutdown;
}
+ public void setVisibilityDelay(double delay) {
+ this.visibilityDelay=delay;
+ if (hasIndexedCluster()) {
+ indexedCluster.setVisibilityDelay(delay);
+ }
+ }
+
private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) {
- addSearchDefinitions(deployState, documentDefs, cluster);
+ addSchemas(deployState, documentDefs, cluster);
if (queryTimeout != null) {
cluster.setQueryTimeout(queryTimeout);
}
cluster.defaultDocumentsConfig();
- cluster.deriveSearchDefinitions(deployState);
+ cluster.deriveSchemas(deployState);
addCluster(cluster);
}
- private void addSearchDefinitions(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) {
+ private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) {
for (ModelElement e : searchDefs) {
- SearchDefinitionXMLHandler searchDefinitionXMLHandler = new SearchDefinitionXMLHandler(e);
- SearchDefinition searchDefinition =
- searchDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSearchDefinitions());
+ SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e);
+ NamedSchema searchDefinition =
+ schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas());
if (searchDefinition == null)
throw new RuntimeException("Search definition parsing error or file does not exist: '" +
- searchDefinitionXMLHandler.getName() + "'");
+ schemaDefinitionXMLHandler.getName() + "'");
// TODO: remove explicit building of user configs when the complete content model is built using builders.
- sc.getLocalSDS().add(new AbstractSearchCluster.SearchDefinitionSpec(searchDefinition,
- UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger())));
+ sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition,
+ UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger())));
//need to get the document names from this sdfile
sc.addDocumentNames(searchDefinition);
}
@@ -307,7 +315,6 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
@Override
public void getConfig(ProtonConfig.Builder builder) {
- double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0;
builder.feeding.concurrency(0.40); // As if specified 0.8 in services.xml
boolean hasAnyNonIndexedCluster = false;
for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
index 0d15207b6ce..0f9eb5341ab 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
@@ -11,18 +11,25 @@ public class DispatchTuning {
public static final DispatchTuning empty = new DispatchTuning.Builder().build();
- public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE};
+ public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE}
private final Integer maxHitsPerPartition;
private DispatchPolicy dispatchPolicy;
private final Double minGroupCoverage;
private final Double minActiveDocsCoverage;
+ public Double getTopkProbability() {
+ return topkProbability;
+ }
+
+ private final Double topkProbability;
+
private DispatchTuning(Builder builder) {
maxHitsPerPartition = builder.maxHitsPerPartition;
dispatchPolicy = builder.dispatchPolicy;
minGroupCoverage = builder.minGroupCoverage;
minActiveDocsCoverage = builder.minActiveDocsCoverage;
+ topkProbability = builder.topKProbability;
}
/** Returns the max number of hits to fetch from each partition, or null to fetch all */
@@ -46,6 +53,7 @@ public class DispatchTuning {
private DispatchPolicy dispatchPolicy;
private Double minGroupCoverage;
private Double minActiveDocsCoverage;
+ private Double topKProbability;
public DispatchTuning build() {
return new DispatchTuning(this);
@@ -55,6 +63,10 @@ public class DispatchTuning {
this.maxHitsPerPartition = maxHitsPerPartition;
return this;
}
+ public Builder setTopKProbability(Double topKProbability) {
+ this.topKProbability = topKProbability;
+ return this;
+ }
public Builder setDispatchPolicy(String policy) {
if (policy != null)
dispatchPolicy = toDispatchPolicy(policy);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
index adfc703f747..f0c374b398f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -376,6 +376,10 @@ public class StorageGroup {
* specified using a group count attribute.
* <li>Neither element is present: Create a single node.
* </ul>
+ *
+ * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different
+ * set of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be
+ * changed simultaneously for all active config models.
*/
private GroupBuilder collectGroup(boolean isHosted, Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) {
StorageGroup group = new StorageGroup(
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 9b17412b83a..6dd3e619ec2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -136,10 +136,7 @@ public class ContentCluster extends AbstractConfigProducer implements
c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c);
validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup);
c.search.handleRedundancy(c.redundancy);
-
- IndexedSearchCluster index = c.search.getIndexed();
- if (index != null)
- setupIndexedCluster(index, contentElement, deployState.getDeployLogger());
+ setupSearchCluster(c.search, contentElement, deployState.getDeployLogger());
if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) )
throw new RuntimeException("Indexed search requires proton as engine");
@@ -166,17 +163,25 @@ public class ContentCluster extends AbstractConfigProducer implements
return c;
}
- private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element, DeployLogger logger) {
+ private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) {
ContentSearch search = DomContentSearchBuilder.build(element);
+ Double visibilityDelay = search.getVisibilityDelay();
+ if (visibilityDelay != null) {
+ csc.setVisibilityDelay(visibilityDelay);
+ }
+ if (csc.hasIndexedCluster()) {
+ setupIndexedCluster(csc.getIndexed(), search, element, logger);
+ }
+
+
+ }
+ private void setupIndexedCluster(IndexedSearchCluster index, ContentSearch search, ModelElement element, DeployLogger logger) {
Double queryTimeout = search.getQueryTimeout();
if (queryTimeout != null) {
Preconditions.checkState(index.getQueryTimeout() == null,
- "In " + index + ": You may not specify query-timeout in both proton and content.");
+ "In " + index + ": You may not specify query-timeout in both proton and content.");
index.setQueryTimeout(queryTimeout);
}
- Double visibilityDelay = search.getVisibilityDelay();
- if (visibilityDelay != null)
- index.setVisibilityDelay(visibilityDelay);
index.setSearchCoverage(DomSearchCoverageBuilder.build(element));
index.setDispatchSpec(DomDispatchBuilder.build(element));
@@ -281,7 +286,7 @@ public class ContentCluster extends AbstractConfigProducer implements
.orElse(NodesSpecification.nonDedicated(3, context));
Collection<HostResource> hosts = nodesSpecification.isDedicated() ?
getControllerHosts(nodesSpecification, admin, clusterName, context) :
- drawControllerHosts(nodesSpecification.count(), rootGroup, containers);
+ drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup, containers);
clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"),
hosts,
clusterName,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
index b53d66632a8..d599a1a1aca 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
@@ -23,6 +23,7 @@ public class DomTuningDispatchBuilder {
return builder.build();
}
builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition"));
+ builder.setTopKProbability(dispatchElement.childAsDouble("top-k-probability"));
builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy"));
builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage"));
builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
index 9662540e8df..2c6808b5773 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
@@ -20,8 +20,9 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer {
private final Map<Host, FileDistributionConfigProvider> fileDistributionConfigProviders = new IdentityHashMap<>();
private final FileDistributor fileDistributor;
- public FileDistributionConfigProducer(AbstractConfigProducer ancestor, FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpec) {
- this(ancestor, new FileDistributor(fileRegistry, configServerSpec));
+ public FileDistributionConfigProducer(AbstractConfigProducer ancestor, FileRegistry fileRegistry,
+ List<ConfigServerSpec> configServerSpec, boolean isHosted) {
+ this(ancestor, new FileDistributor(fileRegistry, configServerSpec, isHosted));
}
private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index 576b009c846..5ccf86f9ba8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -11,7 +11,10 @@ import com.yahoo.vespa.model.Host;
import java.util.*;
/**
- * Responsible for directing distribution of files to hosts.
+ * Sends RPC requests to hosts (tenant hosts and config servers) to start download of files. This is used during prepare
+ * of an application. Services themselves will also request files, the work done in this class is done so that hosts can
+ * start downloading files before services gets new config that needs these files. This also tries to make sure that
+ * all config servers (not just the one where the application was deployed) have the files available.
*
* @author Tony Vaagenes
*/
@@ -19,61 +22,42 @@ public class FileDistributor {
private final FileRegistry fileRegistry;
private final List<ConfigServerSpec> configServerSpecs;
+ private final boolean isHosted;
- /** A map from files to the hosts to which that file should be distributed */
+ /** A map from file reference to the hosts to which that file reference should be distributed */
private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>();
+ public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs, boolean isHosted) {
+ this.fileRegistry = fileRegistry;
+ this.configServerSpecs = configServerSpecs;
+ this.isHosted = isHosted;
+ }
+
/**
* Adds the given file to the associated application packages' registry of file and marks the file
- * for distribution to the given hosts.
+ * for distribution to the given host.
* <b>Note: This class receives ownership of the given collection.</b>
*
* @return the reference to the file, created by the application package
*/
- public FileReference sendFileToHosts(String relativePath, Collection<Host> hosts) {
- FileReference reference = fileRegistry.addFile(relativePath);
- addToFilesToDistribute(reference, hosts);
-
- return reference;
+ public FileReference sendFileToHost(String relativePath, Host host) {
+ return addFileReference(fileRegistry.addFile(relativePath), host);
}
/**
* Adds the given file to the associated application packages' registry of file and marks the file
- * for distribution to the given hosts.
+ * for distribution to the given host.
* <b>Note: This class receives ownership of the given collection.</b>
*
* @return the reference to the file, created by the application package
*/
- public FileReference sendUriToHosts(String uri, Collection<Host> hosts) {
- FileReference reference = fileRegistry.addUri(uri);
- if (reference != null) {
- addToFilesToDistribute(reference, hosts);
- }
-
- return reference;
- }
-
- /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */
- public FileReference sendFileToHost(String relativePath, Host host) {
- return sendFileToHosts(relativePath, Arrays.asList(host));
- }
-
public FileReference sendUriToHost(String uri, Host host) {
- return sendUriToHosts(uri, Arrays.asList(host));
- }
-
- private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) {
- Set<Host> oldHosts = getHosts(reference);
- oldHosts.addAll(hosts);
- }
-
- private Set<Host> getHosts(FileReference reference) {
- return filesToHosts.computeIfAbsent(reference, k -> new HashSet<>());
+ return addFileReference(fileRegistry.addUri(uri), host);
}
- public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs) {
- this.fileRegistry = fileRegistry;
- this.configServerSpecs = configServerSpecs;
+ private FileReference addFileReference(FileReference reference, Host host) {
+ filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host);
+ return reference;
}
/** Returns the files which has been marked for distribution to the given host */
@@ -107,16 +91,20 @@ public class FileDistributor {
// should only be called during deploy
public void sendDeployedFiles(FileDistribution dbHandler) {
String fileSourceHost = fileSourceHost();
- for (Host host : getTargetHosts()) {
- if ( ! host.getHostname().equals(fileSourceHost)) {
- dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host));
- }
- }
+
// Ask other config servers to download, for redundancy
- if (configServerSpecs != null)
- configServerSpecs.stream()
- .filter(configServerSpec -> !configServerSpec.getHostName().equals(fileSourceHost))
- .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend()));
+ configServerSpecs.stream()
+ .filter(spec -> !spec.getHostName().equals(fileSourceHost))
+ .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend()));
+
+ // Skip starting download for application hosts when on hosted, since this is just a hint and requests for files
+ // will fail until the application is activated (this call is done when preparing an application deployment)
+ // due to authorization of RPC requests on config servers only considering files belonging to active applications
+ if (isHosted) return;
+
+ getTargetHosts().stream()
+ .filter(host -> ! host.getHostname().equals(fileSourceHost))
+ .forEach(host -> dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host)));
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index 8a88e720bed..fe6c6c52e2d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -29,7 +29,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
protected int index;
private Double visibilityDelay = 0.0;
private List<String> documentNames = new ArrayList<>();
- private List<SearchDefinitionSpec> localSDS = new LinkedList<>();
+ private List<SchemaSpec> localSDS = new LinkedList<>();
public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
super(parent, "cluster." + clusterName);
@@ -38,11 +38,11 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
}
public void prepareToDistributeFiles(List<SearchNode> backends) {
- for (SearchDefinitionSpec sds : localSDS)
+ for (SchemaSpec sds : localSDS)
sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
}
- public void addDocumentNames(SearchDefinition searchDefinition) {
+ public void addDocumentNames(NamedSchema searchDefinition) {
String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName();
documentNames.add(dName);
}
@@ -50,7 +50,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
/** Returns a List with document names used in this search cluster */
public List<String> getDocumentNames() { return documentNames; }
- public List<SearchDefinitionSpec> getLocalSDS() {
+ public List<SchemaSpec> getLocalSDS() {
return localSDS;
}
@@ -107,18 +107,17 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
}
}
- public static final class SearchDefinitionSpec {
+ public static final class SchemaSpec {
- private final SearchDefinition searchDefinition;
+ private final NamedSchema searchDefinition;
private final UserConfigRepo userConfigRepo;
- public SearchDefinitionSpec(SearchDefinition searchDefinition,
- UserConfigRepo userConfigRepo) {
+ public SchemaSpec(NamedSchema searchDefinition, UserConfigRepo userConfigRepo) {
this.searchDefinition = searchDefinition;
this.userConfigRepo = userConfigRepo;
}
- public SearchDefinition getSearchDefinition() {
+ public NamedSchema getSearchDefinition() {
return searchDefinition;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 1b4c03a2182..56adc227df4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -53,6 +53,7 @@ public class IndexedSearchCluster extends SearchCluster
private final DispatchGroup rootDispatch;
private DispatchSpec dispatchSpec;
private final boolean useAdaptiveDispatch;
+ private final double defaultTopKProbability;
private List<SearchNode> searchNodes = new ArrayList<>();
/**
@@ -70,6 +71,7 @@ public class IndexedSearchCluster extends SearchCluster
unionCfg = new UnionConfiguration(this, documentDbs);
rootDispatch = new DispatchGroup(this);
useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch();
+ defaultTopKProbability = deployState.getProperties().defaultTopKProbability();
}
@Override
@@ -154,8 +156,7 @@ public class IndexedSearchCluster extends SearchCluster
private void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
ddbB.inputdoctypename(sdoc.getInputDocType())
- .configid(sdoc.getConfigId())
- .visibilitydelay(getVisibilityDelay());
+ .configid(sdoc.getConfigId());
}
@Override
@@ -196,13 +197,15 @@ public class IndexedSearchCluster extends SearchCluster
routingSelector = sb.toString();
}
}
+
@Override
- protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState) {
- for (SearchDefinitionSpec spec : localSearches) {
+ protected void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState) {
+ for (SchemaSpec spec : localSearches) {
com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch();
if ( ! (search instanceof DocumentOnlySearch)) {
DocumentDatabase db = new DocumentDatabase(this, search.getName(),
- new DerivedConfiguration(search, deployState.getDeployLogger(),
+ new DerivedConfiguration(search,
+ deployState.getDeployLogger(),
deployState.getProperties(),
deployState.rankProfileRegistry(),
deployState.getQueryProfiles().getRegistry(),
@@ -306,7 +309,11 @@ public class IndexedSearchCluster extends SearchCluster
}
if (useAdaptiveDispatch)
builder.distributionPolicy(DistributionPolicy.ADAPTIVE);
-
+ if (tuning.dispatch.getTopkProbability() != null) {
+ builder.topKProbability(tuning.dispatch.getTopkProbability());
+ } else {
+ builder.topKProbability(defaultTopKProbability);
+ }
if (tuning.dispatch.getMinActiveDocsCoverage() != null)
builder.minActivedocsPercentage(tuning.dispatch.getMinActiveDocsCoverage());
if (tuning.dispatch.getMinGroupCoverage() != null)
@@ -334,6 +341,7 @@ public class IndexedSearchCluster extends SearchCluster
if (searchCoverage.getMaxWaitAfterCoverageFactor() != null)
builder.maxWaitAfterCoverageFactor(searchCoverage.getMaxWaitAfterCoverageFactor());
}
+ builder.warmuptime(5.0);
}
@Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java
index 860f89792e2..ba81073709e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java
@@ -8,7 +8,8 @@ import java.util.Collection;
/**
* @author Tony Vaagenes
*/
-public class SearchDefinition {
+// TODO: This class is quite pointless
+public class NamedSchema {
private final Search search;
private final String name;
@@ -23,15 +24,15 @@ public class SearchDefinition {
return name;
}
- public SearchDefinition(String name, Search search) {
+ public NamedSchema(String name, Search search) {
this.name = name;
this.search = search;
}
//Find search definition from a collection with the name specified
- public static SearchDefinition findByName(final String searchDefinitionName, Collection<SearchDefinition> searchDefinitions) {
- for (SearchDefinition candidate : searchDefinitions) {
- if (candidate.getName().equals(searchDefinitionName) )
+ public static NamedSchema findByName(String schemaName, Collection<NamedSchema> schemas) {
+ for (NamedSchema candidate : schemas) {
+ if (candidate.getName().equals(schemaName) )
return candidate;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index c337c77d2a2..5d8b0b688d8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -30,6 +30,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
public void getConfig(ProtonConfig.Builder builder) {
setHwInfo(builder);
tuneDiskWriteSpeed(builder);
+ tuneRequestThreads(builder);
tuneDocumentStoreMaxFileSize(builder.summary.log);
tuneFlushStrategyMemoryLimits(builder.flush.memory);
tuneFlushStrategyTlsSize(builder.flush.memory);
@@ -105,6 +106,12 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
}
}
+ private void tuneRequestThreads(ProtonConfig.Builder builder) {
+ int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores());
+ builder.numsearcherthreads(numCores);
+ builder.numsummarythreads(numCores);
+ }
+
private void tuneWriteFilter(ProtonConfig.Writefilter.Builder builder) {
// "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy)
double reservedMemoryGb = 1;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java
index 1054253e3f0..b505b5e681c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java
@@ -7,15 +7,15 @@ import java.io.Serializable;
import java.util.List;
/**
- * Represents a single searchdefinition file.
+ * Represents a single schema file.
*
* @author arnej27959
*/
-public class SearchDefinitionXMLHandler implements Serializable {
+public class SchemaDefinitionXMLHandler implements Serializable {
private String sdName;
- public SearchDefinitionXMLHandler(ModelElement elem) {
+ public SchemaDefinitionXMLHandler(ModelElement elem) {
sdName = elem.stringAttribute("name");
if (sdName == null) {
sdName = elem.stringAttribute("type");
@@ -24,8 +24,8 @@ public class SearchDefinitionXMLHandler implements Serializable {
public String getName() { return sdName; }
- public SearchDefinition getResponsibleSearchDefinition(List<SearchDefinition> searchDefinitions) {
- return SearchDefinition.findByName( getName(), searchDefinitions );
+ public NamedSchema getResponsibleSearchDefinition(List<NamedSchema> schemas) {
+ return NamedSchema.findByName(getName(), schemas );
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
index 60b3cb6987c..0139e949c7a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -35,22 +35,14 @@ public abstract class SearchCluster extends AbstractSearchCluster
super(parent, clusterName, index);
}
- public void writeFiles(File directory) throws java.io.IOException {
- if (!directory.isDirectory() && !directory.mkdirs()) {
- throw new java.io.IOException("Cannot create directory: "+ directory);
- }
- writeSdFiles(directory);
- super.writeFiles(directory);
- }
-
/**
* Must be called after cluster is built, to derive SD configs
* Derives the search definitions from the application package..
* Also stores the document names contained in the search
* definitions.
*/
- public void deriveSearchDefinitions(DeployState deployState) {
- deriveAllSearchDefinitions(getLocalSDS(), deployState);
+ public void deriveSchemas(DeployState deployState) {
+ deriveAllSchemas(getLocalSDS(), deployState);
}
@Override
@@ -140,7 +132,7 @@ public abstract class SearchCluster extends AbstractSearchCluster
return false;
}
- protected abstract void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState);
+ protected abstract void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState);
public abstract void defaultDocumentsConfig();
public abstract DerivedConfiguration getSdConfig();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
index d668adea116..28e7b3eb37a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -91,7 +91,7 @@ public class StreamingSearchCluster extends SearchCluster implements
}
@Override
- protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> local, DeployState deployState) {
+ protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) {
if (local.size() == 1) {
deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState);
} else if (local.size() > 1){
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
index f5a91297e9e..f93bf6fc872 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
@@ -241,7 +241,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
public Integer level = null;
public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) {
- if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name));
+ if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name));
+ if (level != null) compression.level(level);
+ }
+
+ public void getConfig(ProtonConfig.Summary.Log.Compact.Compression.Builder compression) {
+ if (type != null) compression.type(ProtonConfig.Summary.Log.Compact.Compression.Type.Enum.valueOf(type.name));
if (level != null) compression.level(level);
}
@@ -281,6 +286,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
}
}
+ public void getConfig(ProtonConfig.Summary.Log.Compact.Builder compact) {
+ if (compression != null) {
+ compression.getConfig(compact.compression);
+ }
+ }
+
public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) {
if (outputInt) {
if (maxSize!=null) chunk.maxbytes(maxSize.intValue());
@@ -288,7 +299,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
throw new IllegalStateException("Fix this, chunk does not have long types");
}
if (compression != null) {
- compression.getConfig(chunk.compression);
+ compression.getConfig(chunk.compression);
}
}
}
@@ -303,6 +314,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor);
if (chunk != null) {
chunk.getConfig(log.chunk);
+ chunk.getConfig(log.compact);
}
}
}
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 52665ff56a9..3560cf2cd84 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -197,6 +197,7 @@ TOKEN :
< NL: "\n" >
| < ANNOTATION: "annotation" >
| < ANNOTATIONREFERENCE: "annotationreference" >
+| < SCHEMA: "schema" >
| < SEARCH: "search" >
| < DIVERSITY: "diversity" >
| < MIN_GROUPS: "min-groups" >
@@ -332,6 +333,10 @@ TOKEN :
| < UPPERBOUND: "upper-bound" >
| < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" >
| < ENABLE_BM25: "enable-bm25" >
+| < HNSW: "hnsw" >
+| < MAXLINKSPERNODE: "max-links-per-node" >
+| < DISTANCEMETRIC: "distance-metric" >
+| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
| < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? >
| < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
| < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? >
@@ -400,38 +405,38 @@ Search search(DocumentTypeManager docMan, String dir) :
Search search;
}
{
- (<NL>)* (search = rootSearch(dir) | search = rootDocument(dir))
+ (<NL>)* (search = rootSchema(dir) | search = rootDocument(dir))
{ return search; }
}
/**
- * This rule consumes a proper search block. This and rootDocument() are the only rules that should ever consume
+ * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume
* trailing newline tokens.
*
- * @param dir The directory containing the file being parsed.
- * @return The search definition object.
+ * @param dir the directory containing the file being parsed.
+ * @return the schema definition object.
*/
-Search rootSearch(String dir) :
+Search rootSchema(String dir) :
{
String name;
Search search;
}
{
- ( <SEARCH> name = identifier() { search = new Search(name, app);
+ ( ( <SCHEMA> | <SEARCH> ) name = identifier() { search = new Search(name, app);
rankProfileRegistry.add(new DefaultRankProfile(search, rankProfileRegistry));
rankProfileRegistry.add(new UnrankedRankProfile(search, rankProfileRegistry));}
- lbrace() (rootSearchItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
+ lbrace() (rootSchemaItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
{ return search; }
}
/**
- * Consumes an element of a search block. This and rootSearch() are the only rules that should ever consume
+ * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume
* trailing newline tokens.
*
* @param search The search object to modify.
* @return Null.
*/
-Object rootSearchItem(Search search) : { }
+Object rootSchemaItem(Search search) : { }
{
( document(search)
| documentSummary(search)
@@ -449,10 +454,10 @@ Object rootSearchItem(Search search) : { }
}
/**
- * Consumes a search definition that contains only documents to be used for inheritance, etc.
+ * Consumes a schema definition that contains only documents to be used for inheritance, etc.
*
- * @param dir The directory containing the file being parsed.
- * @return The search definition object.
+ * @param dir the directory containing the file being parsed.
+ * @return the schema definition object.
*/
Search rootDocument(String dir) :
{
@@ -478,7 +483,7 @@ Object rootDocumentItem(Search search) : { }
/**
* Consumes a use-document statement. This currently does nothing.
*
- * @param search The search object to modify.
+ * @param search the search object to modify.
*/
void useDocument(Search search) : { }
{
@@ -488,7 +493,7 @@ void useDocument(Search search) : { }
/**
* Consumes a document element. The name defaults to the search's name, but may be set.
*
- * @param search The search object to add content to.
+ * @param search the search object to add content to.
*/
void document(Search search) :
{
@@ -507,7 +512,7 @@ void document(Search search) :
/**
* Consumes a document element, explicitly named
*
- * @param search The search object to add content to.
+ * @param search the search object to add content to.
*/
void namedDocument(Search search) :
{
@@ -637,7 +642,7 @@ void field(SDDocumentType document, Search search) :
if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) {
throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
}
- field = new TemporarySDField(name, type, true, document);
+ field = new TemporarySDField(name, type, document);
}
lbrace() (fieldBody(field, search, document) (<NL>)*)* <RBRACE>
{
@@ -651,27 +656,35 @@ void field(SDDocumentType document, Search search) :
void fieldSet(Search search) :
{
- String name;
-}
-{
- <FIELDSET> name = identifier() lbrace()
- (fieldSetItem(name, search)(<NL>)*)+
- <RBRACE>
-}
-
-void fieldSetItem(String setName, Search search) :
-{
+ String setName;
String field;
String queryCommand;
- SDField matchSettings = new SDField(setName, DataType.STRING); // match etc for fieldset represented as SDField or ease of parsing
+ List queryCommands = new ArrayList();
+ FieldOperationContainer matchSetting;
+ List matchSettings = new ArrayList();
}
{
- ( <FIELDS><COLON> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); }
- ( <COMMA> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* )
- |
- ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { search.fieldSets().userFieldSets().get(setName).queryCommands().add(queryCommand);}
- |
- ( match(matchSettings) ) { matchSettings.applyOperations(); search.fieldSets().userFieldSets().get(setName).setMatching(matchSettings.getMatching());}
+ <FIELDSET> setName = identifier() lbrace()
+ ((
+ ( <FIELDS><COLON> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); }
+ ( <COMMA> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* )
+ |
+ ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); }
+ |
+ ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
+ )(<NL>)*)+
+ <RBRACE>
+ {
+ // Apply settings after parsing since all user field items must be set first
+
+ for (Object command : queryCommands)
+ search.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command);
+
+ for (Object setting : matchSettings) {
+ ((SDField)setting).applyOperations();
+ search.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching());
+ }
+ }
}
/**
@@ -923,7 +936,7 @@ void structFieldDefinition(SDDocumentType struct) :
if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) {
throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
}
- field = new TemporarySDField(name, type, true, struct);
+ field = new TemporarySDField(name, type, struct);
struct.addField(field);
}
lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> {
@@ -1531,7 +1544,7 @@ void queryCommand(FieldOperationContainer container) :
QueryCommandOperation field = new QueryCommandOperation();
}
{
- <QUERYCOMMAND> <COLON> command = identifierWithDash()
+ <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() )
{
field.addQueryCommand(command);
container.addOperation(field);
@@ -1551,11 +1564,11 @@ void alias(FieldOperationContainer container) :
}
}
-Object match(FieldOperationContainer field) : { }
+FieldOperationContainer match(FieldOperationContainer field) : { }
{
<MATCH> ( (<COLON> matchType(field))
| (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) )
- { return null; }
+ { return field; }
}
/**
@@ -1803,10 +1816,35 @@ Object indexBody(IndexOperation index) :
| <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); }
| <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); }
| <ENABLE_BM25> { index.setEnableBm25(true); }
+ | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { index.setDistanceMetric(str); }
+ | hnswIndex(index) { }
)
{ return null; }
}
+void hnswIndex(IndexOperation index) :
+{
+ HnswIndexParams.Builder params = new HnswIndexParams.Builder();
+}
+{
+ ( LOOKAHEAD(<HNSW> lbrace())
+ <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) |
+ <HNSW> )
+ {
+ index.setHnswIndexParams(params);
+ }
+}
+
+void hnswIndexBody(HnswIndexParams.Builder params) :
+{
+ int num;
+ String str;
+}
+{
+ ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); }
+ | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } )
+}
+
/**
* Consumes a constant block of a search element.
*
@@ -2583,6 +2621,7 @@ String identifier() : { }
| <REFERENCE>
| <REMOVEIFZERO>
| <RERANKCOUNT>
+ | <SCHEMA>
| <SEARCH>
| <SECONDARY>
| <SECONDPHASE>
diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy
index 59a84f5b0c0..a128e4a8d4c 100755
--- a/config-model/src/main/perl/vespa-deploy
+++ b/config-model/src/main/perl/vespa-deploy
@@ -154,7 +154,7 @@ my $command = shift;
$command ||= "help";
# The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl
-my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper --insecure -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';
+my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';
my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT';
my $CURL_GET = $curl_command . ' --request GET';
diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc
index 7a3e2916f94..1eac11a879c 100644
--- a/config-model/src/main/resources/schema/admin.rnc
+++ b/config-model/src/main/resources/schema/admin.rnc
@@ -82,10 +82,27 @@ Metrics = element metrics {
element metric {
attribute id { xsd:Name } &
attribute display-name { xsd:Name }?
- }*
+ }* &
+ Cloudwatch?
}+
}
+Cloudwatch = element cloudwatch {
+ attribute region { xsd:Name } &
+ attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } &
+ (
+ element credentials {
+ attribute access-key-name { xsd:Name } &
+ attribute secret-key-name { xsd:Name }
+ }
+ |
+ element shared-credentials {
+ attribute file { string } &
+ attribute profile { xsd:Name }?
+ }
+ )?
+}
+
ClusterControllers = element cluster-controllers {
attribute standalone-zookeeper { xsd:string }? &
element cluster-controller {
diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc
index e3ad942e7b3..878faabfec1 100644
--- a/config-model/src/main/resources/schema/common.rnc
+++ b/config-model/src/main/resources/schema/common.rnc
@@ -17,14 +17,14 @@ anyElement = element * {
JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" }
Nodes = element nodes {
- attribute count { xsd:positiveInteger } &
+ attribute count { xsd:positiveInteger | xsd:string } &
attribute flavor { xsd:string }? &
attribute docker-image { xsd:string }? &
Resources?
}
Resources = element resources {
- attribute vcpu { xsd:double { minExclusive = "0.0" } } &
+ attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } &
attribute memory { xsd:string } &
attribute disk { xsd:string } &
attribute disk-speed { xsd:string }? &
@@ -32,12 +32,13 @@ Resources = element resources {
}
OptionalDedicatedNodes = element nodes {
- attribute count { xsd:positiveInteger } &
+ attribute count { xsd:positiveInteger | xsd:string } &
attribute flavor { xsd:string }? &
attribute required { xsd:boolean }? &
attribute docker-image { xsd:string }? &
attribute dedicated { xsd:boolean }? &
- attribute exclusive { xsd:boolean }?
+ attribute exclusive { xsd:boolean }? &
+ Resources?
}
GenericConfig = element config {
diff --git a/config-model/src/main/resources/schema/container.rnc b/config-model/src/main/resources/schema/container.rnc
index 3f0e1f626ac..034de1113c2 100644
--- a/config-model/src/main/resources/schema/container.rnc
+++ b/config-model/src/main/resources/schema/container.rnc
@@ -23,11 +23,10 @@ Server = element server {
}
AccessControl = element access-control {
- attribute domain { xsd:NCName } &
- attribute read { string "true" | string "false" }? &
- attribute write { string "true" | string "false" }? &
- element vespa-domain { xsd:NCName }? &
- element application { xsd:NCName }? &
+ attribute domain { xsd:NCName }? & # TODO Vespa 8 Remove
+ attribute read { string "true" | string "false" }? & # TODO Vespa 8 Remove
+ attribute write { string "true" | string "false" }? & # TODO Vespa 8 Remove
+ element vespa-domain { xsd:NCName }? & # TODO Remove after end of March 2020
element exclude {
Binding+
}?
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index 6e4346d96ee..3c8b60fb84b 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -96,7 +96,9 @@ Ssl = element ssl {
element private-key-file { string } &
element certificate-file { string } &
element ca-certificates-file { string }? &
- element client-authentication { string "disabled" | string "want" | string "need" }?
+ element client-authentication { string "disabled" | string "want" | string "need" }? &
+ element cipher-suites { string }? &
+ element protocols { string }?
}
SslProvider = element ssl-provider {
@@ -237,7 +239,7 @@ NodesOfContainerCluster = element nodes {
attribute type { xsd:string }
|
(
- attribute count { xsd:positiveInteger } &
+ attribute count { xsd:positiveInteger | xsd:string } &
attribute flavor { xsd:string }? &
attribute required { xsd:boolean }? &
attribute exclusive { xsd:boolean }? &
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 8b3868c132e..481d82ebb4b 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -57,11 +57,14 @@ PersistenceThread = element thread {
## Declare which storage threads each disk should have.
PersistenceThreads = element persistence-threads {
+ ## The number of threads to create
+ attribute count { xsd:integer }? &
+ ## All of the below settings are deprecated.
## Operations with priority worse than this can be blocked
attribute highest-priority-to-block { xsd:string } ? &
## Operations with priority better than this can block others
attribute lowest-priority-to-block-others { xsd:string } ? &
- Thread+
+ Thread*
}
MinNodeRatioPerGroup = element min-node-ratio-per-group {
@@ -82,6 +85,7 @@ DispatchTuning = element dispatch {
element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? &
element min-group-coverage { xsd:double }? &
element min-active-docs-coverage { xsd:double }? &
+ element top-k-probability { xsd:double }? &
element use-local-node { string "true" | string "false" }?
}
@@ -218,11 +222,11 @@ ContentNodes = element nodes {
attribute vespamalloc-debug-stacktrace { xsd:string }? &
(
(
- attribute count { xsd:positiveInteger } &
+ attribute count { xsd:positiveInteger | xsd:string } &
attribute flavor { xsd:string }? &
attribute required { xsd:boolean }? &
attribute docker-image { xsd:string }? &
- attribute groups { xsd:positiveInteger }?
+ attribute groups { xsd:positiveInteger | xsd:string }?
)
|
ContentNode +
@@ -263,12 +267,12 @@ Group = element group {
|
(
element nodes {
- attribute count { xsd:positiveInteger } &
+ attribute count { xsd:positiveInteger | xsd:string } &
attribute flavor { xsd:string }? &
attribute required { xsd:boolean }? &
attribute exclusive { xsd:boolean }? &
attribute docker-image { xsd:string }? &
- attribute groups { xsd:positiveInteger }?
+ attribute groups { xsd:positiveInteger | xsd:string }?
}
)
|
diff --git a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg b/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg
deleted file mode 100644
index 6e18bdf64e7..00000000000
--- a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace=config
-foo bar
-baz []678
diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
index ab5e42f983d..247df8a0241 100644
--- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
+++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
@@ -33,8 +33,12 @@ search test {
expression: xgboost("xgboost_2_2")
}
+ function my_lightgbm() {
+ expression: lightgbm("lightgbm_regression")
+ }
+
first-phase {
- expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost
+ expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm
}
}
diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index c5b1dfef610..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19125
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 7aed9bdc244..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19131
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 49e5f59b9be..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19137
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 8d5d4fdde7f..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19143
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index be7d8e44c16..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19110
-slobrok.name "search/cluster.music/rtx/0/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg
deleted file mode 100644
index f39dc6adc03..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19112
-slobrok.name "search/cluster.music/rtx/1/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 5c9f46bf8ce..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19156
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 69f91eab48c..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19105
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg
deleted file mode 100644
index eebb1cb6b40..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19111
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 82e9aafc5a8..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19162
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index aa557a9ae04..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19144
-slobrok.name "search/cluster.rt/rtx/0/clustercontroller"
-slobrok.config search/cluster.rt/rtx
-servicemonitor.autodisable true
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg
deleted file mode 100644
index 1407c3ff209..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19146
-slobrok.name "search/cluster.rt/rtx/1/clustercontroller"
-slobrok.config search/cluster.rt/rtx
-servicemonitor.autodisable true
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index c5b1dfef610..00000000000
--- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19125
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index d0888e9be96..00000000000
--- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19115
-slobrok.name "search/cluster.music/rtx/0/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 53dcc3f9686..00000000000
--- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19118
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index 474b8c68cdb..00000000000
--- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19108
-slobrok.name "search/cluster.music/rtx/0/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 6dabdfc6af7..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19115
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index 3c1b537236a..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19105
-slobrok.name "search/cluster.music1/rtx/0/clustercontroller"
-slobrok.config search/cluster.music1/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index b6d62fc678b..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19126
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 887eaa6b634..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19132
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index c00fcc456d6..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19116
-slobrok.name "search/cluster.music2/rtx/0/clustercontroller"
-slobrok.config search/cluster.music2/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7b50176625d
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
@@ -0,0 +1,104 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id 595216861
+datatype[1].referencetype[0].target_type_id -1318255918
+datatype[2].id 542332920
+datatype[2].referencetype[0].target_type_id 443162583
+datatype[3].id 959075962
+datatype[3].structtype[0].name "ad.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].field[0].name "campaign_ref"
+datatype[3].structtype[0].field[0].datatype 595216861
+datatype[3].structtype[0].field[0].detailedtype ""
+datatype[3].structtype[0].field[1].name "person_ref"
+datatype[3].structtype[0].field[1].datatype 542332920
+datatype[3].structtype[0].field[1].detailedtype ""
+datatype[4].id -255288561
+datatype[4].structtype[0].name "ad.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compressminsize 800
+datatype[5].id 2987301
+datatype[5].documenttype[0].name "ad"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0].name "document"
+datatype[5].documenttype[0].inherits[0].version 0
+datatype[5].documenttype[0].headerstruct 959075962
+datatype[5].documenttype[0].bodystruct -255288561
+datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+datatype[5].documenttype[0].importedfield[0].name "my_cool_field"
+datatype[5].documenttype[0].importedfield[1].name "my_swag_field"
+datatype[5].documenttype[0].importedfield[2].name "my_name"
+datatype[6].id -2041471955
+datatype[6].structtype[0].name "campaign.header"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "cool_field"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].detailedtype ""
+datatype[6].structtype[0].field[1].name "swag_field"
+datatype[6].structtype[0].field[1].datatype 4
+datatype[6].structtype[0].field[1].detailedtype ""
+datatype[7].id 1448849794
+datatype[7].structtype[0].name "campaign.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[8].id -1318255918
+datatype[8].documenttype[0].name "campaign"
+datatype[8].documenttype[0].version 0
+datatype[8].documenttype[0].inherits[0].name "document"
+datatype[8].documenttype[0].inherits[0].version 0
+datatype[8].documenttype[0].headerstruct -2041471955
+datatype[8].documenttype[0].bodystruct 1448849794
+datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field"
+datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field"
+datatype[9].id 3129224
+datatype[9].structtype[0].name "person.header"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "name"
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].detailedtype ""
+datatype[10].id -2003767395
+datatype[10].structtype[0].name "person.body"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].compresstype NONE
+datatype[10].structtype[0].compresslevel 0
+datatype[10].structtype[0].compressthreshold 95
+datatype[10].structtype[0].compressminsize 800
+datatype[11].id 443162583
+datatype[11].documenttype[0].name "person"
+datatype[11].documenttype[0].version 0
+datatype[11].documenttype[0].inherits[0].name "document"
+datatype[11].documenttype[0].inherits[0].version 0
+datatype[11].documenttype[0].headerstruct 3129224
+datatype[11].documenttype[0].bodystruct -2003767395
+datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name" \ No newline at end of file
diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7859703ffe0
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
@@ -0,0 +1,141 @@
+enablecompression false
+documenttype[0].id 2987301
+documenttype[0].name "ad"
+documenttype[0].version 0
+documenttype[0].headerstruct 959075962
+documenttype[0].bodystruct -255288561
+documenttype[0].inherits[0].id 8
+documenttype[0].datatype[0].id 959075962
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name "ad.header"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref"
+documenttype[0].datatype[0].sstruct.field[0].id 23963250
+documenttype[0].datatype[0].sstruct.field[0].datatype 595216861
+documenttype[0].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[0].datatype[0].sstruct.field[1].name "person_ref"
+documenttype[0].datatype[0].sstruct.field[1].id 100779805
+documenttype[0].datatype[0].sstruct.field[1].datatype 542332920
+documenttype[0].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[0].datatype[1].id -255288561
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "ad.body"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+documenttype[0].referencetype[0].id 595216861
+documenttype[0].referencetype[0].target_type_id -1318255918
+documenttype[0].referencetype[1].id 542332920
+documenttype[0].referencetype[1].target_type_id 443162583
+documenttype[0].importedfield[0].name "my_cool_field"
+documenttype[0].importedfield[1].name "my_swag_field"
+documenttype[0].importedfield[2].name "my_name"
+documenttype[1].id -1318255918
+documenttype[1].name "campaign"
+documenttype[1].version 0
+documenttype[1].headerstruct -2041471955
+documenttype[1].bodystruct 1448849794
+documenttype[1].inherits[0].id 8
+documenttype[1].datatype[0].id -2041471955
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "campaign.header"
+documenttype[1].datatype[0].sstruct.version 0
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 95
+documenttype[1].datatype[0].sstruct.compression.minsize 200
+documenttype[1].datatype[0].sstruct.field[0].name "cool_field"
+documenttype[1].datatype[0].sstruct.field[0].id 1588702436
+documenttype[1].datatype[0].sstruct.field[0].datatype 2
+documenttype[1].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[1].datatype[0].sstruct.field[1].name "swag_field"
+documenttype[1].datatype[0].sstruct.field[1].id 1691224741
+documenttype[1].datatype[0].sstruct.field[1].datatype 4
+documenttype[1].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[1].datatype[1].id 1448849794
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "campaign.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 95
+documenttype[1].datatype[1].sstruct.compression.minsize 200
+documenttype[1].fieldsets{[document]}.fields[0] "cool_field"
+documenttype[1].fieldsets{[document]}.fields[1] "swag_field"
+documenttype[2].id 443162583
+documenttype[2].name "person"
+documenttype[2].version 0
+documenttype[2].headerstruct 3129224
+documenttype[2].bodystruct -2003767395
+documenttype[2].inherits[0].id 8
+documenttype[2].datatype[0].id 3129224
+documenttype[2].datatype[0].type STRUCT
+documenttype[2].datatype[0].array.element.id 0
+documenttype[2].datatype[0].map.key.id 0
+documenttype[2].datatype[0].map.value.id 0
+documenttype[2].datatype[0].wset.key.id 0
+documenttype[2].datatype[0].wset.createifnonexistent false
+documenttype[2].datatype[0].wset.removeifzero false
+documenttype[2].datatype[0].annotationref.annotation.id 0
+documenttype[2].datatype[0].sstruct.name "person.header"
+documenttype[2].datatype[0].sstruct.version 0
+documenttype[2].datatype[0].sstruct.compression.type NONE
+documenttype[2].datatype[0].sstruct.compression.level 0
+documenttype[2].datatype[0].sstruct.compression.threshold 95
+documenttype[2].datatype[0].sstruct.compression.minsize 200
+documenttype[2].datatype[0].sstruct.field[0].name "name"
+documenttype[2].datatype[0].sstruct.field[0].id 1160796772
+documenttype[2].datatype[0].sstruct.field[0].datatype 2
+documenttype[2].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[2].datatype[1].id -2003767395
+documenttype[2].datatype[1].type STRUCT
+documenttype[2].datatype[1].array.element.id 0
+documenttype[2].datatype[1].map.key.id 0
+documenttype[2].datatype[1].map.value.id 0
+documenttype[2].datatype[1].wset.key.id 0
+documenttype[2].datatype[1].wset.createifnonexistent false
+documenttype[2].datatype[1].wset.removeifzero false
+documenttype[2].datatype[1].annotationref.annotation.id 0
+documenttype[2].datatype[1].sstruct.name "person.body"
+documenttype[2].datatype[1].sstruct.version 0
+documenttype[2].datatype[1].sstruct.compression.type NONE
+documenttype[2].datatype[1].sstruct.compression.level 0
+documenttype[2].datatype[1].sstruct.compression.threshold 95
+documenttype[2].datatype[1].sstruct.compression.minsize 200
+documenttype[2].fieldsets{[document]}.fields[0] "name" \ No newline at end of file
diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg
index 97f480745cf..0a76e44c4ac 100644
--- a/config-model/src/test/derived/advanced/attributes.cfg
+++ b/config-model/src/test/derived/advanced/attributes.cfg
@@ -19,3 +19,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
index 56fd15f9f5d..006400c09d4 100644
--- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
+++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "elem_array.weight"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -40,3 +44,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg
index 022bdbd31a4..cd91c15700b 100644
--- a/config-model/src/test/derived/attributeprefetch/attributes.cfg
+++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "multibyte"
attribute[].datatype INT8
attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "wsbyte"
attribute[].datatype INT8
attribute[].collectiontype WEIGHTEDSET
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "singleint"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "multiint"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "wsint"
attribute[].datatype INT32
attribute[].collectiontype WEIGHTEDSET
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "singlelong"
attribute[].datatype INT64
attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "multilong"
attribute[].datatype INT64
attribute[].collectiontype ARRAY
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "wslong"
attribute[].datatype INT64
attribute[].collectiontype WEIGHTEDSET
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "singlefloat"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "multifloat"
attribute[].datatype FLOAT
attribute[].collectiontype ARRAY
@@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "wsfloat"
attribute[].datatype FLOAT
attribute[].collectiontype WEIGHTEDSET
@@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "singledouble"
attribute[].datatype DOUBLE
attribute[].collectiontype SINGLE
@@ -271,6 +319,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "multidouble"
attribute[].datatype DOUBLE
attribute[].collectiontype ARRAY
@@ -292,6 +344,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "wsdouble"
attribute[].datatype DOUBLE
attribute[].collectiontype WEIGHTEDSET
@@ -313,6 +369,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "singlestring"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -334,6 +394,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "multistring"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -355,6 +419,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "wsstring"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -376,3 +444,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg
index 7a21001a9ed..29e4c209ef2 100644
--- a/config-model/src/test/derived/attributes/attributes.cfg
+++ b/config-model/src/test/derived/attributes/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a2"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a3"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a5"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a6"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b1"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b2"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b3"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b4"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b5"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b6"
attribute[].datatype INT64
attribute[].collectiontype ARRAY
@@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b7"
attribute[].datatype DOUBLE
attribute[].collectiontype WEIGHTEDSET
@@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a9"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -271,6 +319,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a10"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -292,6 +344,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a11"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -313,6 +369,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a12"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -334,6 +394,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a7_arr"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -355,6 +419,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "a8_arr"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -376,3 +444,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg
index dda746da5b4..5606e5ea730 100644
--- a/config-model/src/test/derived/complex/attributes.cfg
+++ b/config-model/src/test/derived/complex/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "fleeting"
attribute[].datatype FLOAT
attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "fleeting2"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "foundat"
attribute[].datatype INT64
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "collapseby"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "ts"
attribute[].datatype INT64
attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "combineda"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "year_arr"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "year_sub"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -187,3 +219,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg
new file mode 100644
index 00000000000..b61fd7e2ee5
--- /dev/null
+++ b/config-model/src/test/derived/hnsw_index/attributes.cfg
@@ -0,0 +1,25 @@
+attribute[].name "t1"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor(x[128])"
+attribute[].imported false
+attribute[].index.hnsw.enabled true
+attribute[].index.hnsw.maxlinkspernode 32
+attribute[].index.hnsw.distancemetric ANGULAR
+attribute[].index.hnsw.neighborstoexploreatinsert 300
diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg
new file mode 100644
index 00000000000..e9fc265ca67
--- /dev/null
+++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg
@@ -0,0 +1,5 @@
+maxtermoccurrences 100
+fieldmatchmaxlength 1000000
+ilscript[].doctype "test"
+ilscript[].docfield[] "t1"
+ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }"
diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd
new file mode 100644
index 00000000000..207ed764a87
--- /dev/null
+++ b/config-model/src/test/derived/hnsw_index/test.sd
@@ -0,0 +1,14 @@
+search test {
+ document test {
+ field t1 type tensor(x[128]) {
+ indexing: attribute | index
+ index {
+ distance-metric: angular
+ hnsw {
+ max-links-per-node: 32
+ neighbors-to-explore-at-insert: 300
+ }
+ }
+ }
+ }
+}
diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg
index db2280a7846..8f68068c8e7 100644
--- a/config-model/src/test/derived/imported_position_field/attributes.cfg
+++ b/config-model/src/test/derived/imported_position_field/attributes.cfg
@@ -1,42 +1,50 @@
-attribute[0].name "parent_ref"
-attribute[0].datatype REFERENCE
-attribute[0].collectiontype SINGLE
-attribute[0].removeifzero false
-attribute[0].createifnonexistent false
-attribute[0].fastsearch false
-attribute[0].huge false
-attribute[0].ismutable false
-attribute[0].sortascending true
-attribute[0].sortfunction UCA
-attribute[0].sortstrength PRIMARY
-attribute[0].sortlocale ""
-attribute[0].enablebitvectors false
-attribute[0].enableonlybitvector false
-attribute[0].fastaccess false
-attribute[0].arity 8
-attribute[0].lowerbound -9223372036854775808
-attribute[0].upperbound 9223372036854775807
-attribute[0].densepostinglistthreshold 0.4
-attribute[0].tensortype ""
-attribute[0].imported false
-attribute[1].name "my_pos_zcurve"
-attribute[1].datatype INT64
-attribute[1].collectiontype SINGLE
-attribute[1].removeifzero false
-attribute[1].createifnonexistent false
-attribute[1].fastsearch true
-attribute[1].huge false
-attribute[1].ismutable false
-attribute[1].sortascending true
-attribute[1].sortfunction UCA
-attribute[1].sortstrength PRIMARY
-attribute[1].sortlocale ""
-attribute[1].enablebitvectors false
-attribute[1].enableonlybitvector false
-attribute[1].fastaccess false
-attribute[1].arity 8
-attribute[1].lowerbound -9223372036854775808
-attribute[1].upperbound 9223372036854775807
-attribute[1].densepostinglistthreshold 0.4
-attribute[1].tensortype ""
-attribute[1].imported true \ No newline at end of file
+attribute[].name "parent_ref"
+attribute[].datatype REFERENCE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_pos_zcurve"
+attribute[].datatype INT64
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg
index ce6ff5e54ae..eb87c6cf53e 100644
--- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg
+++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg
@@ -1,168 +1,200 @@
-attribute[0].name "parent_ref"
-attribute[0].datatype REFERENCE
-attribute[0].collectiontype SINGLE
-attribute[0].removeifzero false
-attribute[0].createifnonexistent false
-attribute[0].fastsearch false
-attribute[0].huge false
-attribute[0].ismutable false
-attribute[0].sortascending true
-attribute[0].sortfunction UCA
-attribute[0].sortstrength PRIMARY
-attribute[0].sortlocale ""
-attribute[0].enablebitvectors false
-attribute[0].enableonlybitvector false
-attribute[0].fastaccess false
-attribute[0].arity 8
-attribute[0].lowerbound -9223372036854775808
-attribute[0].upperbound 9223372036854775807
-attribute[0].densepostinglistthreshold 0.4
-attribute[0].tensortype ""
-attribute[0].imported false
-attribute[1].name "my_elem_array.name"
-attribute[1].datatype STRING
-attribute[1].collectiontype SINGLE
-attribute[1].removeifzero false
-attribute[1].createifnonexistent false
-attribute[1].fastsearch true
-attribute[1].huge false
-attribute[1].ismutable false
-attribute[1].sortascending true
-attribute[1].sortfunction UCA
-attribute[1].sortstrength PRIMARY
-attribute[1].sortlocale ""
-attribute[1].enablebitvectors false
-attribute[1].enableonlybitvector false
-attribute[1].fastaccess false
-attribute[1].arity 8
-attribute[1].lowerbound -9223372036854775808
-attribute[1].upperbound 9223372036854775807
-attribute[1].densepostinglistthreshold 0.4
-attribute[1].tensortype ""
-attribute[1].imported true
-attribute[2].name "my_elem_array.weight"
-attribute[2].datatype INT32
-attribute[2].collectiontype SINGLE
-attribute[2].removeifzero false
-attribute[2].createifnonexistent false
-attribute[2].fastsearch false
-attribute[2].huge false
-attribute[2].ismutable false
-attribute[2].sortascending true
-attribute[2].sortfunction UCA
-attribute[2].sortstrength PRIMARY
-attribute[2].sortlocale ""
-attribute[2].enablebitvectors false
-attribute[2].enableonlybitvector false
-attribute[2].fastaccess false
-attribute[2].arity 8
-attribute[2].lowerbound -9223372036854775808
-attribute[2].upperbound 9223372036854775807
-attribute[2].densepostinglistthreshold 0.4
-attribute[2].tensortype ""
-attribute[2].imported true
-attribute[3].name "my_elem_map.key"
-attribute[3].datatype STRING
-attribute[3].collectiontype SINGLE
-attribute[3].removeifzero false
-attribute[3].createifnonexistent false
-attribute[3].fastsearch true
-attribute[3].huge false
-attribute[3].ismutable false
-attribute[3].sortascending true
-attribute[3].sortfunction UCA
-attribute[3].sortstrength PRIMARY
-attribute[3].sortlocale ""
-attribute[3].enablebitvectors false
-attribute[3].enableonlybitvector false
-attribute[3].fastaccess false
-attribute[3].arity 8
-attribute[3].lowerbound -9223372036854775808
-attribute[3].upperbound 9223372036854775807
-attribute[3].densepostinglistthreshold 0.4
-attribute[3].tensortype ""
-attribute[3].imported true
-attribute[4].name "my_elem_map.value.name"
-attribute[4].datatype STRING
-attribute[4].collectiontype SINGLE
-attribute[4].removeifzero false
-attribute[4].createifnonexistent false
-attribute[4].fastsearch true
-attribute[4].huge false
-attribute[4].ismutable false
-attribute[4].sortascending true
-attribute[4].sortfunction UCA
-attribute[4].sortstrength PRIMARY
-attribute[4].sortlocale ""
-attribute[4].enablebitvectors false
-attribute[4].enableonlybitvector false
-attribute[4].fastaccess false
-attribute[4].arity 8
-attribute[4].lowerbound -9223372036854775808
-attribute[4].upperbound 9223372036854775807
-attribute[4].densepostinglistthreshold 0.4
-attribute[4].tensortype ""
-attribute[4].imported true
-attribute[5].name "my_elem_map.value.weight"
-attribute[5].datatype INT32
-attribute[5].collectiontype SINGLE
-attribute[5].removeifzero false
-attribute[5].createifnonexistent false
-attribute[5].fastsearch false
-attribute[5].huge false
-attribute[5].ismutable false
-attribute[5].sortascending true
-attribute[5].sortfunction UCA
-attribute[5].sortstrength PRIMARY
-attribute[5].sortlocale ""
-attribute[5].enablebitvectors false
-attribute[5].enableonlybitvector false
-attribute[5].fastaccess false
-attribute[5].arity 8
-attribute[5].lowerbound -9223372036854775808
-attribute[5].upperbound 9223372036854775807
-attribute[5].densepostinglistthreshold 0.4
-attribute[5].tensortype ""
-attribute[5].imported true
-attribute[6].name "my_str_int_map.key"
-attribute[6].datatype STRING
-attribute[6].collectiontype SINGLE
-attribute[6].removeifzero false
-attribute[6].createifnonexistent false
-attribute[6].fastsearch true
-attribute[6].huge false
-attribute[6].ismutable false
-attribute[6].sortascending true
-attribute[6].sortfunction UCA
-attribute[6].sortstrength PRIMARY
-attribute[6].sortlocale ""
-attribute[6].enablebitvectors false
-attribute[6].enableonlybitvector false
-attribute[6].fastaccess false
-attribute[6].arity 8
-attribute[6].lowerbound -9223372036854775808
-attribute[6].upperbound 9223372036854775807
-attribute[6].densepostinglistthreshold 0.4
-attribute[6].tensortype ""
-attribute[6].imported true
-attribute[7].name "my_str_int_map.value"
-attribute[7].datatype INT32
-attribute[7].collectiontype SINGLE
-attribute[7].removeifzero false
-attribute[7].createifnonexistent false
-attribute[7].fastsearch false
-attribute[7].huge false
-attribute[7].ismutable false
-attribute[7].sortascending true
-attribute[7].sortfunction UCA
-attribute[7].sortstrength PRIMARY
-attribute[7].sortlocale ""
-attribute[7].enablebitvectors false
-attribute[7].enableonlybitvector false
-attribute[7].fastaccess false
-attribute[7].arity 8
-attribute[7].lowerbound -9223372036854775808
-attribute[7].upperbound 9223372036854775807
-attribute[7].densepostinglistthreshold 0.4
-attribute[7].tensortype ""
-attribute[7].imported true \ No newline at end of file
+attribute[].name "parent_ref"
+attribute[].datatype REFERENCE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_array.name"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_array.weight"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_map.key"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_map.value.name"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_map.value.weight"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_str_int_map.key"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_str_int_map.value"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg
index 15b1771a2e8..d3a51523e05 100644
--- a/config-model/src/test/derived/importedfields/attributes.cfg
+++ b/config-model/src/test/derived/importedfields/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b_ref"
attribute[].datatype REFERENCE
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "b_ref_with_summary"
attribute[].datatype REFERENCE
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "my_int_field"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "my_string_field"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "my_int_array_field"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "my_int_wset_field"
attribute[].datatype INT32
attribute[].collectiontype WEIGHTEDSET
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "my_ancient_int_field"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -166,3 +194,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg
index a83ec45c5e9..388b212689a 100644
--- a/config-model/src/test/derived/indexschema/index-info.cfg
+++ b/config-model/src/test/derived/indexschema/index-info.cfg
@@ -44,6 +44,8 @@ indexinfo[].command[].command "normalize"
indexinfo[].command[].indexname "sd"
indexinfo[].command[].command "plain-tokens"
indexinfo[].command[].indexname "sd"
+indexinfo[].command[].command "phrase-segmenting false"
+indexinfo[].command[].indexname "sd"
indexinfo[].command[].command "literal-boost"
indexinfo[].command[].indexname "pos.x"
indexinfo[].command[].command "index"
@@ -307,6 +309,16 @@ indexinfo[].command[].indexname "exactexplicit"
indexinfo[].command[].command "exact ARNOLD"
indexinfo[].command[].indexname "exactexplicit"
indexinfo[].command[].command "dynteaser"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "lowercase"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "index"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "plain-tokens"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "stem:BEST"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "normalize"
indexinfo[].command[].indexname "exactexplicit2"
indexinfo[].command[].command "lowercase"
indexinfo[].command[].indexname "exactexplicit2"
@@ -322,6 +334,8 @@ indexinfo[].command[].command "stem:BEST"
indexinfo[].command[].indexname "gram"
indexinfo[].command[].command "normalize"
indexinfo[].command[].indexname "gram"
+indexinfo[].command[].command "phrase-segmenting false"
+indexinfo[].command[].indexname "gram"
indexinfo[].command[].command "ngram 2"
indexinfo[].command[].indexname "nostem1"
indexinfo[].command[].command "lowercase"
diff --git a/config-model/src/test/derived/indexschema/indexschema.sd b/config-model/src/test/derived/indexschema/indexschema.sd
index 49f0f7dfca6..01f552a8c32 100644
--- a/config-model/src/test/derived/indexschema/indexschema.sd
+++ b/config-model/src/test/derived/indexschema/indexschema.sd
@@ -24,6 +24,7 @@ search indexschema {
field sd type string {
indexing: index
rank:literal
+ query-command: "phrase-segmenting false"
}
field pos type position {
indexing: attribute
@@ -130,17 +131,17 @@ search indexschema {
}
fieldset exactexplicit {
- fields:sa, sb
query-command: "exact ARNOLD"
+ fields:sa, sb
query-command: dynteaser
}
fieldset exactexplicit2 {
- fields:sc, sd
match {
exact
exact-terminator: "Arnold"
}
+ fields:sc, sd
}
fieldset gram {
diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg
index 4a081edbf54..397d8878792 100644
--- a/config-model/src/test/derived/inheritance/attributes.cfg
+++ b/config-model/src/test/derived/inheritance/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "overridden"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "onlymother"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg
index 13f59d4925f..3b30af10a8f 100644
--- a/config-model/src/test/derived/inheritfromparent/attributes.cfg
+++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg
@@ -19,3 +19,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg
index 8901acf63d1..4de78c52efd 100644
--- a/config-model/src/test/derived/map_attribute/attributes.cfg
+++ b/config-model/src/test/derived/map_attribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "str_map.value"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "int_map.key"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
index 665edcdf45d..72e60b857a2 100644
--- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
+++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "str_elem_map.value.name"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "str_elem_map.value.weight"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "int_elem_map.key"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "int_elem_map.value.name"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -103,3 +119,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg
index 7f9592dafc8..4d19dc69b33 100644
--- a/config-model/src/test/derived/music/attributes.cfg
+++ b/config-model/src/test/derived/music/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "pto"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "mid"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "weight"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "bgnpfrom"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "newestedition"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "year"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "did"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "cbid"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "hiphopvalue_arr"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "metalvalue_arr"
attribute[].datatype STRING
attribute[].collectiontype ARRAY
@@ -229,3 +269,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml
new file mode 100644
index 00000000000..b8140f34617
--- /dev/null
+++ b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml
@@ -0,0 +1 @@
+<query-profile id="default" type="root" />
diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml
new file mode 100644
index 00000000000..895e0663181
--- /dev/null
+++ b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml
@@ -0,0 +1,3 @@
+<query-profile-type id="root" inherits="native">
+ <field name="ranking.features.query(q_vec)" type="tensor&lt;float&gt;(x[5])" />
+</query-profile-type>
diff --git a/config-model/src/test/derived/nearestneighbor/test.sd b/config-model/src/test/derived/nearestneighbor/test.sd
new file mode 100644
index 00000000000..ab5f6d85448
--- /dev/null
+++ b/config-model/src/test/derived/nearestneighbor/test.sd
@@ -0,0 +1,27 @@
+search test {
+ document test {
+ field id type int {
+ indexing: attribute | summary
+ }
+ field vec type tensor<float>(x[5]) {
+ indexing: attribute | summary
+ }
+ field vec_hnsw type tensor<float>(x[5]) {
+ indexing: attribute | index | summary
+ index {
+ hnsw {
+ max-links-per-node: 16
+ neighbors-to-explore-at-insert: 200
+ }
+ }
+ }
+ }
+ rank-profile default {
+ first-phase {
+ expression: 10000 - itemRawScore(nns)
+ }
+ }
+ document-summary minimal {
+ summary id type int {}
+ }
+}
diff --git a/config-model/src/test/derived/neuralnet/query-profiles.cfg b/config-model/src/test/derived/neuralnet/query-profiles.cfg
new file mode 100644
index 00000000000..817640a89fd
--- /dev/null
+++ b/config-model/src/test/derived/neuralnet/query-profiles.cfg
@@ -0,0 +1,54 @@
+queryprofile[].id "default"
+queryprofile[].type "DefaultQueryProfileType"
+queryprofiletype[].id "DefaultQueryProfileType"
+queryprofiletype[].strict false
+queryprofiletype[].matchaspath false
+queryprofiletype[].inherit[] "native"
+queryprofiletype[].field[].name "ranking"
+queryprofiletype[].field[].type "query-profile:ranking_0_0"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].id "ranking_0_0"
+queryprofiletype[].strict false
+queryprofiletype[].matchaspath false
+queryprofiletype[].inherit[] "ranking"
+queryprofiletype[].field[].name "features"
+queryprofiletype[].field[].type "query-profile:features_0_1"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias "rankfeature"
+queryprofiletype[].id "features_0_1"
+queryprofiletype[].strict false
+queryprofiletype[].matchaspath false
+queryprofiletype[].field[].name "query(W_0)"
+queryprofiletype[].field[].type "tensor(hidden[9],x[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(W_1)"
+queryprofiletype[].field[].type "tensor(hidden[9],out[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(W_out)"
+queryprofiletype[].field[].type "tensor(out[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(b_0)"
+queryprofiletype[].field[].type "tensor(hidden[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(b_1)"
+queryprofiletype[].field[].type "tensor(out[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(b_out)"
+queryprofiletype[].field[].type "tensor(out[1])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+enableGroupingSessionCache true
diff --git a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml
index e74152638fb..42336098a9a 100644
--- a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml
+++ b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml
@@ -1,5 +1,5 @@
<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<query-profile-type id="DefaultQueryProfileType">
+<query-profile-type id="DefaultQueryProfileType" inherits="native">
<field name="ranking.features.query(W_0)" type="tensor(x[9],hidden[9])" />
<field name="ranking.features.query(b_0)" type="tensor(hidden[9])" />
<field name="ranking.features.query(W_1)" type="tensor(hidden[9],out[9])" />
diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg
index b33c2fbdf9b..2aed2288773 100644
--- a/config-model/src/test/derived/newrank/attributes.cfg
+++ b/config-model/src/test/derived/newrank/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "pto"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "mid"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "weight"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "bgnpfrom"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "newestedition"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "year"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "did"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "scorekey"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "cbid"
attribute[].datatype INT32
attribute[].collectiontype SINGLE
@@ -208,3 +244,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg
index 47e07e2a524..3a9daf7af94 100644
--- a/config-model/src/test/derived/predicate_attribute/attributes.cfg
+++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg
@@ -19,3 +19,7 @@ attribute[].upperbound 200
attribute[].densepostinglistthreshold 0.2
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/predicate_attribute/index-info.cfg b/config-model/src/test/derived/predicate_attribute/index-info.cfg
index 3d9f57dd84b..4ebac65e1f5 100644
--- a/config-model/src/test/derived/predicate_attribute/index-info.cfg
+++ b/config-model/src/test/derived/predicate_attribute/index-info.cfg
@@ -4,6 +4,8 @@ indexinfo[].command[].command "index"
indexinfo[].command[].indexname "sddocname"
indexinfo[].command[].command "word"
indexinfo[].command[].indexname "some_predicate_field"
+indexinfo[].command[].command "predicate"
+indexinfo[].command[].indexname "some_predicate_field"
indexinfo[].command[].command "predicate-bounds [3..200]"
indexinfo[].command[].indexname "some_predicate_field"
indexinfo[].command[].command "index"
diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg
index d7922a0de69..0a8cbd82186 100644
--- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg
+++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "attributefield2"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -40,3 +44,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg
index 12dbf896edc..e83b187a0cc 100644
--- a/config-model/src/test/derived/reference_fields/attributes.cfg
+++ b/config-model/src/test/derived/reference_fields/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "other_ref"
attribute[].datatype REFERENCE
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "yet_another_ref"
attribute[].datatype REFERENCE
attribute[].collectiontype SINGLE
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg
index e88dfde03bb..83c310ca5ca 100644
--- a/config-model/src/test/derived/sorting/attributes.cfg
+++ b/config-model/src/test/derived/sorting/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "syntaxcheck2"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "infieldonly"
attribute[].datatype STRING
attribute[].collectiontype SINGLE
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg
index 4634e120a3a..780f47ee3d5 100644
--- a/config-model/src/test/derived/tensor/attributes.cfg
+++ b/config-model/src/test/derived/tensor/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype "tensor<float>(x[2],y[1])"
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "f3"
attribute[].datatype TENSOR
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype "tensor(x{})"
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "f4"
attribute[].datatype TENSOR
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype "tensor(x[10],y[10])"
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "f5"
attribute[].datatype TENSOR
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype "tensor<float>(x[10])"
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "f6"
attribute[].datatype FLOAT
attribute[].collectiontype SINGLE
@@ -103,3 +119,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg
index e6ffc37e871..82535324864 100644
--- a/config-model/src/test/derived/types/attributes.cfg
+++ b/config-model/src/test/derived/types/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "along"
attribute[].datatype INT64
attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "abool"
attribute[].datatype BOOL
attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "ashortfloat"
attribute[].datatype FLOAT16
attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "arrayfield"
attribute[].datatype INT32
attribute[].collectiontype ARRAY
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "setfield"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "setfield2"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "setfield3"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "setfield4"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "tagfield"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "juletre"
attribute[].datatype INT64
attribute[].collectiontype SINGLE
@@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "album1"
attribute[].datatype STRING
attribute[].collectiontype WEIGHTEDSET
@@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
attribute[].name "other"
attribute[].datatype INT64
attribute[].collectiontype SINGLE
@@ -271,3 +319,7 @@ attribute[].upperbound 9223372036854775807
attribute[].densepostinglistthreshold 0.4
attribute[].tensortype ""
attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd
index e61b64dc0ed..a8c801b1421 100644
--- a/config-model/src/test/examples/simple.sd
+++ b/config-model/src/test/examples/simple.sd
@@ -3,7 +3,7 @@
# You can get a reasonable configuration by only configuring
# a document
# ...this has become less and less simple over time actually
-search simple {
+schema simple {
document simple {
diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/integration/lightgbm/models/regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java
index cb1577417b4..fe82f2406f2 100644
--- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java
@@ -20,7 +20,7 @@ import com.yahoo.searchdefinition.DocumentOnlySearch;
import com.yahoo.vespa.config.ConfigDefinition;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
@@ -31,7 +31,6 @@ import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -60,9 +59,9 @@ public class ApplicationDeployTest {
public void testVespaModel() throws SAXException, IOException {
ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1");
VespaModel model = new VespaModel(tester.app());
- List<SearchDefinition> searchDefinitions = tester.getSearchDefinitions();
- assertEquals(searchDefinitions.size(), 5);
- for (SearchDefinition searchDefinition : searchDefinitions) {
+ List<NamedSchema> schemas = tester.getSchemas();
+ assertEquals(schemas.size(), 5);
+ for (NamedSchema searchDefinition : schemas) {
Search s = searchDefinition.getSearch();
switch (s.getName()) {
case "music":
@@ -72,21 +71,18 @@ public class ApplicationDeployTest {
break;
case "product":
assertTrue(s instanceof DocumentOnlySearch);
- assertEquals(s.getDocument().getField("title").getDataType(), DataType.STRING);
+ assertEquals(DataType.STRING, s.getDocument().getField("title").getDataType());
break;
default:
fail();
}
}
- File[] truth = new File[]{new File(TESTSDDIR + "laptop.sd"),
- new File(TESTSDDIR + "music.sd"),
- new File(TESTSDDIR + "pc.sd"),
- new File(TESTSDDIR + "product.sd"),
- new File(TESTSDDIR + "sock.sd")};
- Arrays.sort(truth);
- List<File> appSdFiles = tester.app().getSearchDefinitionFiles();
- Collections.sort(appSdFiles);
- assertEquals(appSdFiles, Arrays.asList(truth));
+ assertEquals(Set.of(new File(TESTSDDIR + "laptop.sd"),
+ new File(TESTSDDIR + "music.sd"),
+ new File(TESTSDDIR + "pc.sd"),
+ new File(TESTSDDIR + "product.sd"),
+ new File(TESTSDDIR + "sock.sd")),
+ new HashSet<>(tester.app().getSearchDefinitionFiles()));
List<FilesApplicationPackage.Component> components = tester.app().getComponents();
assertEquals(1, components.size());
@@ -103,7 +99,7 @@ public class ApplicationDeployTest {
// Check that getFilename works
ArrayList<String> sdFileNames = new ArrayList<>();
- for (SearchDefinition sd : searchDefinitions)
+ for (NamedSchema sd : schemas)
sdFileNames.add(sd.getFilename());
Collections.sort(sdFileNames);
assertEquals("laptop.sd", sdFileNames.get(0));
@@ -190,11 +186,11 @@ public class ApplicationDeployTest {
File tmpDir = tmpFolder.getRoot();
IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir);
ApplicationPackageTester tester = ApplicationPackageTester.create(tmpDir.getAbsolutePath());
- assertEquals(5, tester.getSearchDefinitions().size());
- File sdDir = new File(tmpDir, "searchdefinitions");
+ assertEquals(5, tester.getSchemas().size());
+ File sdDir = new File(tmpDir, "schemas");
File sd = new File(sdDir, "testfoo.sd");
IOUtils.writeFile(sd, "search testfoo { document testfoo { field bar type string { } } }", false);
- assertEquals(6, tester.getSearchDefinitions().size());
+ assertEquals(6, tester.getSchemas().size());
}
@Test
@@ -300,6 +296,16 @@ public class ApplicationDeployTest {
@Test
public void testGetJarEntryName() {
+ JarEntry e = new JarEntry("/schemas/foo.sd");
+ assertEquals(ApplicationPackage.getFileName(e), "foo.sd");
+ e = new JarEntry("bar");
+ assertEquals(ApplicationPackage.getFileName(e), "bar");
+ e = new JarEntry("");
+ assertEquals(ApplicationPackage.getFileName(e), "");
+ }
+
+ @Test
+ public void testGetJarEntryNameForLegacyPath() {
JarEntry e = new JarEntry("/searchdefinitions/foo.sd");
assertEquals(ApplicationPackage.getFileName(e), "foo.sd");
e = new JarEntry("bar");
diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java
index 87b6efa83d6..8e7d5aadb36 100644
--- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java
+++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java
@@ -5,7 +5,7 @@ import com.yahoo.component.Version;
import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator;
import com.yahoo.config.model.application.provider.FilesApplicationPackage;
import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
import java.io.File;
import java.io.IOException;
@@ -39,8 +39,8 @@ public class ApplicationPackageTester {
public FilesApplicationPackage app() { return applicationPackage; }
- public List<SearchDefinition> getSearchDefinitions() {
- return new DeployState.Builder().applicationPackage(app()).build().getSearchDefinitions();
+ public List<NamedSchema> getSchemas() {
+ return new DeployState.Builder().applicationPackage(app()).build().getSchemas();
}
public static ApplicationPackageTester create(String applicationPackageDir) {
diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
index 38e438e4d3a..f8ab3cc54c8 100644
--- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
+++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo;
import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.model.api.Model;
import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo;
@@ -53,6 +54,9 @@ public class MockModelContext implements ModelContext {
}
@Override
+ public Provisioned provisioned() { return new Provisioned(); }
+
+ @Override
public DeployLogger deployLogger() {
return new BaseDeployLogger();
}
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 7b4b650295c..7208d8c5fc1 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -264,7 +264,7 @@ public class ModelProvisioningTest {
assertEquals("Heap size is lowered with combined clusters",
17, physicalMemoryPercentage(model.getContainerClusters().get("container1")));
assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model);
- assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.combined, model);
+ assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model);
}
}
@@ -1205,6 +1205,62 @@ public class ModelProvisioningTest {
}
@Test
+ public void testRequestingRangesMin() {
+ String services =
+ "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<services>" +
+ " <container version='1.0' id='container'>" +
+ " <nodes count='[4, 6]'>" +
+ " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" +
+ " </nodes>" +
+ " </container>" +
+ " <content version='1.0' id='foo'>" +
+ " <documents>" +
+ " <document type='type1' mode='index'/>" +
+ " </documents>" +
+ " <nodes count='[6, 20]' groups='[3,4]'>" +
+ " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" +
+ " </nodes>" +
+ " </content>" +
+ "</services>";
+
+ int totalHosts = 10;
+ VespaModelTester tester = new VespaModelTester();
+ tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6);
+ tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
+ VespaModel model = tester.createModel(services, true);
+ assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
+ }
+
+ @Test
+ public void testRequestingRangesMax() {
+ String services =
+ "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<services>" +
+ " <container version='1.0' id='container'>" +
+ " <nodes count='[4, 6]'>" +
+ " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" +
+ " </nodes>" +
+ " </container>" +
+ " <content version='1.0' id='foo'>" +
+ " <documents>" +
+ " <document type='type1' mode='index'/>" +
+ " </documents>" +
+ " <nodes count='[6, 20]' groups='[3,4]'>" +
+ " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" +
+ " </nodes>" +
+ " </content>" +
+ "</services>";
+
+ int totalHosts = 26;
+ VespaModelTester tester = new VespaModelTester();
+ tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6);
+ tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
+ VespaModel model = tester.createModel(services, true, true);
+ assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
+ }
+
+ @Test
public void testContainerOnly() {
String services =
"<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -1308,14 +1364,14 @@ public class ModelProvisioningTest {
" </http>" +
"</container>";
VespaModelTester tester = new VespaModelTester();
- tester.addHosts(1);
+ tester.addHosts(2);
VespaModel model = tester.createModel(services, true);
- assertEquals(1, model.getHosts().size());
+ assertEquals(2, model.getHosts().size());
assertEquals(1, model.getContainerClusters().size());
+ assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
}
@Test
- @Ignore // TODO: Enable when turning the port check on
public void testThatStandaloneSyntaxOnHostedVespaRequiresDefaultPort() {
try {
String services =
@@ -1375,7 +1431,7 @@ public class ModelProvisioningTest {
}
@Test
- public void testNoNodeTagMeans1Node() {
+ public void testNoNodeTagMeansTwoNodes() {
String services =
"<?xml version='1.0' encoding='utf-8' ?>\n" +
"<services>" +
@@ -1390,16 +1446,16 @@ public class ModelProvisioningTest {
" </content>" +
"</services>";
VespaModelTester tester = new VespaModelTester();
- tester.addHosts(1);
+ tester.addHosts(3);
VespaModel model = tester.createModel(services, true);
- assertEquals(1, model.getRoot().hostSystem().getHosts().size());
- assertEquals(1, model.getAdmin().getSlobroks().size());
- assertEquals(1, model.getContainerClusters().get("foo").getContainers().size());
+ assertEquals(3, model.getRoot().hostSystem().getHosts().size());
+ assertEquals(2, model.getAdmin().getSlobroks().size());
+ assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes());
}
@Test
- public void testNoNodeTagMeans1NodeNoContent() {
+ public void testNoNodeTagMeansTwoNodesNoContent() {
String services =
"<?xml version='1.0' encoding='utf-8' ?>\n" +
"<services>" +
@@ -1409,11 +1465,11 @@ public class ModelProvisioningTest {
" </container>" +
"</services>";
VespaModelTester tester = new VespaModelTester();
- tester.addHosts(1);
+ tester.addHosts(2);
VespaModel model = tester.createModel(services, true);
- assertEquals(1, model.getRoot().hostSystem().getHosts().size());
- assertEquals(1, model.getAdmin().getSlobroks().size());
- assertEquals(1, model.getContainerClusters().get("foo").getContainers().size());
+ assertEquals(2, model.getRoot().hostSystem().getHosts().size());
+ assertEquals(2, model.getAdmin().getSlobroks().size());
+ assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
}
@Test
@@ -1806,12 +1862,17 @@ public class ModelProvisioningTest {
assertTrue(logdConfig.logserver().use());
}
- private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) {
- assertEquals("Nodes in cluster " + id + " with type " + type, nodeCount,
+ private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Id combinedId,
+ ClusterSpec.Type type, VespaModel model) {
+ assertEquals("Nodes in cluster " + id + " with type " + type + (combinedId != null ? ", combinedId " + combinedId : ""), nodeCount,
model.hostSystem().getHosts().stream()
.map(h -> h.spec().membership().get().cluster())
- .filter(spec -> spec.id().equals(id) && spec.type().equals(type))
+ .filter(spec -> spec.id().equals(id) && spec.type().equals(type) && spec.combinedId().equals(Optional.ofNullable(combinedId)))
.count());
}
+ private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) {
+ assertProvisioned(nodeCount, id, null, type, model);
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java
index 94602d5201a..127c121197b 100644
--- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.document.test;
import com.yahoo.document.DataType;
import com.yahoo.document.DataTypeName;
import com.yahoo.documentmodel.VespaDocumentType;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import org.junit.Test;
@@ -17,7 +17,7 @@ import static org.junit.Assert.*;
* @author Thomas Gundersen
* @author bratseth
*/
-public class SDDocumentTypeTestCase extends SearchDefinitionTestCase {
+public class SDDocumentTypeTestCase extends SchemaTestCase {
// Verify that we can register and retrieve fields.
@Test
diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java
index 7dcbd92655b..b3109c3c2e4 100644
--- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java
+++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java
@@ -2,7 +2,7 @@
package com.yahoo.document.test;
import com.yahoo.document.DataType;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDDocumentType;
import org.junit.Test;
@@ -11,7 +11,7 @@ import static org.junit.Assert.fail;
/**
* @author Thomas Gundersen
*/
-public class SDFieldTestCase extends SearchDefinitionTestCase {
+public class SDFieldTestCase extends SchemaTestCase {
@Test
public void testIdSettingConflict() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java
index 846166ae93c..6a40778c9c4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java
@@ -18,7 +18,7 @@ import static org.junit.Assert.assertTrue;
*
* @author bratseth
*/
-public class ArraysTestCase extends SearchDefinitionTestCase {
+public class ArraysTestCase extends SchemaTestCase {
@Test
public void testArrayImporting() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java
index cfd02c22b89..af1e061b7fa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertTrue;
*
* @author Einar M R Rosenvinge
*/
-public class ArraysWeightedSetsTestCase extends SearchDefinitionTestCase {
+public class ArraysWeightedSetsTestCase extends SchemaTestCase {
@Test
public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException {
Search search = SearchBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
index 084cbcfdfc0..83cad4cf266 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
@@ -25,7 +25,7 @@ import static org.junit.Assert.*;
*
* @author bratseth
*/
-public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
+public class AttributeSettingsTestCase extends SchemaTestCase {
@Rule
public final ExpectedException exceptionRule = ExpectedException.none();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
new file mode 100644
index 00000000000..2c13427760f
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
@@ -0,0 +1,15 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * Convenience class for tests that need to set attribute properties on fields.
+ */
+public class AttributeUtils {
+
+ public static void addAttributeAspect(SDField field) {
+ field.parseIndexingScript("{ attribute }");
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java
index 8ccb1ed969a..3bb464c5fa5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java
@@ -14,7 +14,7 @@ import static org.junit.Assert.assertEquals;
*
* @author bratseth
*/
-public class CommentTestCase extends SearchDefinitionTestCase {
+public class CommentTestCase extends SchemaTestCase {
@Test
public void testComments() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index 8378ec811a5..e46208c770d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest {
// Create foo document with document reference to bar and add another field
SDField fooRefToBarField = new SDField
("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
- addAttributeAspect(fooRefToBarField);
+ AttributeUtils.addAttributeAspect(fooRefToBarField);
SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT);
Search fooSearch = new Search();
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
@@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest {
// Create foo document with document reference to non-existing document bar
SDField fooRefToBarField = new SDField(
"bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar")));
- addAttributeAspect(fooRefToBarField);
+ AttributeUtils.addAttributeAspect(fooRefToBarField);
Search fooSearch = new Search();
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
fooDocument.addField(fooRefToBarField);
@@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest {
resolver.resolveReferences(fooDocument);
}
- private static void addAttributeAspect(SDField fooRefToBarField) {
- fooRefToBarField.parseIndexingScript("{ attribute }");
- }
-
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java
index be3bae05c5b..1c7b3e19663 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.assertSame;
/**
* @author Einar M R Rosenvinge
*/
-public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase {
+public class FieldOfTypeDocumentTestCase extends SchemaTestCase {
@Test
public void testDocument() throws IOException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
new file mode 100644
index 00000000000..fcbb89b5c42
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
@@ -0,0 +1,66 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.ReferenceDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedField;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class ImportedFieldsEnumeratorTest {
+
+ @Test
+ public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() {
+ Search parentSearch = new Search();
+ SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch);
+ var parentField = new SDField("their_field", DataType.INT);
+ AttributeUtils.addAttributeAspect(parentField);
+ parentDocument.addField(parentField);
+ parentSearch.addDocument(parentDocument);
+
+ Search fooSearch = new Search();
+ SDField fooRefToParent = new SDField(
+ "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+ AttributeUtils.addAttributeAspect(fooRefToParent);
+ var fooImports = fooSearch.temporaryImportedFields().get();
+ fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field"));
+ fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field"));
+ SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
+ fooSearch.addDocument(fooDocument);
+
+ Search barSearch = new Search();
+ SDField barRefToParent = new SDField(
+ "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+ AttributeUtils.addAttributeAspect(barRefToParent);
+ var barImports = barSearch.temporaryImportedFields().get();
+ barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field"));
+ SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
+ barSearch.addDocument(barDocument);
+
+ var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch));
+
+ enumerator.enumerateImportedFields(parentDocument);
+ assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent
+
+ enumerator.enumerateImportedFields(fooDocument);
+ assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import"));
+
+ enumerator.enumerateImportedFields(barDocument);
+ assertImportedFieldsAre(barDocument, List.of("my_cool_import"));
+ }
+
+ private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) {
+ assertNotNull(documentType.getTemporaryImportedFields());
+ var actualNames = documentType.getTemporaryImportedFields().fields().keySet();
+ var expectedNameSet = new HashSet<>(expectedNames);
+ assertEquals(expectedNameSet, actualNames);
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
index 519828497fe..4453f327bb4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
@@ -18,7 +18,7 @@ import static org.junit.Assert.fail;
/**
* @author bratseth
*/
-public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionTestCase {
+public class IncorrectRankingExpressionFileRefTestCase extends SchemaTestCase {
@Test
public void testIncorrectRef() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
index c145c0e5634..91ab5e2b5df 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.fail;
*
* @author bratseth
*/
-public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase {
+public class IncorrectSummaryTypesTestCase extends SchemaTestCase {
@Test
public void testImportingIncorrect() throws ParseException {
try {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java
index 2cfb542d06b..f992d5ee0ba 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java
@@ -15,7 +15,7 @@ import static org.junit.Assert.assertEquals;
*
* @author bratseth
*/
-public class IndexSettingsTestCase extends SearchDefinitionTestCase {
+public class IndexSettingsTestCase extends SchemaTestCase {
@Test
public void testStemmingSettings() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java
index 21ba3a5e80a..70119ad42f9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull;
*
* @author frodelu
*/
-public class IndexingParsingTestCase extends SearchDefinitionTestCase {
+public class IndexingParsingTestCase extends SchemaTestCase {
@Test
public void requireThatIndexingExpressionsCanBeParsed() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java
index df9f5778614..5721dbf06e8 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java
@@ -11,7 +11,7 @@ import java.io.IOException;
*
* @author bratseth
*/
-public class MultipleSummariesTestCase extends SearchDefinitionTestCase {
+public class MultipleSummariesTestCase extends SchemaTestCase {
@Test
public void testArrayImporting() throws IOException, ParseException {
SearchBuilder.buildFromFile("src/test/examples/multiplesummaries.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
index d2360453976..47b6905c677 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
@@ -14,7 +14,7 @@ import static org.junit.Assert.fail;
*
* @author Lars Christian Jensen
*/
-public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
+public class NameFieldCheckTestCase extends SchemaTestCase {
@Test
public void testNameField() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java
index 5ac37bf0a3a..64527e7f7a5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java
@@ -13,7 +13,7 @@ import static org.junit.Assert.assertTrue;
*
* @author bratseth
*/
-public class OutsideTestCase extends SearchDefinitionTestCase {
+public class OutsideTestCase extends SchemaTestCase {
@Test
public void testOutsideIndex() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
index 5ba508e3ef3..e2f2c1fd407 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
@@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue;
*
* @author bratseth
*/
-public class RankProfileTestCase extends SearchDefinitionTestCase {
+public class RankProfileTestCase extends SchemaTestCase {
@Test
public void testRankProfileInheritance() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java
index f4666f7fb3b..3fe2861de0c 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java
@@ -13,7 +13,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author bratseth
*/
-public class RankPropertiesTestCase extends SearchDefinitionTestCase {
+public class RankPropertiesTestCase extends SchemaTestCase {
@Test
public void testRankPropertyInheritance() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
index 51508414205..d84d967a184 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
@@ -17,7 +17,7 @@ import static org.junit.Assert.*;
/**
* @author bratseth
*/
-public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionConstantsTestCase extends SchemaTestCase {
@Test
public void testConstants() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
index 58e62353e5c..e0679eb5175 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
@@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue;
/**
* @author bratseth
*/
-public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionInliningTestCase extends SchemaTestCase {
@Test
public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
index 3d842be129f..5c1134f928c 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author lesters
*/
-public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionShadowingTestCase extends SchemaTestCase {
@Test
public void testBasicFunctionShadowing() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
index c5027af2a0c..c1fe5e42dfa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
@@ -15,7 +15,7 @@ import static org.junit.Assert.fail;
/**
* @author bratseth
*/
-public class RankingExpressionValidationTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionValidationTestCase extends SchemaTestCase {
@Test
public void testInvalidExpressionProducesException() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java
index 5a5fc1cc312..1a939d71937 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull;
/**
* @author bratseth
*/
-public class ReservedWordsAsFieldNamesTestCase extends SearchDefinitionTestCase {
+public class ReservedWordsAsFieldNamesTestCase extends SchemaTestCase {
@Test
public void testIt() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java
index fd4bb393c49..0ae39b7f8b6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java
@@ -15,7 +15,7 @@ import static org.junit.Assert.*;
*
* @author hmusum
*/
-public class SearchDefinitionsParsingTestCase extends SearchDefinitionTestCase {
+public class SchemaParsingTestCase extends SchemaTestCase {
@Test
public void requireThatIndexingExpressionsCanBeParsed() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
index ba6da8792fa..7f3ea7d14bc 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
@@ -10,7 +10,7 @@ import java.io.IOException;
import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals;
import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals;
-public abstract class SearchDefinitionTestCase {
+public abstract class SchemaTestCase {
protected static void assertConfigFile(String filename, String cfg) throws IOException {
assertSerializedConfigFileEquals(filename, cfg);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
index 66ff1877994..018703153ac 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
@@ -3,7 +3,6 @@ package com.yahoo.searchdefinition;
import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.document.DataType;
-import com.yahoo.document.Document;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.document.Attribute;
import com.yahoo.searchdefinition.document.RankType;
@@ -31,7 +30,7 @@ import static org.junit.Assert.fail;
*
* @author bratseth
*/
-public class SearchImporterTestCase extends SearchDefinitionTestCase {
+public class SearchImporterTestCase extends SchemaTestCase {
@Test
@SuppressWarnings("deprecation")
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java
index 9b27d338ced..e5b8ec85d75 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertNull;
*
* @author bratseth
*/
-public class StemmingSettingTestCase extends SearchDefinitionTestCase {
+public class StemmingSettingTestCase extends SchemaTestCase {
@Test
public void testStemmingSettings() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java
index 001ad64e2da..77df5b391dc 100755
--- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java
@@ -14,7 +14,7 @@ import static org.junit.Assert.fail;
*
* @author bratseth
*/
-public class StructTestCase extends SearchDefinitionTestCase {
+public class StructTestCase extends SchemaTestCase {
@Test
public void testStruct() throws IOException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
index d67df3a5239..a345cabe909 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
@@ -1,11 +1,13 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.document.DocumenttypesConfig;
import com.yahoo.document.config.DocumentmanagerConfig;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
import com.yahoo.vespa.configmodel.producers.DocumentManager;
@@ -19,22 +21,27 @@ import java.io.IOException;
*
* @author bratseth
*/
-public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase {
+public abstract class AbstractExportingTestCase extends SchemaTestCase {
private static final String tempDir = "temp/";
private static final String searchDefRoot = "src/test/derived/";
- private DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException {
+ private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
File toDir = new File(tempDir + dirName);
toDir.mkdirs();
deleteContent(toDir);
SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/");
- return derive(dirName, searchDefinitionName, builder);
+ return derive(dirName, searchDefinitionName, properties, builder);
}
- private DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException {
+ private DerivedConfiguration derive(String dirName,
+ String searchDefinitionName,
+ TestProperties properties,
+ SearchBuilder builder) throws IOException {
DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName),
+ new BaseDeployLogger(),
+ properties,
builder.getRankProfileRegistry(),
builder.getQueryProfileRegistry(),
new ImportedMlModels());
@@ -53,6 +60,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
String path = exportConfig(name, config);
DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path);
DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path);
+ DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path);
return config;
}
@@ -76,7 +84,11 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
}
protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException {
- DerivedConfiguration derived = derive(dirName, searchDefinitionName);
+ return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties());
+ }
+
+ protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
+ DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties);
assertCorrectConfigFiles(dirName);
return derived;
}
@@ -87,7 +99,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
*/
protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException {
builder.build();
- DerivedConfiguration derived = derive(dirName, null, builder);
+ DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder);
assertCorrectConfigFiles(dirName);
return derived;
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
index 79ec3027c20..80a92a5b5ec 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.Attribute;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse;
*
* @author bratseth
*/
-public class AttributeListTestCase extends SearchDefinitionTestCase {
+public class AttributeListTestCase extends SchemaTestCase {
@Test
public void testDeriving() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java
index b47a268d95a..07762fc6937 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals;
*
* @author vegardh
*/
-public class CasingTestCase extends SearchDefinitionTestCase {
+public class CasingTestCase extends SchemaTestCase {
@Test
public void testCasing() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java
index 1209da6d64b..8b09a4efd57 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentTypeManager;
import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import org.junit.Test;
import java.util.List;
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals;
*
* @author bratseth
*/
-public class DeriverTestCase extends SearchDefinitionTestCase {
+public class DeriverTestCase extends SchemaTestCase {
@Test
public void testDeriveDocManager() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
index aaac1631722..47862a2611b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
@@ -17,7 +17,7 @@ import org.junit.Test;
*
* @author bratseth
*/
-public class EmptyRankProfileTestCase extends SearchDefinitionTestCase {
+public class EmptyRankProfileTestCase extends SchemaTestCase {
@Test
public void testDeriving() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index 61065cd4bcc..3c55aa808b5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
+import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -150,4 +151,9 @@ public class ExportingTestCase extends AbstractExportingTestCase {
assertCorrectConfigFiles("tensor2");
}
+ @Test
+ public void testHnswIndex() throws IOException, ParseException {
+ assertCorrectDeriving("hnsw_index");
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java
index c8ba5168e1c..69c247b94d4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java
@@ -1,7 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.RankType;
import org.junit.Test;
@@ -14,7 +14,7 @@ import static org.junit.Assert.*;
*
* @author geirst
*/
-public class NativeRankTypeDefinitionsTestCase extends SearchDefinitionTestCase {
+public class NativeRankTypeDefinitionsTestCase extends SchemaTestCase {
@Test
public void testTables() {
assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java
new file mode 100644
index 00000000000..9f57b22fd58
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java
@@ -0,0 +1,40 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.prelude.query.QueryException;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileConfigurer;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class NearestNeighborTestCase extends AbstractExportingTestCase {
+
+ @Test
+ public void testNearestNeighbor() throws IOException, ParseException {
+ try {
+ ComponentId.resetGlobalCountersForTests();
+ DerivedConfiguration c = assertCorrectDeriving("nearestneighbor");
+
+ CompiledQueryProfileRegistry queryProfiles =
+ QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile();
+ Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5
+ queryProfiles.getComponent("default"));
+ fail("This should fail when q_vec is parsed as a tensor");
+ } catch (QueryException e) {
+ // success
+ assertEquals("Invalid request parameter", e.getMessage());
+ } catch (RuntimeException e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java
index b299c7fa299..a6171901d2d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java
@@ -1,16 +1,34 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.derived;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileConfigurer;
import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
import org.junit.Test;
import java.io.IOException;
+import com.yahoo.component.ComponentId;
+
+import static org.junit.Assert.assertEquals;
+
public class NeuralNetTestCase extends AbstractExportingTestCase {
@Test
public void testNeuralNet() throws IOException, ParseException {
- assertCorrectDeriving("neuralnet");
+ ComponentId.resetGlobalCountersForTests();
+ DerivedConfiguration c = assertCorrectDeriving("neuralnet");
+
+ // Verify that query profiles end up correct when passed through the same intermediate forms as a full system
+ CompiledQueryProfileRegistry queryProfiles =
+ QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile();
+ Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]",
+ queryProfiles.getComponent("default"));
+ assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]",
+ q.properties().get("ranking.features.query(b_1)").toString());
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
index b770024ebf1..0c677456a87 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.document.TemporaryStructuredDataType;
import com.yahoo.searchdefinition.DocumentReference;
import com.yahoo.searchdefinition.DocumentReferences;
import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.document.TemporarySDField;
@@ -26,47 +26,47 @@ import static org.junit.Assert.assertEquals;
* @author bratseth
* @author bjorncs
*/
-public class SearchOrdererTestCase extends SearchDefinitionTestCase {
+public class SearchOrdererTestCase extends SchemaTestCase {
- private static Map<String, Search> createSearchDefinitions() {
- Map<String, Search> searchDefinitions = new HashMap<>();
+ private static Map<String, Search> createSchemas() {
+ Map<String, Search> schemas = new HashMap<>();
- Search grandParent = createSearchDefinition("grandParent", searchDefinitions);
+ Search grandParent = createSchema("grandParent", schemas);
- Search mother = createSearchDefinition("mother", searchDefinitions);
+ Search mother = createSchema("mother", schemas);
inherit(mother, grandParent);
- Search father = createSearchDefinition("father", searchDefinitions);
+ Search father = createSchema("father", schemas);
inherit(father, grandParent);
createDocumentReference(father, mother, "wife_ref");
- Search daugther = createSearchDefinition("daughter", searchDefinitions);
+ Search daugther = createSchema("daughter", schemas);
inherit(daugther, father);
inherit(daugther, mother);
- Search son = createSearchDefinition("son", searchDefinitions);
+ Search son = createSchema("son", schemas);
inherit(son, father);
inherit(son, mother);
- Search product = createSearchDefinition("product", searchDefinitions);
+ Search product = createSchema("product", schemas);
- Search pc = createSearchDefinition("pc", searchDefinitions);
+ Search pc = createSchema("pc", schemas);
inherit(pc, product);
- Search pcAccessory = createSearchDefinition("accessory-pc", searchDefinitions);
+ Search pcAccessory = createSchema("accessory-pc", schemas);
inherit(pcAccessory, product);
createDocumentReference(pcAccessory, pc, "pc_ref");
- createSearchDefinition("alone", searchDefinitions);
+ createSchema("alone", schemas);
- return searchDefinitions;
+ return schemas;
}
- private static Search createSearchDefinition(String name, Map<String, Search> searchDefinitions) {
+ private static Search createSchema(String name, Map<String, Search> schemas) {
Search search = new Search(name, null);
SDDocumentType document = new SDDocumentType(name);
document.setDocumentReferences(new DocumentReferences(emptyMap()));
search.addDocument(document);
- searchDefinitions.put(search.getName(), search);
+ schemas.put(search.getName(), search);
return search;
}
@@ -75,13 +75,13 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase {
}
private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) {
- Map<String, Search> searchDefinitions = createSearchDefinitions();
- List<Search> inputSearchDefinitions = inputNames.stream()
- .map(searchDefinitions::get)
+ Map<String, Search> schemas = createSchemas();
+ List<Search> inputSchemas = inputNames.stream()
+ .map(schemas::get)
.map(Objects::requireNonNull)
.collect(toList());
List<String> actualSearchOrder = new SearchOrderer()
- .order(inputSearchDefinitions)
+ .order(inputSchemas)
.stream()
.map(Search::getName)
.collect(toList());
@@ -104,31 +104,37 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone"));
}
+
@Test
public void testOneLevelReordering() {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone"));
}
+
@Test
public void testMultiLevelReordering() {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone"));
}
+
@Test
public void testAloneIsKeptInPlaceWithMultiLevelReordering() {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product"));
}
+
@Test
public void testPartialMultiLevelReordering() {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone"));
}
+
@Test
public void testMultilevelReorderingAccrossHierarchies() {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone"));
}
+
@Test
public void referees_are_ordered_before_referrer() {
assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"),
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
index f85f9994e04..07d7405b1db 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
@@ -25,7 +25,7 @@ import static org.junit.Assert.assertTrue;
*
* @author bratseth
*/
-public class SummaryMapTestCase extends SearchDefinitionTestCase {
+public class SummaryMapTestCase extends SchemaTestCase {
@Test
public void testDeriving() throws IOException, ParseException {
Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
index b82620b1cf5..afbc9f52f6b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived;
import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertNull;
*
* @author bratseth
*/
-public class SummaryTestCase extends SearchDefinitionTestCase {
+public class SummaryTestCase extends SchemaTestCase {
@Test
public void testDeriving() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
index 24575df8c91..c03e915aa8b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.document.DataType;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.processing.Processing;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse;
*
* @author bratseth
*/
-public class TypeConversionTestCase extends SearchDefinitionTestCase {
+public class TypeConversionTestCase extends SchemaTestCase {
/** Tests that exact-string stuff is not spilled over to the default index */
@Test
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java
new file mode 100644
index 00000000000..e3dcc925e5e
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java
@@ -0,0 +1,45 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.searchdefinition.document;
+
+import java.util.Optional;
+import org.junit.Test;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class HnswIndexParamsTestCase {
+
+ @Test
+ public void override_from() throws Exception {
+ var empty = new HnswIndexParams();
+ var builder = new HnswIndexParams.Builder();
+ builder.setMaxLinksPerNode(7);
+ var one = builder.build();
+ builder.setNeighborsToExploreAtInsert(42);
+ var three = builder.build();
+ builder.setMaxLinksPerNode(17);
+ builder.setNeighborsToExploreAtInsert(500);
+ var four = builder.build();
+
+ assertThat(empty.maxLinksPerNode(), is(16));
+ assertThat(empty.neighborsToExploreAtInsert(), is(200));
+
+ assertThat(one.maxLinksPerNode(), is(7));
+ assertThat(three.neighborsToExploreAtInsert(), is(42));
+
+ assertThat(four.maxLinksPerNode(), is(17));
+ assertThat(four.neighborsToExploreAtInsert(), is(500));
+
+ var five = four.overrideFrom(Optional.of(empty));
+ assertThat(five.maxLinksPerNode(), is(17));
+ assertThat(five.neighborsToExploreAtInsert(), is(500));
+
+ var six = four.overrideFrom(Optional.of(one));
+ assertThat(six.maxLinksPerNode(), is(7));
+ assertThat(six.neighborsToExploreAtInsert(), is(500));
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java
index 7236ccbc117..35ce4dff730 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.Matching;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -17,7 +17,7 @@ import static org.junit.Assert.assertFalse;
* @author vegardh
*
*/
-public class AttributesExactMatchTestCase extends SearchDefinitionTestCase {
+public class AttributesExactMatchTestCase extends SchemaTestCase {
@Test
public void testAttributesExactMatch() throws IOException, ParseException {
Search search = SearchBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
index ac3ba1d98d9..9a4357c5d65 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
@@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.vespa.model.container.search.QueryProfiles;
import org.junit.Test;
@@ -18,7 +18,7 @@ import static org.junit.Assert.fail;
/**
* @author Mathias Mølster Lidal
*/
-public class BoldingTestCase extends SearchDefinitionTestCase {
+public class BoldingTestCase extends SchemaTestCase {
@Test
public void testBoldingNonString() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java
index 620cee49ac4..809ccdb3a3a 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.derived.DerivedConfiguration;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.parser.ParseException;
@@ -18,7 +18,7 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase {
+public class ImplicitSearchFieldsTestCase extends SchemaTestCase {
@Test
public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java
index f2d81414b5a..c9ea57c5b9b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing;
import com.yahoo.document.*;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.parser.ParseException;
@@ -13,7 +13,7 @@ import org.junit.Test;
import java.io.IOException;
import static org.junit.Assert.*;
-public class ImplicitStructTypesTestCase extends SearchDefinitionTestCase {
+public class ImplicitStructTypesTestCase extends SchemaTestCase {
@Test
public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException {
Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java
index 7acbf67772a..ae00e4f3079 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.vespa.documentmodel.DocumentSummary;
import org.junit.Test;
@@ -13,7 +13,7 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-public class ImplicitSummaryFieldsTestCase extends SearchDefinitionTestCase {
+public class ImplicitSummaryFieldsTestCase extends SchemaTestCase {
@Test
public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
index d313c2391fd..7863c544b60 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
@@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.Index;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
@@ -28,7 +28,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author Simon Thoresen Hult
*/
-public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase {
+public class IndexingScriptRewriterTestCase extends SchemaTestCase {
@Test
public void testSetLanguageRewriting() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
index cac50354dc2..fcf1c39f5b4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
@@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.vespa.model.container.search.QueryProfiles;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue;
/**
* @author baldersheim
*/
-public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase {
+public class IntegerIndex2AttributeTestCase extends SchemaTestCase {
@Test
public void testIntegerIndex2Attribute() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java
index 385d1df90ad..c792d3bf40b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.document.Matching;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.document.Stemming;
@@ -21,7 +21,7 @@ import static org.junit.Assert.fail;
/**
* @author bratseth
*/
-public class NGramTestCase extends SearchDefinitionTestCase {
+public class NGramTestCase extends SchemaTestCase {
@Test
public void testNGram() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java
index 0d6334a5223..4ab56f809c9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -14,7 +14,7 @@ import java.io.IOException;
* @author vegardh
*
*/
-public class RankModifierTestCase extends SearchDefinitionTestCase {
+public class RankModifierTestCase extends SchemaTestCase {
@Test
public void testLiteral() throws IOException, ParseException {
Search search = SearchBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index 08dd5148b29..0cd6674751e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException;
import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
import java.util.HashMap;
@@ -33,6 +34,7 @@ class RankProfileSearchFixture {
private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
new OnnxImporter(),
+ new LightGBMImporter(),
new XGBoostImporter());
private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
private final QueryProfileRegistry queryProfileRegistry;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java
index d740884d3e5..502fc4472bc 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java
@@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.RankProfile.RankProperty;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -16,7 +16,7 @@ import java.util.List;
import static org.junit.Assert.fail;
-public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase {
+public class RankPropertyVariablesTestCase extends SchemaTestCase {
@Test
public void testRankPropVariables() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
index 9e9ef2589be..a306e0f2c90 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
@@ -275,7 +275,7 @@ public class RankingExpressionTypeResolverTestCase {
builder.build(true, logger);
String message = logger.findMessage("The following query features");
assertNotNull(message);
- assertEquals("WARNING: The following query features are not declared in query profile types and " +
+ assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " +
"will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]",
message);
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java
new file mode 100644
index 00000000000..79d19371f1c
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java
@@ -0,0 +1,88 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * @author lesters
+ */
+public class RankingExpressionWithLightGBMTestCase {
+
+ private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/");
+
+ private final static String lightGBMExpression =
+ "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)";
+
+ @After
+ public void removeGeneratedModelFiles() {
+ IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+ }
+
+ @Test
+ public void testLightGBMReference() {
+ RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')");
+ search.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+ }
+
+ @Test
+ public void testNestedLightGBMReference() {
+ RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))");
+ search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile");
+ }
+
+ @Test
+ public void testImportingFromStoredExpressions() throws IOException {
+ RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')");
+ search.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+
+ // At this point the expression is stored - copy application to another location which do not have a models dir
+ Path storedApplicationDirectory = applicationDir.getParentPath().append("copy");
+ try {
+ storedApplicationDirectory.toFile().mkdirs();
+ IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
+ storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+ RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory);
+ RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')");
+ searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+ }
+ finally {
+ IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
+ }
+ }
+
+ private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) {
+ return fixtureWith(firstPhaseExpression, null, null,
+ new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir));
+ }
+
+ private RankProfileSearchFixture fixtureWith(String firstPhaseExpression,
+ String constant,
+ String field,
+ RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) {
+ try {
+ RankProfileSearchFixture fixture = new RankProfileSearchFixture(
+ application,
+ application.getQueryProfiles(),
+ " rank-profile my_profile {\n" +
+ " first-phase {\n" +
+ " expression: " + firstPhaseExpression +
+ " }\n" +
+ " }",
+ constant,
+ field);
+ fixture.compileRankProfile("my_profile", applicationDir.append("models"));
+ return fixture;
+ } catch (ParseException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+}
+
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index 96fa59a77cc..b3eda9b7e13 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -19,7 +19,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
-public class RankingExpressionsTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionsTestCase extends SchemaTestCase {
@Test
public void testFunctions() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
index dbcfc8c202d..ca8744a07bb 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.vespa.model.container.search.QueryProfiles;
import org.junit.Test;
@@ -16,7 +16,7 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase {
+public class SummaryFieldsMustHaveValidSourceTestCase extends SchemaTestCase {
@Test
public void requireThatInvalidSourceIsCaught() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
index b6569357495..b9702c6c4f7 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
@@ -1,11 +1,16 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.processing;
-import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.config.model.test.TestUtil;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
+
+import static com.yahoo.searchdefinition.SearchBuilder.createFromString;
+import static com.yahoo.config.model.test.TestUtil.joinLines;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
@@ -16,7 +21,7 @@ public class TensorFieldTestCase {
@Test
public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException {
try {
- SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}"));
+ createFromString(getSd("field f1 type array<tensor(x{})> {}"));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
@@ -28,11 +33,12 @@ public class TensorFieldTestCase {
@Test
public void requireThatTensorFieldCannotBeIndexField() throws ParseException {
try {
- SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }"));
+ createFromString(getSd("field f1 type tensor(x{}) { indexing: index }"));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
- assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.",
+ assertEquals("For search 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " +
+ "Currently, only tensors with 1 indexed dimension supports that.",
e.getMessage());
}
}
@@ -40,7 +46,7 @@ public class TensorFieldTestCase {
@Test
public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException {
try {
- SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }"));
+ createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }"));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
@@ -51,7 +57,7 @@ public class TensorFieldTestCase {
@Test
public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException {
try {
- SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }"));
+ createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }"));
fail("Expected exception");
}
catch (IllegalArgumentException e) {
@@ -59,8 +65,67 @@ public class TensorFieldTestCase {
}
}
+ @Test
+ public void hnsw_index_is_default_turned_off() throws ParseException {
+ var attr = createFromString(getSd("field t1 type tensor(x[64]) { indexing: attribute }"))
+ .getSearch().getAttribute("t1");
+ assertFalse(attr.hnswIndexParams().isPresent());
+ }
+
+ @Test
+ public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException {
+ assertHnswIndexParams("", 16, 200);
+ assertHnswIndexParams("index: hnsw", 16, 200);
+ }
+
+ @Test
+ public void hnsw_index_parameters_can_be_specified() throws ParseException {
+ assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200);
+ assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300);
+ assertHnswIndexParams(joinLines("index {",
+ " hnsw {",
+ " max-links-per-node: 32",
+ " neighbors-to-explore-at-insert: 300",
+ " }",
+ "}"),
+ 32, 300);
+ }
+
+ @Test
+ public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException {
+ try {
+ createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }"));
+ fail("Expected exception");
+ }
+ catch (IllegalArgumentException e) {
+ assertEquals("For search 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage());
+ }
+ }
+
private static String getSd(String field) {
- return "search test {\n document test {\n" + field + "}\n}\n";
+ return joinLines("search test {",
+ " document test {",
+ " " + field,
+ " }",
+ "}");
+ }
+
+ private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException {
+ var sd = getSdWithIndexSpec(indexSpec);
+ System.out.println(sd);
+ var search = createFromString(sd).getSearch();
+ var attr = search.getAttribute("t1");
+ var params = attr.hnswIndexParams();
+ assertTrue(params.isPresent());
+ assertEquals(maxLinksPerNode, params.get().maxLinksPerNode());
+ assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert());
+ }
+
+ private String getSdWithIndexSpec(String indexSpec) {
+ return getSd(joinLines("field t1 type tensor(x[64]) {",
+ " indexing: attribute | index",
+ " " + indexSpec,
+ "}"));
}
private void assertStartsWith(String prefix, String string) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index f90320ad686..8308b638497 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -13,7 +13,7 @@ import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.derived.AttributeFields;
import com.yahoo.searchdefinition.derived.RawRankProfile;
import com.yahoo.searchdefinition.parser.ParseException;
@@ -25,7 +25,7 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-public class TensorTransformTestCase extends SearchDefinitionTestCase {
+public class TensorTransformTestCase extends SchemaTestCase {
@Test
public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java
index ef6bc57223d..957b5c55889 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import org.junit.Test;
@@ -12,7 +12,7 @@ import java.io.IOException;
import static org.junit.Assert.assertNotNull;
/** @author bratseth */
-public class WeightedSetSummaryToTestCase extends SearchDefinitionTestCase {
+public class WeightedSetSummaryToTestCase extends SchemaTestCase {
@Test
public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
new file mode 100644
index 00000000000..9144ad411b2
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
@@ -0,0 +1,35 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SchemaTestCase;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+
+import java.io.IOException;
+
+/**
+ * Utility functions for testing generated configs for reference/imported fields.
+ */
+public abstract class AbstractReferenceFieldTestCase extends SchemaTestCase {
+
+ private static String TEST_FOLDER = "src/test/configmodel/types/references/";
+
+ protected void assertDocumentConfigs(DocumentModel model,
+ String cfgFileSpec) throws IOException {
+ assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
+ assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
+ }
+
+ protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
+ DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
+ assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
+ }
+
+ protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
+ DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
+ assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
new file mode 100644
index 00000000000..599ae77a456
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+
+public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase {
+
+ @Test
+ public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException {
+ assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines(
+ "search ad {",
+ " document ad {",
+ " field campaign_ref type reference<campaign> { indexing: attribute }",
+ " field person_ref type reference<person> { indexing: attribute }",
+ " }",
+ " import field campaign_ref.cool_field as my_cool_field {}",
+ " import field campaign_ref.swag_field as my_swag_field {}",
+ " import field person_ref.name as my_name {}",
+ "}")),
+ "multiple_imported_fields");
+ }
+
+ private static class TestDocumentModelBuilder {
+ private final SearchBuilder builder = new SearchBuilder();
+ public TestDocumentModelBuilder addCampaign() throws ParseException {
+ builder.importString(joinLines("search campaign {",
+ " document campaign {",
+ " field cool_field type string { indexing: attribute }",
+ " field swag_field type long { indexing: attribute }",
+ " }",
+ "}"));
+ return this;
+ }
+ public TestDocumentModelBuilder addPerson() throws ParseException {
+ builder.importString(joinLines("search person {",
+ " document person {",
+ " field name type string { indexing: attribute }",
+ " }",
+ "}"));
+ return this;
+ }
+ public DocumentModel build(String adSdContent) throws ParseException {
+ builder.importString(adSdContent);
+ builder.build();
+ return builder.getModel();
+ }
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
index e9a7a6ed33e..55980ee5fea 100644
--- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
@@ -1,15 +1,10 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.documentmodel;
-import com.yahoo.document.DocumenttypesConfig;
import com.yahoo.document.ReferenceDataType;
-import com.yahoo.document.config.DocumentmanagerConfig;
import com.yahoo.documentmodel.NewDocumentType;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
-import com.yahoo.vespa.configmodel.producers.DocumentManager;
-import com.yahoo.vespa.configmodel.producers.DocumentTypes;
import org.junit.Test;
import java.io.IOException;
@@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author geirst
*/
-public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase {
+public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase {
@Test
public void reference_fields_can_reference_other_document_types() throws ParseException, IOException {
@@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT
assertEquals(campaignRefType.getTargetType(), campaignType);
}
- private static String TEST_FOLDER = "src/test/configmodel/types/references/";
-
- private void assertDocumentConfigs(DocumentModel model,
- String cfgFileSpec) throws IOException {
- assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
- assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
- }
-
- private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
- DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
- assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
- }
-
- private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
- DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
- assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
- }
-
private static class TestDocumentModelBuilder {
private final SearchBuilder builder = new SearchBuilder();
public TestDocumentModelBuilder addCampaign() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java
index f5034d0530b..91152648b10 100644
--- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.documentmodel;
import com.yahoo.document.DocumenttypesConfig;
import com.yahoo.document.config.DocumentmanagerConfig;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.vespa.configmodel.producers.DocumentManager;
import com.yahoo.vespa.configmodel.producers.DocumentTypes;
@@ -13,7 +13,7 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-public class DocumentModelBuilderTestCase extends SearchDefinitionTestCase {
+public class DocumentModelBuilderTestCase extends SchemaTestCase {
@Test
public void testDocumentManagerSimple() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
index 234841f2b6c..6aea0593f8a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model;
-import com.yahoo.component.Version;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.config.provision.ClusterMembership;
@@ -9,17 +8,12 @@ import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostSpec;
import org.junit.Test;
-import java.util.Arrays;
import java.util.Optional;
-import static com.yahoo.config.provision.ClusterSpec.Type.admin;
import static com.yahoo.config.provision.ClusterSpec.Type.container;
-import static com.yahoo.config.provision.ClusterSpec.Type.content;
import static org.hamcrest.Matchers.endsWith;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
/**
* @author gjoranv
@@ -54,7 +48,7 @@ public class HostResourceTest {
}
private static ClusterSpec clusterSpec(ClusterSpec.Type type, String id) {
- return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(0), Version.fromString("6.42"), false);
+ return ClusterSpec.specification(type, ClusterSpec.Id.from(id)).group(ClusterSpec.Group.from(0)).vespaVersion("6.42").build();
}
private static HostResource hostResourceWithMemberships(ClusterMembership membership) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
index 90ec1779f39..cf1ae637cf9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model;
-import com.yahoo.component.Version;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.MockModelContext;
import com.yahoo.config.model.NullConfigModelRegistry;
@@ -23,7 +22,6 @@ import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -105,23 +103,23 @@ public class VespaModelFactoryTest {
@Override
public HostSpec allocateHost(String alias) {
return new HostSpec(hostName,
- Collections.emptyList(),
- ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.admin,
- new ClusterSpec.Id(routingClusterName),
- ClusterSpec.Group.from(0),
- Version.fromString("6.42"), false),
+ List.of(),
+ ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
0));
}
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
- return Collections.singletonList(new HostSpec(hostName,
- Collections.emptyList(),
- ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.container,
- new ClusterSpec.Id(routingClusterName),
- ClusterSpec.Group.from(0),
- Version.fromString("6.42"), false),
- 0)));
+ return prepare(cluster, capacity.withGroups(groups), logger);
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+ return List.of(new HostSpec(hostName,
+ List.of(),
+ ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
+ 0)));
}
};
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java
index cd67e432b8f..4b23a7e2e71 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java
@@ -52,7 +52,7 @@ public class ClusterControllerTestCase extends DomBuilderTest {
@Before
public void setup() {
- sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+ sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
}
@Test
@@ -457,7 +457,7 @@ public class ClusterControllerTestCase extends DomBuilderTest {
private VespaModel createVespaModel(String servicesXml, boolean isHosted) throws IOException, SAXException {
ApplicationPackage applicationPackage = new MockApplicationPackage.Builder()
.withServices(servicesXml)
- .withSearchDefinitions(sds)
+ .withSchemas(sds)
.build();
// Need to create VespaModel to make deploy properties have effect
DeployLogger logger = new DeployLoggerStub();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java
new file mode 100644
index 00000000000..12a10a7e354
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java
@@ -0,0 +1,244 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.monitoring.Metric;
+import com.yahoo.vespa.model.admin.monitoring.MetricSet;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
+import static java.util.Collections.singleton;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers.
+ *
+ * @author gjoranv
+ */
+public class MetricsConsumersTest {
+
+ private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size();
+ private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size();
+ private static int numVespaMetrics = vespaMetricSet.getMetrics().size();
+ private static int numSystemMetrics = systemMetricSet.getMetrics().size();
+ private static int numNetworkMetrics = networkMetricSet.getMetrics().size();
+ private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void default_public_consumer_is_set_up_for_self_hosted() {
+ ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
+ assertEquals(2, config.consumer().size());
+ assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
+
+ int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics;
+ assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size());
+ }
+
+ @Test
+ public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() {
+ ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted);
+ assertEquals(2, config.consumer().size());
+ assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
+ assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
+ }
+
+ @Test
+ public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() {
+ ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
+ assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
+ assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size());
+ }
+
+ @Test
+ public void vespa_consumer_can_be_amended_via_admin_object() {
+ VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
+ var additionalMetric = new Metric("additional-metric");
+ model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric)));
+
+ ConsumersConfig config = consumersConfigFromModel(model);
+ assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size());
+
+ ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
+ assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric));
+ }
+
+ @Test
+ public void vespa_is_a_reserved_consumer_id() {
+ assertReservedConsumerId("Vespa");
+ }
+
+ @Test
+ public void default_is_a_reserved_consumer_id() {
+ assertReservedConsumerId("default");
+ }
+
+ private void assertReservedConsumerId(String consumerId) {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='" + consumerId + "'/>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id");
+ consumersConfigFromXml(services, self_hosted);
+ }
+
+ @Test
+ public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() {
+ String services = String.join("\n",
+ "<services application-type='hosted-infrastructure'>",
+ " <admin version='4.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='Vespa'>",
+ " <metric id='custom.metric1'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ VespaModel hostedModel = getModel(services, hosted);
+ ConsumersConfig config = consumersConfigFromModel(hostedModel);
+ assertEquals(2, config.consumer().size());
+
+ // All default metrics are retained
+ ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
+ assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size());
+
+ Metric customMetric1 = new Metric("custom.metric1");
+ assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1));
+ }
+
+ @Test
+ public void consumer_id_is_case_insensitive() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='A'/>",
+ " <consumer id='a'/>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("'a' is used as id for two metrics consumers");
+ consumersConfigFromXml(services, self_hosted);
+ }
+
+ @Test
+ public void non_existent_metric_set_causes_exception() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-non-existent-default-set'>",
+ " <metric-set id='non-existent'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("No such metric-set: non-existent");
+ consumersConfigFromXml(services, self_hosted);
+ }
+
+ @Test
+ public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-metrics-only'>",
+ " <metric id='custom.metric1'/>",
+ " <metric id='custom.metric2'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+
+ assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size());
+
+ Metric customMetric1 = new Metric("custom.metric1");
+ Metric customMetric2 = new Metric("custom.metric2");
+ assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1));
+ assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2));
+ }
+
+ @Test
+ public void consumer_with_default_metric_set_has_all_its_metrics_plus_all_system_metrics_plus_its_own() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-public-default-set'>",
+ " <metric-set id='default'/>",
+ " <metric id='custom.metric'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+
+ assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size());
+
+ Metric customMetric = new Metric("custom.metric");
+ assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
+ }
+
+ @Test
+ public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-vespa-set'>",
+ " <metric-set id='vespa'/>",
+ " <metric id='my.extra.metric'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+ assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size());
+
+ Metric customMetric = new Metric("my.extra.metric");
+ assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
index de40e557265..bed77bd5c77 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
@@ -5,10 +5,9 @@
package com.yahoo.vespa.model.admin.metricsproxy;
-import ai.vespa.metricsproxy.core.ConsumersConfig;
-import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler;
import ai.vespa.metricsproxy.http.application.MetricsNodesConfig;
+import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler;
import ai.vespa.metricsproxy.http.yamas.YamasHandler;
import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
@@ -21,13 +20,9 @@ import com.yahoo.container.core.ApplicationMetadataConfig;
import com.yahoo.search.config.QrStartConfig;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames;
-import com.yahoo.vespa.model.admin.monitoring.Metric;
-import com.yahoo.vespa.model.admin.monitoring.MetricSet;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.Handler;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
import java.util.Collection;
@@ -39,26 +34,17 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.M
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getMetricsNodesConfig;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
-import static java.util.Collections.singleton;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
import static java.util.stream.Collectors.toList;
import static org.hamcrest.CoreMatchers.endsWith;
import static org.hamcrest.CoreMatchers.hasItem;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -67,15 +53,6 @@ import static org.junit.Assert.assertTrue;
*/
public class MetricsProxyContainerClusterTest {
- private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size();
- private static int numVespaMetrics = vespaMetricSet.getMetrics().size();
- private static int numSystemMetrics = systemMetricSet.getMetrics().size();
- private static int numNetworkMetrics = networkMetricSet.getMetrics().size();
- private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics;
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@Test
public void metrics_proxy_bundle_is_included_in_bundles_config() {
VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
@@ -104,10 +81,11 @@ public class MetricsProxyContainerClusterTest {
assertEquals(512, qrStartConfig.jvm().heapsize());
assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
assertEquals(2, qrStartConfig.jvm().availableProcessors());
- assertEquals(false, qrStartConfig.jvm().verbosegc());
+ assertFalse(qrStartConfig.jvm().verbosegc());
assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts());
assertEquals(512, qrStartConfig.jvm().stacksize());
assertEquals(0, qrStartConfig.jvm().directMemorySizeCache());
+ assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize());
}
@@ -130,161 +108,6 @@ public class MetricsProxyContainerClusterTest {
}
@Test
- public void default_public_consumer_is_set_up_for_self_hosted() {
- ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
- assertEquals(2, config.consumer().size());
- assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
-
- int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics;
- assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size());
- }
-
- @Test
- public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() {
- ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted);
- assertEquals(2, config.consumer().size());
- assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
- assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
- }
-
- @Test
- public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() {
- ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
- assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
- assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size());
- }
-
- @Test
- public void vespa_consumer_can_be_amended_via_admin_object() {
- VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
- var additionalMetric = new Metric("additional-metric");
- model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric)));
-
- ConsumersConfig config = consumersConfigFromModel(model);
- assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size());
-
- ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
- assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric));
- }
-
- @Test
- public void vespa_is_a_reserved_consumer_id() {
- assertReservedConsumerId("Vespa");
- }
-
- @Test
- public void default_is_a_reserved_consumer_id() {
- assertReservedConsumerId("default");
- }
-
- private void assertReservedConsumerId(String consumerId) {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='" + consumerId + "'/>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id");
- consumersConfigFromXml(services, self_hosted);
- }
-
- @Test
- public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() {
- String services = String.join("\n",
- "<services application-type='hosted-infrastructure'>",
- " <admin version='4.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='Vespa'>",
- " <metric id='custom.metric1'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- VespaModel hostedModel = getModel(services, hosted);
- ConsumersConfig config = consumersConfigFromModel(hostedModel);
- assertEquals(2, config.consumer().size());
-
- // All default metrics are retained
- ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
- assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size());
-
- Metric customMetric1 = new Metric("custom.metric1");
- assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1));
- }
-
- @Test
- public void consumer_id_is_case_insensitive() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='A'/>",
- " <consumer id='a'/>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("'a' is used as id for two metrics consumers");
- consumersConfigFromXml(services, self_hosted);
- }
-
- @Test
- public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='consumer-with-metrics-only'>",
- " <metric id='custom.metric1'/>",
- " <metric id='custom.metric2'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- ConsumersConfig.Consumer consumer = getCustomConsumer(services);
-
- assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size());
-
- Metric customMetric1 = new Metric("custom.metric1");
- Metric customMetric2 = new Metric("custom.metric2");
- assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1));
- assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2));
- }
-
- @Test
- public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='consumer-with-vespa-set'>",
- " <metric-set id='vespa'/>",
- " <metric id='my.extra.metric'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- ConsumersConfig.Consumer consumer = getCustomConsumer(services);
- assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size());
-
- Metric customMetric = new Metric("my.extra.metric");
- assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
- }
-
- @Test
public void hosted_application_propagates_application_dimensions() {
VespaModel hostedModel = getModel(servicesWithAdminOnly(), hosted);
ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel);
@@ -314,16 +137,6 @@ public class MetricsProxyContainerClusterTest {
assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath());
}
- private static String servicesWithAdminOnly() {
- return String.join("\n",
- "<services>",
- " <admin version='4.0'>",
- " <adminserver hostalias='node1'/>",
- " </admin>",
- "</services>"
- );
- }
-
private static String servicesWithTwoNodes() {
return String.join("\n",
"<services>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
index 621cebd6246..eddad6fce89 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
@@ -1,6 +1,7 @@
// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.admin.metricsproxy;
+import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig;
import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
import ai.vespa.metricsproxy.rpc.RpcConnectorConfig;
@@ -10,10 +11,10 @@ import com.yahoo.vespa.model.test.VespaModelTester;
import org.junit.Test;
import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig;
@@ -90,13 +91,28 @@ public class MetricsProxyContainerTest {
String services = servicesWithContent();
VespaModel hostedModel = getModel(services, hosted);
assertEquals(1, hostedModel.getHosts().size());
- String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname();
+ String configId = containerConfigId(hostedModel, hosted);
NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId);
assertEquals("content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_TYPE));
assertEquals("my-content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_ID));
}
+ @Test
+ public void metrics_v2_handler_is_set_up_with_node_info_config() {
+ String services = servicesWithContent();
+ VespaModel hostedModel = getModel(services, hosted);
+
+ var container = (MetricsProxyContainer)hostedModel.id2producer().get(containerConfigId(hostedModel, hosted));
+ var handlers = container.getHandlers().getComponents();
+
+ assertEquals(1, handlers.size());
+ var metricsV2Handler = handlers.iterator().next();
+
+ NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId());
+ assertTrue(config.role().startsWith("content/my-content/0/"));
+ assertTrue(config.hostname().startsWith("node-1-3-9-"));
+ }
@Test
public void vespa_services_config_has_all_services() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
index f3140aafdaf..8ecb13d7ae5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
@@ -49,12 +49,22 @@ class MetricsProxyModelTester {
return tester.createModel(servicesXml, true);
}
- static String configId(VespaModel model, MetricsProxyModelTester.TestMode mode) {
+ static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) {
return (mode == hosted)
? CLUSTER_CONFIG_ID + "/" + model.getHosts().iterator().next().getHostname()
: CONTAINER_CONFIG_ID;
}
+ static String servicesWithAdminOnly() {
+ return String.join("\n",
+ "<services>",
+ " <admin version='4.0'>",
+ " <adminserver hostalias='node1'/>",
+ " </admin>",
+ "</services>"
+ );
+ }
+
static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) {
for (ConsumersConfig.Consumer.Metric m : consumer.metric()) {
if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname()))
@@ -77,32 +87,32 @@ class MetricsProxyModelTester {
}
static ConsumersConfig consumersConfigFromModel(VespaModel model) {
- return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(ConsumersConfig.class, CLUSTER_CONFIG_ID);
}
static MetricsNodesConfig getMetricsNodesConfig(VespaModel model) {
- return new MetricsNodesConfig((MetricsNodesConfig.Builder) model.getConfig(new MetricsNodesConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(MetricsNodesConfig.class, CLUSTER_CONFIG_ID);
}
static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) {
- return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID);
}
static QrStartConfig getQrStartConfig(VespaModel model) {
- return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID);
}
static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) {
- return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId));
+ return model.getConfig(NodeDimensionsConfig.class, configId);
}
static VespaServicesConfig getVespaServicesConfig(String servicesXml) {
VespaModel model = getModel(servicesXml, self_hosted);
- return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID));
+ return model.getConfig(VespaServicesConfig.class, CONTAINER_CONFIG_ID);
}
static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) {
- return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID));
+ return model.getConfig(RpcConnectorConfig.class, CONTAINER_CONFIG_ID);
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
new file mode 100644
index 00000000000..9a904592744
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
@@ -0,0 +1,151 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.telegraf.Telegraf;
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import ai.vespa.metricsproxy.telegraf.TelegrafRegistry;
+import com.yahoo.component.ComponentId;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Test;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class TelegrafTest {
+
+ @Test
+ public void telegraf_components_are_set_up_when_cloudwatch_is_configured() {
+ String services = servicesWithCloudwatch();
+ VespaModel hostedModel = getModel(services, hosted);
+
+ var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap();
+ assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName())));
+ assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())));
+ }
+
+ @Test
+ public void telegraf_components_are_not_set_up_when_no_external_systems_are_added_in_services() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='foo' />",
+ " </metrics>",
+ " </admin>",
+ "</services>");
+ VespaModel hostedModel = getModel(services, hosted);
+
+ var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap();
+ assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName()))));
+ assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))));
+ }
+
+ @Test
+ public void telegraf_config_is_generated_for_cloudwatch_in_services() {
+ String services = servicesWithCloudwatch();
+ VespaModel hostedModel = getModel(services, hosted);
+ TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+ assertTrue(config.isHostedVespa());
+
+ var cloudWatch0 = config.cloudWatch(0);
+ assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+ assertEquals("us-east-1", cloudWatch0.region());
+ assertEquals("my-namespace", cloudWatch0.namespace());
+ assertEquals("my-access-key", cloudWatch0.accessKeyName());
+ assertEquals("my-secret-key", cloudWatch0.secretKeyName());
+ assertEquals("default", cloudWatch0.profile());
+ }
+
+ private String servicesWithCloudwatch() {
+ return String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='cloudwatch-consumer'>",
+ " <metric id='my-metric'/>",
+ " <cloudwatch region='us-east-1' namespace='my-namespace' >",
+ " <credentials access-key-name='my-access-key' ",
+ " secret-key-name='my-secret-key' />",
+ " </cloudwatch>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ }
+
+ @Test
+ public void multiple_cloudwatches_are_allowed_for_the_same_consumer() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='cloudwatch-consumer'>",
+ " <metric id='my-metric'/>",
+ " <cloudwatch region='us-east-1' namespace='namespace-1' >",
+ " <credentials access-key-name='access-key-1' ",
+ " secret-key-name='secret-key-1' />",
+ " </cloudwatch>",
+ " <cloudwatch region='us-east-1' namespace='namespace-2' >",
+ " <shared-credentials profile='profile-2' />",
+ " </cloudwatch>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ VespaModel hostedModel = getModel(services, hosted);
+ TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+
+ var cloudWatch0 = config.cloudWatch(0);
+ assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+ assertEquals("us-east-1", cloudWatch0.region());
+ assertEquals("namespace-1", cloudWatch0.namespace());
+ assertEquals("access-key-1", cloudWatch0.accessKeyName());
+ assertEquals("secret-key-1", cloudWatch0.secretKeyName());
+ assertEquals("default", cloudWatch0.profile());
+
+ var cloudWatch1 = config.cloudWatch(1);
+ assertEquals("cloudwatch-consumer", cloudWatch1.consumer());
+ assertEquals("us-east-1", cloudWatch1.region());
+ assertEquals("namespace-2", cloudWatch1.namespace());
+ assertEquals("", cloudWatch1.accessKeyName());
+ assertEquals("", cloudWatch1.secretKeyName());
+ assertEquals("profile-2", cloudWatch1.profile());
+ }
+
+ @Test
+ public void profile_named_default_is_used_when_no_profile_is_given_in_shared_credentials() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='cloudwatch-consumer'>",
+ " <metric id='my-metric'/>",
+ " <cloudwatch region='us-east-1' namespace='foo' >",
+ " <shared-credentials file='/path/to/file' />",
+ " </cloudwatch>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ VespaModel model = getModel(services, self_hosted);
+ TelegrafConfig config = model.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+ assertEquals("default", config.cloudWatch(0).profile());
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java
new file mode 100644
index 00000000000..58b65199729
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java
@@ -0,0 +1,174 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+import static com.yahoo.config.provision.Environment.prod;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public abstract class AccessControlValidatorTestBase {
+
+ protected Validator validator;
+ protected Zone zone;
+
+ @Rule
+ public final ExpectedException exceptionRule = ExpectedException.none();
+
+ private static String servicesXml(boolean addHandler, boolean protection) {
+ return joinLines("<services version='1.0'>",
+ " <container id='default' version='1.0'>",
+ addHandler ? httpHandlerXml : "",
+ " <http>",
+ " <filtering>",
+ " <access-control domain='foo' read='" + protection + "' write='" + protection + "' />",
+ " </filtering>",
+ " </http>",
+ " </container>",
+ "</services>");
+ }
+
+ private static final String httpHandlerXml =
+ joinLines(" <handler id='foo'>",
+ " <binding>http://foo/bar</binding>",
+ " </handler>");
+
+ @Test
+ public void cluster_with_protection_passes_validation() throws IOException, SAXException {
+ DeployState deployState = deployState(servicesXml(true, true));
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void cluster_with_no_handlers_passes_validation_without_protection() throws IOException, SAXException{
+ DeployState deployState = deployState(servicesXml(false, false));
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void cluster_without_custom_components_passes_validation_without_protection() throws IOException, SAXException{
+ String servicesXml = joinLines("<services version='1.0'>",
+ " <container id='default' version='1.0' />",
+ "</services>");
+ DeployState deployState = deployState(servicesXml);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void cluster_with_handler_fails_validation_without_protection() throws IOException, SAXException{
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage(containsString("Access-control must be enabled"));
+ exceptionRule.expectMessage(containsString("production zones: [default]"));
+
+ DeployState deployState = deployState(servicesXml(true, false));
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void no_http_element_has_same_effect_as_no_write_protection() throws IOException, SAXException{
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage(containsString("Access-control must be enabled"));
+ exceptionRule.expectMessage(containsString("production zones: [default]"));
+
+ String servicesXml = joinLines("<services version='1.0'>",
+ " <container id='default' version='1.0'>",
+ httpHandlerXml,
+ " </container>",
+ "</services>");
+ DeployState deployState = deployState(servicesXml);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void cluster_with_mbus_handler_passes_validation_without_write_protection() throws IOException, SAXException{
+ String servicesXml = joinLines("<services version='1.0'>",
+ " <container id='default' version='1.0'>",
+ " <handler id='foo'>",
+ " <binding>mbus://*/foo</binding>",
+ " </handler>",
+ " </container>",
+ "</services>");
+ DeployState deployState = deployState(servicesXml);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void write_protection_is_not_required_for_non_default_application_type() throws IOException, SAXException{
+ String servicesXml = joinLines("<services version='1.0' application-type='hosted-infrastructure'>",
+ " <container id='default' version='1.0'>",
+ httpHandlerXml,
+ " </container>",
+ "</services>");
+ DeployState deployState = deployState(servicesXml);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ @Test
+ public void write_protection_is_not_required_with_validation_override() throws IOException, SAXException{
+ DeployState deployState = deployState(servicesXml(true, false),
+ "<validation-overrides><allow until='2000-01-30'>access-control</allow></validation-overrides>",
+ LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant());
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ validator.validate(model, deployState);
+ }
+
+ private DeployState deployState(String servicesXml) {
+ return deployState(servicesXml, "<validation-overrides></validation-overrides>", Instant.now());
+ }
+
+ private DeployState deployState(String servicesXml, String validationOverrides, Instant now) {
+ ApplicationPackage app = new MockApplicationPackage.Builder()
+ .withServices(servicesXml)
+ .withValidationOverrides(validationOverrides)
+ .build();
+
+ DeployState.Builder builder = new DeployState.Builder()
+ .applicationPackage(app)
+ .zone(zone)
+ .properties(new TestProperties().setHostedVespa(true))
+ .now(now);
+ final DeployState deployState = builder.build();
+
+ assertTrue("Test must emulate a hosted deployment.", deployState.isHosted());
+ assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment());
+
+ return deployState;
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java
new file mode 100644
index 00000000000..4eb69ec405a
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java
@@ -0,0 +1,24 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import org.junit.Before;
+
+import static com.yahoo.vespa.model.application.validation.AwsAccessControlValidator.AWS_CLOUD_NAME;
+
+/**
+ * @author gjoranv
+ */
+public class AwsAccessControlValidatorTest extends AccessControlValidatorTestBase {
+
+ @Before
+ public void setup() {
+ validator = new AwsAccessControlValidator();
+ zone = new Zone(CloudName.from(AWS_CLOUD_NAME), SystemName.main, Environment.prod, RegionName.from("foo"));
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java
new file mode 100644
index 00000000000..40b8223479d
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java
@@ -0,0 +1,101 @@
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+
+import static com.yahoo.config.provision.Environment.prod;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchValidatorTest {
+
+ @Rule
+ public final ExpectedException exceptionRule = ExpectedException.none();
+
+ @Test
+ public void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException {
+ DeployState deployState = deployState(servicesWithCloudwatch(), true, true);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ new CloudWatchValidator().validate(model, deployState);
+ }
+
+ @Test
+ public void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException {
+ DeployState deployState = deployState(servicesWithCloudwatch(), false, false);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ new CloudWatchValidator().validate(model, deployState);
+ }
+
+ @Test
+ public void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException {
+ exceptionRule.expect(IllegalArgumentException.class);
+ exceptionRule.expectMessage(
+ "CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]");
+
+ DeployState deployState = deployState(servicesWithCloudwatch(), true, false);
+ VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+ new CloudWatchValidator().validate(model, deployState);
+ }
+
+ private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) {
+ ApplicationPackage app = new MockApplicationPackage.Builder()
+ .withServices(servicesXml)
+ .build();
+
+ DeployState.Builder builder = new DeployState.Builder()
+ .applicationPackage(app)
+ .properties(new TestProperties().setHostedVespa(isHosted));
+ if (isHosted) {
+ var system = isPublic ? SystemName.Public : SystemName.main;
+ builder.zone(new Zone(system, Environment.prod, RegionName.from("foo")));
+ }
+ final DeployState deployState = builder.build();
+
+ if (isHosted) {
+ assertTrue("Test must emulate a hosted deployment.", deployState.isHosted());
+ assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment());
+ }
+ return deployState;
+ }
+
+ private String servicesWithCloudwatch() {
+ return String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='cloudwatch-consumer'>",
+ " <metric id='my-metric'/>",
+ " <cloudwatch region='us-east-1' namespace='my-namespace' >",
+ " <credentials access-key-name='my-access-key' ",
+ " secret-key-name='my-secret-key' />",
+ " </cloudwatch>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java
index c6d56455d44..e0366b62933 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.application.validation;
-import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.NullConfigModelRegistry;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.test.MockApplicationPackage;
@@ -11,8 +10,7 @@ import org.xml.sax.SAXException;
import java.io.IOException;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
@@ -21,48 +19,71 @@ import static org.junit.Assert.fail;
public class DeploymentSpecValidatorTest {
@Test
- public void testDeploymentWithNonExistentGlobalId() throws IOException, SAXException {
- final String simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
- "<hosts> " +
- "<host name=\"localhost\">" +
- "<alias>node0</alias>" +
- "</host>" +
- "</hosts>";
-
- final String services = "<services version='1.0'>" +
- " <admin version='2.0'>" +
- " <adminserver hostalias='node0' />" +
- " </admin>" +
- " <container id='default' version='1.0'>" +
- " <search/>" +
- " <nodes>" +
- " <node hostalias='node0'/>" +
- " </nodes>" +
- " </container>" +
- "</services>";
-
- final String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>" +
+ public void testDeploymentWithNonExistentGlobalId() {
+ var deploymentXml = "<?xml version='1.0' encoding='UTF-8'?>" +
"<deployment version='1.0'>" +
" <test />" +
" <prod global-service-id='non-existing'>" +
" <region active='true'>us-east</region>" +
" </prod>" +
"</deployment>";
+ assertValidationError("Attribute 'globalServiceId' in instance default: 'non-existing' specified in " +
+ "deployment.xml does not match any container cluster ID", deploymentXml);
+ }
- ApplicationPackage app = new MockApplicationPackage.Builder()
+ @Test
+ public void testEndpointNonExistentContainerId() {
+ var deploymentXml = "<?xml version='1.0' encoding='UTF-8'?>" +
+ "<deployment version='1.0'>" +
+ " <test />" +
+ " <prod>" +
+ " <region active='true'>us-east</region>" +
+ " </prod>" +
+ " <endpoints>" +
+ " <endpoint container-id='non-existing'/>" +
+ " </endpoints>" +
+ "</deployment>";
+ assertValidationError("Endpoint 'default' in instance default: 'non-existing' specified in " +
+ "deployment.xml does not match any container cluster ID", deploymentXml);
+ }
+
+ private static void assertValidationError(String message, String deploymentXml) {
+ var simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
+ "<hosts> " +
+ "<host name=\"localhost\">" +
+ "<alias>node0</alias>" +
+ "</host>" +
+ "</hosts>";
+
+ var services = "<services version='1.0'>" +
+ " <admin version='2.0'>" +
+ " <adminserver hostalias='node0' />" +
+ " </admin>" +
+ " <container id='default' version='1.0'>" +
+ " <search/>" +
+ " <nodes>" +
+ " <node hostalias='node0'/>" +
+ " </nodes>" +
+ " </container>" +
+ "</services>";
+
+ var app = new MockApplicationPackage.Builder()
.withHosts(simpleHosts)
.withServices(services)
- .withDeploymentSpec(deploymentSpec)
+ .withDeploymentSpec(deploymentXml)
.build();
- DeployState.Builder builder = new DeployState.Builder().applicationPackage(app);
+ var builder = new DeployState.Builder().applicationPackage(app);
try {
- final DeployState deployState = builder.build();
+ var deployState = builder.build();
VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
new DeploymentSpecValidator().validate(model, deployState);
fail("Did not get expected exception");
} catch (IllegalArgumentException e) {
- assertThat(e.getMessage(), containsString("specified in deployment.xml does not match any container cluster id"));
+ assertEquals(message, e.getMessage());
+ } catch (SAXException|IOException e) {
+ throw new RuntimeException(e);
}
}
}
+
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
index 21df39ebde8..318a0630c4d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
@@ -47,7 +47,7 @@ public class EndpointCertificateSecretsValidatorTest {
VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
exceptionRule.expect(CertificateNotReadyException.class);
- exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet");
+ exceptionRule.expectMessage("TLS enabled, but could not yet retrieve certificate for application default:default:default");
new EndpointCertificateSecretsValidator().validate(model, deployState);
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
index f7b6d7b25e1..915b3c01e1b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
@@ -6,6 +6,7 @@ import com.yahoo.collections.Pair;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.api.ConfigChangeAction;
import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.config.model.provision.InMemoryProvisioner;
@@ -32,7 +33,7 @@ import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SEARCHDEF
*/
public class ValidationTester {
- private final HostProvisioner hostProvisioner;
+ private final InMemoryProvisioner hostProvisioner;
/** Creates a validation tester with 1 node available */
public ValidationTester() {
@@ -45,7 +46,7 @@ public class ValidationTester {
}
/** Creates a validation tester with a given host provisioner */
- public ValidationTester(HostProvisioner hostProvisioner) {
+ public ValidationTester(InMemoryProvisioner hostProvisioner) {
this.hostProvisioner = hostProvisioner;
}
@@ -63,9 +64,10 @@ public class ValidationTester {
Environment environment,
String validationOverrides) {
Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant();
+ Provisioned provisioned = hostProvisioner.startProvisionedRecording();
ApplicationPackage newApp = new MockApplicationPackage.Builder()
.withServices(services)
- .withSearchDefinitions(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION))
+ .withSchemas(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION))
.withValidationOverrides(validationOverrides)
.build();
VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp);
@@ -77,6 +79,7 @@ public class ValidationTester {
.applicationPackage(newApp)
.properties(new TestProperties().setHostedVespa(true))
.modelHostProvisioner(hostProvisioner)
+ .provisioned(provisioned)
.now(now);
if (previousModel != null)
deployStateBuilder.previousModel(previousModel);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
index 895aa4f6a36..87684aca174 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
@@ -27,7 +27,9 @@ public class ClusterSizeReductionValidatorTest {
fail("Expected exception due to cluster size reduction");
}
catch (IllegalArgumentException expected) {
- assertEquals("cluster-size-reduction: Size reduction in 'default' is too large. Current size: 30, new size: 14. New size must be at least 50% of the current size. " +
+ assertEquals("cluster-size-reduction: Size reduction in 'default' is too large: " +
+ "New min size must be at least 50% of the current min size. " +
+ "Current size: 30, new size: 14. " +
ValidationOverrides.toAllowMessage(ValidationId.clusterSizeReduction),
Exceptions.toMessageString(expected));
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java
index bb209e58e24..81b7f870a10 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java
@@ -201,7 +201,7 @@ public class ConfigValueChangeValidatorTest {
" </engine>\n" +
" </content>\n" +
"</services>",
- createSearchDefinitions(docTypes)
+ createSchemas(docTypes)
).create();
}
@@ -213,7 +213,7 @@ public class ConfigValueChangeValidatorTest {
"</documents>";
}
- private static List<String> createSearchDefinitions(List<String> docTypes) {
+ private static List<String> createSchemas(List<String> docTypes) {
return docTypes.stream()
.map(type -> "search " + type + " { document " + type + " { } }")
.collect(Collectors.toList());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
index 43ad1bc0e8a..80127ac6854 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
@@ -7,7 +7,7 @@ import com.yahoo.vespa.model.VespaModel;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder;
import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
import org.junit.Test;
import java.time.Instant;
@@ -41,7 +41,7 @@ public class IndexedSearchClusterChangeValidatorTest {
public static VespaModel newOneDocModel(String sdContent) {
return new ApplicationPackageBuilder().
addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")).
- addSearchDefinition(new SearchDefinitionBuilder().
+ addSchemas(new SchemaBuilder().
name("d1").content(sdContent).build()).buildCreator().create();
}
@@ -52,9 +52,9 @@ public class IndexedSearchClusterChangeValidatorTest {
public static VespaModel newTwoDocModel(String d1Content, String d2Content) {
return new ApplicationPackageBuilder().
addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")).
- addSearchDefinition(new SearchDefinitionBuilder().
+ addSchemas(new SchemaBuilder().
name("d1").content(d1Content).build()).
- addSearchDefinition(new SearchDefinitionBuilder().
+ addSchemas(new SchemaBuilder().
name("d2").content(d2Content).build()).
buildCreator().create();
}
@@ -67,9 +67,9 @@ public class IndexedSearchClusterChangeValidatorTest {
return new ApplicationPackageBuilder().
addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")).
addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")).
- addSearchDefinition(new SearchDefinitionBuilder().
+ addSchemas(new SchemaBuilder().
name("d1").content(d1Content).build()).
- addSearchDefinition(new SearchDefinitionBuilder().
+ addSchemas(new SchemaBuilder().
name("d2").content(d2Content).build()).
buildCreator().create();
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
index 1322a9061ed..9a363789798 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
@@ -30,7 +30,7 @@ public class ResourcesReductionValidatorTest {
fail("Expected exception due to resources reduction");
} catch (IllegalArgumentException expected) {
assertEquals("resources-reduction: Resource reduction in 'default' is too large. " +
- "Current memory GB: 64.00, new: 16.00. New resources must be at least 50% of the current resources. " +
+ "Current memory GB: 64.00, new: 16.00. New min resources must be at least 50% of the current min resources. " +
ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction),
Exceptions.toMessageString(expected));
}
@@ -45,7 +45,7 @@ public class ResourcesReductionValidatorTest {
} catch (IllegalArgumentException expected) {
assertEquals("resources-reduction: Resource reduction in 'default' is too large. " +
"Current vCPU: 8.00, new: 3.00. Current memory GB: 64.00, new: 16.00. Current disk GB: 800.00, new: 200.00. " +
- "New resources must be at least 50% of the current resources. " +
+ "New min resources must be at least 50% of the current min resources. " +
ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction),
Exceptions.toMessageString(expected));
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java
index c5fee3efa99..8edbc964bfb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java
@@ -8,7 +8,7 @@ import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder;
import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
import com.yahoo.vespa.model.content.utils.DocType;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
import org.junit.Test;
import java.time.Instant;
@@ -40,7 +40,7 @@ public class StreamingSearchClusterChangeValidatorTest {
public static VespaModel createOneDocModel(String sdContent) {
return new ApplicationPackageBuilder()
.addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"))))
- .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(sdContent).build())
+ .addSchemas(new SchemaBuilder().name("d1").content(sdContent).build())
.buildCreator().create();
}
@@ -51,8 +51,8 @@ public class StreamingSearchClusterChangeValidatorTest {
public static VespaModel createTwoDocModel(String d1Content, String d2Content) {
return new ApplicationPackageBuilder()
.addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"), DocType.streaming("d2"))))
- .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build())
- .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build())
+ .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build())
+ .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build())
.buildCreator().create();
}
@@ -64,8 +64,8 @@ public class StreamingSearchClusterChangeValidatorTest {
return new ApplicationPackageBuilder()
.addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"))))
.addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList(DocType.streaming("d2"))))
- .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build())
- .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build())
+ .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build())
+ .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build())
.buildCreator().create();
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
index 3827da08679..20ff9afd530 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
@@ -6,7 +6,7 @@ import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeActi
import com.yahoo.vespa.model.content.cluster.ContentCluster;
import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
import com.yahoo.vespa.model.search.DocumentDatabase;
import java.util.Arrays;
@@ -34,7 +34,7 @@ public abstract class ContentClusterFixture {
private static ContentCluster createCluster(String sdContent) throws Exception {
return new ContentClusterBuilder().build(
ContentClusterUtils.createMockRoot(
- Arrays.asList(new SearchDefinitionBuilder().content(sdContent).build())));
+ Arrays.asList(new SchemaBuilder().content(sdContent).build())));
}
protected DocumentDatabase currentDb() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
index 4064e53dfb7..3dfcef70aba 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
@@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest {
headerfields,
new StructDataType("bodyfields"),
new FieldSets(),
+ Collections.emptySet(),
Collections.emptySet());
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java
index c8f1224693b..d1a78a548d9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java
@@ -1,172 +1,21 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.application.validation.first;
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.model.NullConfigModelRegistry;
-import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.config.model.deploy.TestProperties;
-import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.model.VespaModel;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-
-import static com.yahoo.config.model.test.TestUtil.joinLines;
-import static com.yahoo.config.provision.Environment.prod;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.yahoo.vespa.model.application.validation.AccessControlValidatorTestBase;
+import org.junit.Before;
/**
* @author gjoranv
*/
-public class AccessControlOnFirstDeploymentValidatorTest {
-
- @Rule
- public final ExpectedException exceptionRule = ExpectedException.none();
-
- private static String servicesXml(boolean addHandler, boolean writeProtection) {
- return joinLines("<services version='1.0'>",
- " <container id='default' version='1.0'>",
- addHandler ? httpHandlerXml : "",
- " <http>",
- " <filtering>",
- " <access-control domain='foo' write='" + writeProtection + "' />",
- " </filtering>",
- " </http>",
- " </container>",
- "</services>");
- }
-
- private static final String httpHandlerXml =
- joinLines(" <handler id='foo'>",
- " <binding>http://foo/bar</binding>",
- " </handler>");
-
- @Test
- public void cluster_with_write_protection_passes_validation() throws IOException, SAXException{
- DeployState deployState = deployState(servicesXml(true, true));
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void cluster_with_no_handlers_passes_validation_without_write_protection() throws IOException, SAXException{
- DeployState deployState = deployState(servicesXml(false, false));
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void cluster_without_custom_components_passes_validation_without_write_protection() throws IOException, SAXException{
- String servicesXml = joinLines("<services version='1.0'>",
- " <container id='default' version='1.0' />",
- "</services>");
- DeployState deployState = deployState(servicesXml);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void cluster_with_handler_fails_validation_without_write_protection() throws IOException, SAXException{
- exceptionRule.expect(IllegalArgumentException.class);
- exceptionRule.expectMessage(
- "Access-control must be enabled for write operations to container clusters in production zones: [default]");
-
- DeployState deployState = deployState(servicesXml(true, false));
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void no_http_element_has_same_effect_as_no_write_protection() throws IOException, SAXException{
- exceptionRule.expect(IllegalArgumentException.class);
- exceptionRule.expectMessage(
- "Access-control must be enabled for write operations to container clusters in production zones: [default]");
-
- String servicesXml = joinLines("<services version='1.0'>",
- " <container id='default' version='1.0'>",
- httpHandlerXml,
- " </container>",
- "</services>");
- DeployState deployState = deployState(servicesXml);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void cluster_with_mbus_handler_passes_validation_without_write_protection() throws IOException, SAXException{
- String servicesXml = joinLines("<services version='1.0'>",
- " <container id='default' version='1.0'>",
- " <handler id='foo'>",
- " <binding>mbus://*/foo</binding>",
- " </handler>",
- " </container>",
- "</services>");
- DeployState deployState = deployState(servicesXml);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void write_protection_is_not_required_for_non_default_application_type() throws IOException, SAXException{
- String servicesXml = joinLines("<services version='1.0' application-type='hosted-infrastructure'>",
- " <container id='default' version='1.0'>",
- httpHandlerXml,
- " </container>",
- "</services>");
- DeployState deployState = deployState(servicesXml);
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- @Test
- public void write_protection_is_not_required_with_validation_override() throws IOException, SAXException{
- DeployState deployState = deployState(servicesXml(true, false),
- "<validation-overrides><allow until='2000-01-30'>access-control</allow></validation-overrides>",
- LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant());
- VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
- new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
- }
-
- private static DeployState deployState(String servicesXml) {
- return deployState(servicesXml, "<validation-overrides></validation-overrides>", Instant.now());
- }
-
- private static DeployState deployState(String servicesXml, String validationOverrides, Instant now) {
- ApplicationPackage app = new MockApplicationPackage.Builder()
- .withServices(servicesXml)
- .withValidationOverrides(validationOverrides)
- .build();
-
- DeployState.Builder builder = new DeployState.Builder()
- .applicationPackage(app)
- .zone(new Zone(Environment.prod, RegionName.from("foo")) )
- .properties(new TestProperties().setHostedVespa(true))
- .now(now);
- final DeployState deployState = builder.build();
-
- assertTrue("Test must emulate a hosted deployment.", deployState.isHosted());
- assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment());
+public class AccessControlOnFirstDeploymentValidatorTest extends AccessControlValidatorTestBase {
- return deployState;
+ @Before
+ public void setup() {
+ validator = new AccessControlOnFirstDeploymentValidator();
+ zone = new Zone(Environment.prod, RegionName.from("foo"));
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index 31a2a6496d7..b4fe4175707 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -236,8 +236,8 @@ public class ContentBuilderTest extends DomBuilderTest {
}
@Test
- public void requireThatContentStreamingHandlesMultipleSearchDefinitions() {
- final String musicClusterId = "music-cluster-id";
+ public void requireThatContentStreamingHandlesMultipleSchemas() {
+ String musicClusterId = "music-cluster-id";
ContentCluster cluster = createContentWithBooksToo(
"<content version='1.0' id='" + musicClusterId + "'>" +
@@ -825,8 +825,8 @@ public class ContentBuilderTest extends DomBuilderTest {
VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder()
.withHosts(getHosts())
.withServices(combined)
- .withSearchDefinitions(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION,
- MockApplicationPackage.BOOK_SEARCHDEFINITION))
+ .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION,
+ MockApplicationPackage.BOOK_SEARCHDEFINITION))
.build())
.create();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
index 060fff5bf66..73dd1ca3f3b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.model.builder.xml.dom;
import com.yahoo.collections.CollectionUtil;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
-import com.yahoo.vespa.model.content.DispatchTuning;
import com.yahoo.vespa.model.search.Tuning;
import org.junit.Test;
import org.w3c.dom.Element;
@@ -228,6 +227,8 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest {
assertEquals(cfg.summary().log().chunk().maxbytes(), 256);
assertEquals(cfg.summary().log().chunk().compression().type(), ProtonConfig.Summary.Log.Chunk.Compression.Type.LZ4);
assertEquals(cfg.summary().log().chunk().compression().level(), 5);
+ assertEquals(cfg.summary().log().compact().compression().type(), ProtonConfig.Summary.Log.Compact.Compression.Type.LZ4);
+ assertEquals(cfg.summary().log().compact().compression().level(), 5);
}
@Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index d1c5e344c24..33cf0635349 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -97,6 +97,7 @@ public class ContainerClusterTest {
cluster.getConfig(qsB);
QrStartConfig qsC= new QrStartConfig(qsB);
assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory());
+ assertEquals(0, qsC.jvm().compressedClassSpaceSize());
}
@Test
@@ -156,6 +157,7 @@ public class ContainerClusterTest {
QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder);
assertEquals(32, qrStartConfig.jvm().minHeapsize());
assertEquals(512, qrStartConfig.jvm().heapsize());
+ assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
@@ -200,6 +202,34 @@ public class ContainerClusterTest {
}
@Test
+ public void requireThatSoftStartSecondsCanBeControlledByProperties() {
+ DeployState state = new DeployState.Builder().properties(new TestProperties().setSoftStartSeconds(300.0))
+ .build();
+ MockRoot root = new MockRoot("foo", state);
+ ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ addContainer(root.deployLogger(), cluster, "c1", "host-c1");
+
+ ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
+ cluster.getConfig(tpBuilder);
+ ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder);
+ assertEquals(500, threadpoolConfig.maxthreads());
+ assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0);
+ }
+
+ @Test
+ public void requireThatDefaultThreadPoolConfigIsSane() {
+ MockRoot root = new MockRoot("foo");
+ ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ addContainer(root.deployLogger(), cluster, "c1", "host-c1");
+
+ ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
+ cluster.getConfig(tpBuilder);
+ ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder);
+ assertEquals(500, threadpoolConfig.maxthreads());
+ assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0);
+ }
+
+ @Test
public void requireThatRoutingProviderIsDisabledForNonHosted() {
DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(false)).build();
MockRoot root = new MockRoot("foo", state);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java
index 7cd26b2f1e6..28e23ce3222 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java
@@ -3,8 +3,13 @@ package com.yahoo.vespa.model.container.xml;
import com.google.common.collect.ImmutableSet;
import com.yahoo.collections.CollectionUtil;
+import com.yahoo.component.ComponentId;
import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.container.jdisc.state.StateHandler;
+import com.yahoo.vespa.model.container.ApplicationContainer;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.http.AccessControl;
import com.yahoo.vespa.model.container.http.Http;
@@ -16,14 +21,19 @@ import org.w3c.dom.Element;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static com.yahoo.config.model.test.TestUtil.joinLines;
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
/**
@@ -64,10 +74,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase {
Element clusterElem = DomBuilderTest.parse(
" <http>",
" <filtering>",
- " <access-control domain='my-domain'>",
- " <application>my-app</application>",
- " <vespa-domain>custom-vespa-domain</vespa-domain>",
- " </access-control>",
+ " <access-control domain='my-domain'/>",
" </filtering>",
" </http>");
@@ -76,8 +83,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase {
AccessControl accessControl = http.getAccessControl().get();
assertEquals("Wrong domain.", "my-domain", accessControl.domain);
- assertEquals("Wrong application.", "my-app", accessControl.applicationId);
- assertEquals("Wrong vespa-domain.", "custom-vespa-domain", accessControl.vespaDomain.get());
}
@Test
@@ -234,6 +239,58 @@ public class AccessControlTest extends ContainerModelBuilderTestBase {
}
+
+ @Test
+ public void access_control_is_implicitly_added_for_hosted_apps() {
+ Element clusterElem = DomBuilderTest.parse(
+ "<container version='1.0'>",
+ nodesXml,
+ "</container>" );
+ AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain");
+ DeployState state = new DeployState.Builder().properties(
+ new TestProperties()
+ .setAthenzDomain(tenantDomain)
+ .setHostedVespa(true))
+ .build();
+ createModel(root, state, null, clusterElem);
+ Optional<AccessControl> maybeAccessControl =
+ ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl();
+ assertThat(maybeAccessControl.isPresent(), is(true));
+ AccessControl accessControl = maybeAccessControl.get();
+ assertThat(accessControl.writeEnabled, is(false));
+ assertThat(accessControl.readEnabled, is(false));
+ assertThat(accessControl.domain, equalTo(tenantDomain.value()));
+ }
+
+ @Test
+ public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() {
+ Element clusterElem = DomBuilderTest.parse(
+ "<container version='1.0'>",
+ " <http>",
+ " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />",
+ " <filtering>",
+ " <filter id='outer' />",
+ " <request-chain id='myChain'>",
+ " <filter id='inner' />",
+ " </request-chain>",
+ " </filtering>",
+ " </http>",
+ nodesXml,
+ "</container>" );
+ AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain");
+ DeployState state = new DeployState.Builder().properties(
+ new TestProperties()
+ .setAthenzDomain(tenantDomain)
+ .setHostedVespa(true))
+ .build();
+ createModel(root, state, null, clusterElem);
+ Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp();
+ assertThat(http.getAccessControl().isPresent(), is(true));
+ assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true));
+ assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true));
+ }
+
+
private String httpWithExcludedBinding(String excludedBinding) {
return joinLines(
" <http>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
index 4ea10c9a1c1..00ab175f496 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
@@ -18,30 +18,22 @@ import static org.junit.Assert.assertEquals;
/**
* @author gjoranv
- * @since 5.5
*/
public class AccessLogTest extends ContainerModelBuilderTestBase {
@Test
- public void default_access_log_is_only_added_when_search_is_present() {
+ public void default_access_log_is_added_by_default() {
Element cluster1Elem = DomBuilderTest.parse(
"<container id='cluster1' version='1.0'>",
- "<search />",
- nodesXml,
- "</container>");
- Element cluster2Elem = DomBuilderTest.parse(
- "<container id='cluster2' version='1.0'>",
" <nodes>",
" <node hostalias='mockhost' baseport='1234' />",
" </nodes>",
"</container>" );
- createModel(root, cluster1Elem, cluster2Elem);
+ createModel(root, cluster1Elem);
assertNotNull(getJsonAccessLog("cluster1"));
- assertNull( getJsonAccessLog("cluster2"));
assertNull(getVespaAccessLog("cluster1"));
- assertNull(getVespaAccessLog("cluster2"));
}
@Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index 1bbc4ea2684..dcd1c46e52f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -25,6 +25,7 @@ import com.yahoo.container.QrConfig;
import com.yahoo.container.core.ChainsConfig;
import com.yahoo.container.core.VipStatusConfig;
import com.yahoo.container.handler.VipStatusHandler;
+import com.yahoo.container.handler.metrics.MetricsV2Handler;
import com.yahoo.container.handler.observability.ApplicationStatusHandler;
import com.yahoo.container.jdisc.JdiscBindingsConfig;
import com.yahoo.container.servlet.ServletConfigConfig;
@@ -39,7 +40,6 @@ import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.model.AbstractService;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ApplicationContainer;
-import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.SecretStore;
import com.yahoo.vespa.model.container.component.Component;
@@ -47,7 +47,6 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory;
import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
import org.hamcrest.Matchers;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -66,12 +65,17 @@ import java.util.stream.Collectors;
import static com.yahoo.config.model.test.TestUtil.joinLines;
import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern;
import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING;
+import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.isEmptyString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -149,7 +153,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
}
@Test
- @Ignore // TODO: Enable when turning the port check on
public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception {
try {
String servicesXml =
@@ -224,26 +227,49 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
assertThat(defaultRootHandler.serverBindings(), contains("http://*/"));
JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName());
- assertThat(applicationStatusHandler.serverBindings(),
- contains("http://*/ApplicationStatus"));
+ assertThat(applicationStatusHandler.serverBindings(), contains("http://*/ApplicationStatus"));
JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName());
- assertThat(fileRequestHandler.serverBindings(),
- contains("http://*/status.html"));
+ assertThat(fileRequestHandler.serverBindings(), contains("http://*/status.html"));
+
+ JdiscBindingsConfig.Handlers metricsV2Handler = config.handlers(MetricsV2Handler.class.getName());
+ assertThat(metricsV2Handler.serverBindings(), contains("http://*/metrics/v2", "http://*/metrics/v2/*"));
}
@Test
- public void default_root_handler_is_disabled_when_user_adds_a_handler_with_same_binding() {
+ public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() {
Element clusterElem = DomBuilderTest.parse(
"<container id='default' version='1.0'>" +
" <handler id='userRootHandler'>" +
- " <binding>" + ContainerCluster.ROOT_HANDLER_BINDING + "</binding>" +
+ " <binding>" + ROOT_HANDLER_BINDING + "</binding>" +
" </handler>" +
"</container>");
createModel(root, clusterElem);
+ // The handler is still set up.
ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName());
- assertThat(userRootHandler, nullValue());
+ assertThat(userRootHandler, notNullValue());
+
+ // .. but it has no bindings
+ var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default");
+ assertThat(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName()), is(nullValue()));
+ }
+
+ @Test
+ public void reserved_binding_cannot_be_stolen_by_user_configured_handler() {
+ Element clusterElem = DomBuilderTest.parse(
+ "<container id='default' version='1.0'>" +
+ " <handler id='userHandler'>" +
+ " <binding>" + STATE_HANDLER_BINDING_1 + "</binding>" +
+ " </handler>" +
+ "</container>");
+ try {
+ createModel(root, clusterElem);
+ fail("Expected exception when stealing a reserved binding.");
+ } catch (IllegalArgumentException e) {
+ assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " +
+ "and cannot be used by handler: userHandler"));
+ }
}
@Test
@@ -598,7 +624,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
.setMultitenant(true)
.setHostedVespa(true))
.build());
- assertEquals(1, model.hostSystem().getHosts().size());
+ assertEquals(2, model.hostSystem().getHosts().size());
}
@Test(expected = IllegalArgumentException.class)
@@ -783,7 +809,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
// Verify that there are two connectors
- List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().getConnectorFactories();
+ List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories();
assertEquals(2, connectorFactories.size());
List<Integer> ports = connectorFactories.stream()
.map(ConnectorFactory::getListenPort)
@@ -801,6 +827,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
assertEquals("CERT", connectorConfig.ssl().certificate());
assertEquals("KEY", connectorConfig.ssl().privateKey());
assertEquals(4443, connectorConfig.listenPort());
+
+ assertThat("Connector must use Athenz truststore in a non-public system.",
+ connectorConfig.ssl().caCertificateFile(), equalTo("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"));
+ assertThat(connectorConfig.ssl().caCertificate(), isEmptyString());
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
index 9ecd33f4273..eda90b03147 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
@@ -28,7 +28,11 @@ import java.util.Map;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertThat;
+
/**
* @author einarmr
@@ -204,24 +208,25 @@ public class DocprocBuilderTest extends DomBuilderTest {
@Test
public void testBundlesConfig() {
- assertThat(bundlesConfig.bundle().size(), is(0));
+ assertTrue(bundlesConfig.bundle().isEmpty());
}
@Test
public void testSchemaMappingConfig() {
- assertThat(schemamappingConfig.fieldmapping().size(), is(0));
+ assertTrue(schemamappingConfig.fieldmapping().isEmpty());
}
@Test
public void testQrStartConfig() {
QrStartConfig.Jvm jvm = qrStartConfig.jvm();
- assertThat(jvm.server(), is(true));
- assertThat(jvm.verbosegc(), is(true));
- assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15"));
- assertThat(jvm.minHeapsize(), is(1536));
- assertThat(jvm.heapsize(), is(1536));
- assertThat(jvm.stacksize(), is(512));
- assertThat(qrStartConfig.ulimitv(), is(""));
+ assertTrue(jvm.server());
+ assertTrue(jvm.verbosegc());
+ assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", jvm.gcopts());
+ assertEquals(1536, jvm.minHeapsize());
+ assertEquals(1536, jvm.heapsize());
+ assertEquals(512, jvm.stacksize());
+ assertTrue(qrStartConfig.ulimitv().isEmpty());
+ assertEquals(0, jvm.compressedClassSpaceSize());
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
index 68f507c810d..0f9bd506310 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
@@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
" <client-authentication>need</client-authentication>",
" </ssl>",
" </server>",
+ " <server port='9003' id='with-ciphers-and-protocols'>",
+ " <ssl>",
+ " <private-key-file>/foo/key</private-key-file>",
+ " <certificate-file>/foo/cert</certificate-file>",
+ " <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>",
+ " <protocols>TLSv1.3</protocols>",
+ " </ssl>",
+ " </server>",
" </http>",
nodesXml,
"",
@@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo("")));
assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH)));
+ ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols");
+ assertTrue(withCiphersAndProtocols.ssl().enabled());
+ assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key")));
+ assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert")));
+ assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384"))));
+ assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3"))));
+
ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default");
List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class);
connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
index 1f0b0188681..47f37a75fd0 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
@@ -16,8 +16,11 @@ import org.w3c.dom.Element;
import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER;
import static com.yahoo.test.Matchers.hasItemWithMethod;
+import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING;
+import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.*;
@@ -54,8 +57,6 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
if (guiHandler == null) fail();
}
-
-
@Test
public void search_handler_bindings_can_be_overridden() {
Element clusterElem = DomBuilderTest.parse(
@@ -91,6 +92,25 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
assertThat(discBindingsConfig, not(containsString("/search/*")));
}
+ @Test
+ public void search_handler_binding_can_be_stolen_by_user_configured_handler() {
+ var myHandler = "replaces_search_handler";
+ Element clusterElem = DomBuilderTest.parse(
+ "<container id='default' version='1.0'>",
+ " <search />",
+ " <handler id='" + myHandler + "'>",
+ " <binding>" + SEARCH_HANDLER_BINDING + "</binding>",
+ " </handler>",
+ nodesXml,
+ "</container>");
+
+ createModel(root, clusterElem);
+
+ var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default");
+ assertThat(discBindingsConfig.handlers(myHandler).serverBindings(0), is(SEARCH_HANDLER_BINDING));
+ assertThat(discBindingsConfig.handlers(SEARCH_HANDLER_CLASS), is(nullValue()));
+ }
+
// TODO: remove test when all containers are named 'container'
@Test
public void cluster_with_only_search_gets_qrserver_as_service_name() {
@@ -190,7 +210,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
private VespaModel getVespaModelWithMusic(String hosts, String services) {
- return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create();
+ return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSchemas("music")).create();
}
private String hostsXml() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 305bcc1d7d5..4d5df7c1965 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -26,7 +26,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster;
import com.yahoo.vespa.model.content.engines.ProtonEngine;
import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
import com.yahoo.vespa.model.routing.DocumentProtocol;
import com.yahoo.vespa.model.routing.Routing;
import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
@@ -218,7 +218,7 @@ public class ContentClusterTest extends ContentBaseTest {
"\n" +
"</services>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
VespaModel model = (new VespaModelCreatorWithMockPkg(null, xml, sds)).create();
assertEquals(2, model.getContentClusters().get("bar").getDocumentDefinitions().size());
ContainerCluster cluster = model.getAdmin().getClusterControllers();
@@ -259,7 +259,7 @@ public class ContentClusterTest extends ContentBaseTest {
" </services>";
DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties);
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1");
return (new VespaModelCreatorWithMockPkg(null, services, sds)).create(deployStateBuilder);
}
@Test
@@ -301,7 +301,7 @@ public class ContentClusterTest extends ContentBaseTest {
"\n" +
"</services>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
assertTrue(model.getContentClusters().get("bar").getPersistence() instanceof ProtonEngine.Factory);
@@ -340,7 +340,7 @@ public class ContentClusterTest extends ContentBaseTest {
" </content>\n" +
"</services>\n";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
try{
new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
assertTrue("Deploying without redundancy should fail", false);
@@ -697,7 +697,7 @@ public class ContentClusterTest extends ContentBaseTest {
"</services>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
{
@@ -818,7 +818,7 @@ public class ContentClusterTest extends ContentBaseTest {
" </group>" +
"</content>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("true");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("true");
new VespaModelCreatorWithMockPkg(null, xml, sds).create();
}
@@ -865,7 +865,7 @@ public class ContentClusterTest extends ContentBaseTest {
" </group>" +
"</content>" +
"</services>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("bunnies", "hares", "rabbits");
return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
}
@@ -912,8 +912,8 @@ public class ContentClusterTest extends ContentBaseTest {
DeployState.Builder deployStateBuilder = new DeployState.Builder()
.zone(zone)
.properties(new TestProperties().setHostedVespa(true));
- List<String> searchDefinitions = SearchDefinitionBuilder.createSearchDefinitions("test");
- MockRoot root = ContentClusterUtils.createMockRoot(searchDefinitions, deployStateBuilder);
+ List<String> schemas = SchemaBuilder.createSchemas("test");
+ MockRoot root = ContentClusterUtils.createMockRoot(schemas, deployStateBuilder);
ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root);
root.freezeModelTopology();
cluster.validate();
@@ -933,6 +933,17 @@ public class ContentClusterTest extends ContentBaseTest {
assertEquals(distributionBits, storDistributormanagerConfig.minsplitcount());
}
+ private void verifyTopKProbabilityPropertiesControl(double topKProbability) {
+ VespaModel model = createEnd2EndOneNode(new TestProperties().setTopKProbability(topKProbability));
+
+ ContentCluster cc = model.getContentClusters().get("storage");
+ DispatchConfig.Builder builder = new DispatchConfig.Builder();
+ cc.getSearch().getConfig(builder);
+
+ DispatchConfig cfg = new DispatchConfig(builder);
+ assertEquals(topKProbability, cfg.topKProbability(), 0.0);
+ }
+
private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) {
VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch));
@@ -946,7 +957,6 @@ public class ContentClusterTest extends ContentBaseTest {
} else {
assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, cfg.distributionPolicy());
}
-
}
@Test
@@ -955,5 +965,12 @@ public class ContentClusterTest extends ContentBaseTest {
verifyRoundRobinPropertiesControl(true);
}
+ @Test
+ public void default_topKprobability_controlled_by_properties() {
+ verifyTopKProbabilityPropertiesControl(1.0);
+ verifyTopKProbabilityPropertiesControl(0.999);
+ verifyTopKProbabilityPropertiesControl(0.77);
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
index 98fa179b219..3415044b088 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
@@ -8,7 +8,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.vespa.model.content.cluster.ContentCluster;
import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
import com.yahoo.vespa.model.content.utils.DocType;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
import org.junit.Test;
import java.util.ArrayList;
@@ -17,7 +17,7 @@ import java.util.List;
import static com.yahoo.config.model.test.TestUtil.joinLines;
import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster;
-import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions;
+import static com.yahoo.vespa.model.content.utils.SchemaBuilder.createSchemas;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -38,7 +38,7 @@ public class ContentSearchClusterTest {
private static ContentCluster createClusterWithTwoDocumentType() throws Exception {
return createCluster(new ContentClusterBuilder().docTypes("foo", "bar").getXml(),
- createSearchDefinitions("foo", "bar"));
+ createSchemas("foo", "bar"));
}
private static ContentCluster createClusterWithGlobalType() throws Exception {
@@ -55,7 +55,7 @@ public class ContentSearchClusterTest {
"<node distribution-key='1' hostalias='mockhost'/>",
"</group>"));
String clusterXml = builder.getXml();
- return createCluster(clusterXml, createSearchDefinitions(docTypes));
+ return createCluster(clusterXml, createSchemas(docTypes));
}
private static ContentClusterBuilder createClusterBuilderWithGlobalType() {
@@ -127,18 +127,19 @@ public class ContentSearchClusterTest {
}
private static ContentCluster createClusterWithThreeDocumentTypes() throws Exception {
- List<String> searchDefinitions = new ArrayList<>();
- searchDefinitions.add(new SearchDefinitionBuilder().name("a")
- .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }",
- "field ref_to_c type reference<c> { indexing: attribute }")).build());
- searchDefinitions.add(new SearchDefinitionBuilder().name("b")
- .content("field ref_to_c type reference<c> { indexing: attribute }").build());
- searchDefinitions.add(new SearchDefinitionBuilder().name("c").build());
- return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(
- DocType.index("a"),
- DocType.indexGlobal("b"),
- DocType.indexGlobal("c"))).getXml(),
- searchDefinitions);
+ List<String> schemas = new ArrayList<>();
+ schemas.add(new SchemaBuilder().name("a")
+ .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }",
+ "field ref_to_c type reference<c> { indexing: attribute }"))
+ .build());
+ schemas.add(new SchemaBuilder().name("b")
+ .content("field ref_to_c type reference<c> { indexing: attribute }")
+ .build());
+ schemas.add(new SchemaBuilder().name("c").build());
+ return createCluster(new ContentClusterBuilder().docTypes(List.of(DocType.index("a"),
+ DocType.indexGlobal("b"),
+ DocType.indexGlobal("c"))).getXml(),
+ schemas);
}
private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java
index f708d7673e2..8a46aaaa230 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java
@@ -19,11 +19,13 @@ public class DispatchTuningTest {
.setDispatchPolicy("round-robin")
.setMinGroupCoverage(7.5)
.setMinActiveDocsCoverage(12.5)
+ .setTopKProbability(18.3)
.build();
assertEquals(69, dispatch.getMaxHitsPerPartition().intValue());
assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0);
assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0);
assertTrue(DispatchTuning.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy());
+ assertEquals(18.3, dispatch.getTopkProbability(), 0.0);
}
@Test
public void requireThatRandomDispatchWork() {
@@ -52,6 +54,7 @@ public class DispatchTuningTest {
assertNull(dispatch.getDispatchPolicy());
assertNull(dispatch.getMinActiveDocsCoverage());
assertNull(dispatch.getMinGroupCoverage());
+ assertNull(dispatch.getTopkProbability());
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
index f36ef6c3ba3..365dc74274d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
@@ -24,7 +24,7 @@ public class DistributorTest {
ContentCluster parseCluster(String xml) {
try {
- List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies");
+ List<String> searchDefs = ApplicationPackageUtils.generateSchemas("music", "movies", "bunnies");
MockRoot root = ContentClusterUtils.createMockRoot(searchDefs);
return ContentClusterUtils.createCluster(xml, root);
} catch (Exception e) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java
index 992edf6b1bb..51d0afc1f93 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java
@@ -50,7 +50,7 @@ public class GenericConfigTest {
@Before
public void getVespaModel() {
- model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSearchDefinitions("type1"))).create();
+ model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSchemas("type1"))).create();
}
@Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index ecf8f100288..504c3d9ba9c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -112,12 +112,12 @@ public class IndexedTest extends ContentBaseTest {
}
private VespaModelCreatorWithMockPkg getIndexedVespaModelCreator() {
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3");
return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(Arrays.asList("type1", "type2", "type3")), sds);
}
private VespaModel getStreamingVespaModel() {
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1");
return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(Arrays.asList("type1")), sds).create();
}
@@ -229,7 +229,7 @@ public class IndexedTest extends ContentBaseTest {
" </content>\n" +
" </services>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("docstorebench");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("docstorebench");
VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create();
ProtonConfig.Builder pb = new ProtonConfig.Builder();
model.getConfig(pb, "docstore/search/cluster.docstore/0");
@@ -252,7 +252,7 @@ public class IndexedTest extends ContentBaseTest {
" </content>" +
"</services>";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("index_me", "store_me");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("index_me", "store_me");
VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create();
ProtonConfig.Builder pb = new ProtonConfig.Builder();
model.getConfig(pb, "docstore/search/cluster.docstore/0");
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
index 55d070d7247..177b86c953e 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
@@ -164,7 +164,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
" </container>\n" +
"</services>\n";
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "title", "artist");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("music", "title", "artist");
VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(),
services, sds).create();
assertIndexing(model, new DocprocClusterSpec("dokprok"));
@@ -448,7 +448,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
}
private VespaModel getIndexedSearchVespaModel(String xml) {
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "album", "artist");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist");
return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 21c384dfc69..883d8b89765 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -112,11 +112,7 @@ public class StorageClusterTest {
"<cluster id=\"bees\">\n" +
" <documents/>" +
" <tuning>\n" +
- " <persistence-threads>\n" +
- " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" +
- " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" +
- " <thread count=\"1\"/>\n" +
- " </persistence-threads>\n" +
+ " <persistence-threads count=\"7\"/>\n" +
" </tuning>\n" +
" <group>" +
" <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -130,6 +126,44 @@ public class StorageClusterTest {
stc.getConfig(builder);
StorFilestorConfig config = new StorFilestorConfig(builder);
+ assertEquals(7, config.num_threads());
+ assertEquals(false, config.enable_multibit_split_optimalization());
+ }
+ {
+ assertEquals(1, stc.getChildren().size());
+ StorageNode sn = stc.getChildren().values().iterator().next();
+ StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
+ sn.getConfig(builder);
+ StorFilestorConfig config = new StorFilestorConfig(builder);
+ assertEquals(7, config.num_threads());
+ }
+ }
+
+ @Test
+ public void testPersistenceThreadsOld() throws Exception {
+
+ StorageCluster stc = parse(
+ "<cluster id=\"bees\">\n" +
+ " <documents/>" +
+ " <tuning>\n" +
+ " <persistence-threads>\n" +
+ " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" +
+ " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" +
+ " <thread count=\"1\"/>\n" +
+ " </persistence-threads>\n" +
+ " </tuning>\n" +
+ " <group>" +
+ " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
+ " </group>" +
+ "</cluster>",
+ new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build())
+ );
+
+ {
+ StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
+ stc.getConfig(builder);
+ StorFilestorConfig config = new StorFilestorConfig(builder);
+
assertEquals(4, config.num_threads());
assertEquals(false, config.enable_multibit_split_optimalization());
}
@@ -161,7 +195,7 @@ public class StorageClusterTest {
StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
stc.getConfig(builder);
StorFilestorConfig config = new StorFilestorConfig(builder);
- assertEquals(6, config.num_threads());
+ assertEquals(8, config.num_threads());
}
{
assertEquals(1, stc.getChildren().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
index c0ddd49069d..e099476ebb6 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
@@ -49,7 +49,7 @@ public class StorageContentTest extends ContentBaseTest {
}
private VespaModel getStorageVespaModel(String cluster1docs, String cluster2docs) {
- List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3");
+ List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3");
return new VespaModelCreatorWithMockPkg(getHosts(), createStorageVespaServices(cluster1docs, cluster2docs), sds).create();
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
index 62221c206fd..852844fe451 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
@@ -176,7 +176,7 @@ public class ClusterTest {
" </tuning>",
" </content>",
"</services>"))
- .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document"))
+ .withSchemas(ApplicationPackageUtils.generateSearchDefinition("my_document"))
.build();
List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class);
assertEquals(1, contents.size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java
index 7fa27f74d74..abfb03e41dd 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java
@@ -47,6 +47,7 @@ public class DomDispatchTuningBuilderTest {
assertNull(dispatch.getMinGroupCoverage());
assertNull(dispatch.getMinActiveDocsCoverage());
assertNull(dispatch.getDispatchPolicy());
+ assertNull(dispatch.getTopkProbability());
}
@Test
@@ -58,12 +59,14 @@ public class DomDispatchTuningBuilderTest {
" <max-hits-per-partition>69</max-hits-per-partition>" +
" <min-group-coverage>7.5</min-group-coverage>" +
" <min-active-docs-coverage>12.5</min-active-docs-coverage>" +
+ " <top-k-probability>0.999</top-k-probability>" +
" </dispatch>" +
" </tuning>" +
"</content>");
assertEquals(69, dispatch.getMaxHitsPerPartition().intValue());
assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0);
assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0);
+ assertEquals(0.999, dispatch.getTopkProbability().doubleValue(), 0.0);
}
@Test
public void requireThatTuningDispatchPolicyRoundRobin() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java
index 1b7f3e9b14d..5fc213fad1d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java
@@ -14,7 +14,7 @@ import java.util.List;
public class ApplicationPackageBuilder {
private List<ContentClusterBuilder> contentClusters = new ArrayList<>();
- private List<String> searchDefinitions = new ArrayList<>();
+ private List<String> schemas = new ArrayList<>();
public ApplicationPackageBuilder() {
}
@@ -24,13 +24,13 @@ public class ApplicationPackageBuilder {
return this;
}
- public ApplicationPackageBuilder addSearchDefinition(String searchDefinition) {
- searchDefinitions.add(searchDefinition);
+ public ApplicationPackageBuilder addSchemas(String schemas) {
+ this.schemas.add(schemas);
return this;
}
public VespaModelCreatorWithMockPkg buildCreator() {
- return new VespaModelCreatorWithMockPkg(null, getServices(), searchDefinitions);
+ return new VespaModelCreatorWithMockPkg(null, getServices(), schemas);
}
private String getServices() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
index 62d2bc51830..b1550d90f35 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
@@ -30,46 +30,46 @@ import java.util.Optional;
public class ContentClusterUtils {
public static MockRoot createMockRoot(String[] hosts) {
- return createMockRoot(hosts, SearchDefinitionBuilder.createSearchDefinitions("test"));
+ return createMockRoot(hosts, SchemaBuilder.createSchemas("test"));
}
- private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions) {
- return createMockRoot(provisioner, searchDefinitions, new DeployState.Builder());
+ private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas) {
+ return createMockRoot(provisioner, schemas, new DeployState.Builder());
}
- private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions, DeployState.Builder deployStateBuilder) {
- ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build();
+ private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas, DeployState.Builder deployStateBuilder) {
+ ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSchemas(schemas).build();
DeployState deployState = deployStateBuilder.applicationPackage(applicationPackage)
.modelHostProvisioner(provisioner)
.build();
return new MockRoot("", deployState);
}
- public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) {
- return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions);
+ public static MockRoot createMockRoot(String[] hosts, List<String> schemas) {
+ return createMockRoot(new InMemoryProvisioner(true, hosts), schemas);
}
- public static MockRoot createMockRoot(List<String> searchDefinitions) {
- return createMockRoot(new SingleNodeProvisioner(), searchDefinitions);
+ public static MockRoot createMockRoot(List<String> schemas) {
+ return createMockRoot(new SingleNodeProvisioner(), schemas);
}
- public static MockRoot createMockRoot(List<String> searchDefinitions, DeployState.Builder deployStateBuilder) {
- return createMockRoot(new SingleNodeProvisioner(), searchDefinitions, deployStateBuilder);
+ public static MockRoot createMockRoot(List<String> schemas, DeployState.Builder deployStateBuilder) {
+ return createMockRoot(new SingleNodeProvisioner(), schemas, deployStateBuilder);
}
public static ContentCluster createCluster(String clusterXml, MockRoot root) {
Document doc = XML.getDocument(clusterXml);
Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), false,
- new FileDistributionConfigProducer(root, new MockFileRegistry(), null),
+ new FileDistributionConfigProducer(root, new MockFileRegistry(), List.of(), false),
root.getDeployState().isHosted());
ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(),
- null,null, root, null);
+ null,null, root, null);
return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement());
}
- public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception {
- MockRoot root = createMockRoot(searchDefinitions);
+ public static ContentCluster createCluster(String clusterXml, List<String> schemas) throws Exception {
+ MockRoot root = createMockRoot(schemas);
ContentCluster cluster = createCluster(clusterXml, root);
root.freezeModelTopology();
cluster.validate();
@@ -77,7 +77,7 @@ public class ContentClusterUtils {
}
public static ContentCluster createCluster(String clusterXml) throws Exception {
- return createCluster(clusterXml, SearchDefinitionBuilder.createSearchDefinitions("test"));
+ return createCluster(clusterXml, SchemaBuilder.createSchemas("test"));
}
public static String createClusterXml(String groupXml, int redundancy, int searchableCopies) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
index c622bcfaec5..c18dac17064 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
@@ -12,20 +12,20 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
*
* @author geirst
*/
-public class SearchDefinitionBuilder {
+public class SchemaBuilder {
private String name = "test";
private String content = "";
- public SearchDefinitionBuilder() {
+ public SchemaBuilder() {
}
- public SearchDefinitionBuilder name(String name) {
+ public SchemaBuilder name(String name) {
this.name = name;
return this;
}
- public SearchDefinitionBuilder content(String content) {
+ public SchemaBuilder content(String content) {
this.content = content;
return this;
}
@@ -38,10 +38,10 @@ public class SearchDefinitionBuilder {
"}");
}
- public static List<String> createSearchDefinitions(String ... docTypes) {
+ public static List<String> createSchemas(String ... docTypes) {
return Arrays.asList(docTypes)
.stream()
- .map(type -> new SearchDefinitionBuilder().name(type).build())
+ .map(type -> new SchemaBuilder().name(type).build())
.collect(Collectors.toList());
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
index 131a5344116..55672c5aa16 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
@@ -10,31 +10,34 @@ import org.junit.Test;
import java.io.File;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
/**
* @author bratseth
*/
public class FileDistributorTestCase {
+
@Test
public void fileDistributor() {
MockHosts hosts = new MockHosts();
- FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), null);
+ FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), List.of(), false);
String file1 = "component/path1";
String file2 = "component/path2";
- FileReference ref1 = fileDistributor.sendFileToHosts(file1, Arrays.asList(hosts.host1, hosts.host2));
- FileReference ref2 = fileDistributor.sendFileToHosts(file2, Arrays.asList(hosts.host3));
+ FileReference ref1 = fileDistributor.sendFileToHost(file1, hosts.host1);
+ fileDistributor.sendFileToHost(file1, hosts.host2); // same file reference as above
+ FileReference ref2 = fileDistributor.sendFileToHost(file2, hosts.host3);
assertEquals(new HashSet<>(Arrays.asList(hosts.host1, hosts.host2, hosts.host3)),
fileDistributor.getTargetHosts());
- assertTrue( ref1 != null );
- assertTrue( ref2 != null );
+ assertNotNull(ref1);
+ assertNotNull(ref2);
MockFileDistribution dbHandler = new MockFileDistribution();
fileDistributor.sendDeployedFiles(dbHandler);
@@ -54,4 +57,5 @@ public class FileDistributorTestCase {
return null;
}
}
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
index ce36ecc4a1c..7a3b76db7f8 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.ml;
-import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
import com.google.common.collect.ImmutableList;
import com.yahoo.config.model.ApplicationPackageTester;
import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
@@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
import com.yahoo.searchdefinition.RankingConstant;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
+import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.serialization.TypedBinaryFormat;
@@ -35,6 +36,7 @@ public class ImportedModelTester {
private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
new OnnxImporter(),
+ new LightGBMImporter(),
new XGBoostImporter(),
new VespaImporter());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
index c5c475360a3..ced7243adf5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
@@ -45,7 +45,7 @@ public class MlModelsTest {
private void verify(VespaModel model) {
assertEquals("Global models are created (although not used directly here",
- 4, model.rankProfileList().getRankProfiles().size());
+ 5, model.rankProfileList().getRankProfiles().size());
RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder();
model.getSearchClusters().get(0).getConfig(builder);
@@ -71,8 +71,9 @@ public class MlModelsTest {
"rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" +
"rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" +
"rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" +
+ "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" +
"vespa.rank.firstphase: rankingExpression(firstphase)\n" +
- "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" +
+ "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" +
"vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n";
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
index 3d4ac7f2eeb..2d3ddc33afb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
@@ -96,9 +96,10 @@ public class ModelEvaluationTest {
cluster.getConfig(cb);
RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb);
- assertEquals(4, config.rankprofile().size());
+ assertEquals(5, config.rankprofile().size());
Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet());
assertTrue(modelNames.contains("xgboost_2_2"));
+ assertTrue(modelNames.contains("lightgbm_regression"));
assertTrue(modelNames.contains("mnist_saved"));
assertTrue(modelNames.contains("mnist_softmax"));
assertTrue(modelNames.contains("mnist_softmax_saved"));
@@ -112,13 +113,18 @@ public class ModelEvaluationTest {
ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null))
.importFrom(config, constantsConfig));
- assertEquals(4, evaluator.models().size());
+ assertEquals(5, evaluator.models().size());
Model xgboost = evaluator.models().get("xgboost_2_2");
assertNotNull(xgboost);
assertNotNull(xgboost.evaluatorOf());
assertNotNull(xgboost.evaluatorOf("xgboost_2_2"));
+ Model lightgbm = evaluator.models().get("lightgbm_regression");
+ assertNotNull(lightgbm);
+ assertNotNull(lightgbm.evaluatorOf());
+ assertNotNull(lightgbm.evaluatorOf("lightgbm_regression"));
+
Model tensorflow_mnist = evaluator.models().get("mnist_saved");
assertNotNull(tensorflow_mnist);
assertEquals(1, tensorflow_mnist.functions().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index a4b414ba0da..78c6d2eacc9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -69,6 +69,13 @@ public class NodeFlavorTuningTest {
}
@Test
+ public void require_that_num_search_threads_and_summary_threads_follow_cores() {
+ ProtonConfig cfg = configFromNumCoresSetting(4.5);
+ assertEquals(5, cfg.numsearcherthreads());
+ assertEquals(5, cfg.numsummarythreads());
+ }
+
+ @Test
public void require_that_fast_disk_is_reflected_in_proton_config() {
ProtonConfig cfg = configFromDiskSetting(true);
assertEquals(200, cfg.hwinfo().disk().writespeed(), delta);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index 177e741937d..97417f5a522 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -90,7 +90,7 @@ public class DocumentDatabaseTestCase {
private void assertSingleSD(String mode) {
final List<String> sds = Arrays.asList("type1");
VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
- ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+ ApplicationPackageUtils.generateSchemas(sds)).create();
IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
assertEquals(1, indexedSearchCluster.getDocumentDbs().size());
@@ -111,7 +111,7 @@ public class DocumentDatabaseTestCase {
sds.add(nameAndMode.getType());
}
return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning),
- ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+ ApplicationPackageUtils.generateSchemas(sds)).create();
}
@Test
@@ -211,10 +211,10 @@ public class DocumentDatabaseTestCase {
}
@Test
- public void requireThatWeCanHaveMultipleSearchDefinitions() {
- final List<String> sds = Arrays.asList("type1", "type2", "type3");
+ public void testMultipleSchemas() {
+ List<String> sds = List.of("type1", "type2", "type3");
VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
- ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+ ApplicationPackageUtils.generateSchemas(sds)).create();
IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
String type1Id = "test/search/cluster.test/type1";
@@ -264,7 +264,7 @@ public class DocumentDatabaseTestCase {
public void requireThatRelevantConfigIsAvailableForClusterSearcher() {
final List<String> sds = Arrays.asList("type1", "type2");
VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
- ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+ ApplicationPackageUtils.generateSchemas(sds)).create();
String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher";
{ // documentdb-info config
@@ -325,7 +325,7 @@ public class DocumentDatabaseTestCase {
private void assertDocumentDBConfigAvailableForStreaming(String mode) {
final List<String> sds = Arrays.asList("type");
VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
- ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+ ApplicationPackageUtils.generateSchemas(sds)).create();
DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
assertEquals(1, dcfg.documentdb().size());
@@ -343,7 +343,7 @@ public class DocumentDatabaseTestCase {
List<String> documentDBConfigIds,
Map<String, List<String>> expectedAttributesMap) {
VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
- ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+ ApplicationPackageUtils.generateSchemas(sds)).create();
ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
ProtonConfig proton = getProtonCfg(contentSearchCluster);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
index 1c4e005cb67..ea4b3db5ebb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
@@ -18,12 +18,15 @@ import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.search.AbstractSearchCluster;
-import com.yahoo.vespa.model.search.SearchCluster;
import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
import org.junit.Test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
/**
* Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and
@@ -124,7 +127,7 @@ public class SearchClusterTest {
" </content>" +
"</services>";
- VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create();
+ VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSchemas("music")).create();
ContainerCluster containerCluster1 = (ContainerCluster)model.getConfigProducer("j1").get();
assertFalse(containerCluster1.getSearch().getChains().localProviders().isEmpty());
@@ -160,23 +163,30 @@ public class SearchClusterTest {
AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex);
assertEquals("xbulk", searchCluster2.getClusterName());
- Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal"));
- assertNotNull(normalDispatcher);
- assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue());
- assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue());
- assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId());
- DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder();
- model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal");
- assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host());
-
- Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk"));
- assertNotNull(xbulkDispatcher);
- assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue());
- assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue());
- assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId());
- DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder();
- model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk");
- assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host());
+ verifyDispatch(model, containerCluster1, "normal", "node2host");
+ verifyDispatch(model, containerCluster1, "xbulk", "node0host");
+ }
+
+ private void verifyDispatch(VespaModel model, ContainerCluster containerCluster, String cluster, String host) {
+ Component<?,?> dispatcher = (Component<?, ?>)containerCluster.getComponentsMap().get(new ComponentId("dispatcher." + cluster));
+ assertNotNull(dispatcher);
+ assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue());
+ assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue());
+ assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId());
+ DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder();
+ model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId());
+ assertEquals(host, dispatchConfigBuilder.build().node(0).host());
+
+ assertTrue(dispatcher.getInjectedComponentIds().contains("rpcresourcepool." + cluster));
+
+ Component<?,?> rpcResourcePool = (Component<?, ?>)dispatcher.getChildren().get("rpcresourcepool." + cluster);
+ assertNotNull(rpcResourcePool);
+ assertEquals("rpcresourcepool." + cluster, rpcResourcePool.getComponentId().stringValue());
+ assertEquals("com.yahoo.search.dispatch.rpc.RpcResourcePool", rpcResourcePool.getClassId().stringValue());
+ assertEquals("j1/component/dispatcher." + cluster + "/rpcresourcepool." + cluster, rpcResourcePool.getConfigId());
+ dispatchConfigBuilder = new DispatchConfig.Builder();
+ model.getConfig(dispatchConfigBuilder, rpcResourcePool.getConfigId());
+ assertEquals(host, dispatchConfigBuilder.build().node(0).host());
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
index b6180ab78b9..cdfd9fab194 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
@@ -104,33 +104,41 @@ public class VespaModelTester {
/** Creates a model which uses 0 as start index */
public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) {
- return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, 0, retiredHostNames);
+ return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, retiredHostNames);
+ }
+
+ /** Creates a model which uses 0 as start index */
+ public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) {
+ return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, retiredHostNames);
}
/** Creates a model which uses 0 as start index */
public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
- return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, startIndexForClusters, retiredHostNames);
+ return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames);
}
/** Creates a model which uses 0 as start index */
public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) {
- return createModel(zone, services, failOnOutOfCapacity, 0, retiredHostNames);
+ return createModel(zone, services, failOnOutOfCapacity, false, 0, retiredHostNames);
}
/**
* Creates a model using the hosts already added to this
*
* @param services the services xml string
+ * @param useMaxResources false to use the minmal resources (when given a range), true to use max
* @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor
* is available or if we should just silently receive a smaller allocation
* @return the resulting model
*/
- public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
+ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources,
+ int startIndexForClusters, String ... retiredHostNames) {
VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1"));
ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg;
- HostProvisioner provisioner = hosted ?
- new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) :
+ HostProvisioner provisioner = hosted ?
+ new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources,
+ startIndexForClusters, retiredHostNames) :
new SingleNodeProvisioner();
TestProperties properties = new TestProperties()
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
index ee6fc60ba46..df62a3bff07 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
@@ -44,14 +44,14 @@ public class ApplicationPackageUtils {
}
public static List<String> generateSearchDefinition(String name) {
- return generateSearchDefinitions(name);
+ return generateSchemas(name);
}
- public static List<String> generateSearchDefinitions(String ... sdNames) {
- return generateSearchDefinitions(Arrays.asList(sdNames));
+ public static List<String> generateSchemas(String ... sdNames) {
+ return generateSchemas(Arrays.asList(sdNames));
}
- public static List<String> generateSearchDefinitions(List<String> sdNames) {
+ public static List<String> generateSchemas(List<String> sdNames) {
List<String> sds = new ArrayList<>();
int i = 0;
for (String sdName : sdNames) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java
index 814ec008285..70ce588bec1 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java
@@ -31,8 +31,8 @@ public class VespaModelCreatorWithMockPkg {
this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build());
}
- public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> searchDefinitions) {
- this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSearchDefinitions(searchDefinitions).build());
+ public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas) {
+ this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSchemas(schemas).build());
}
public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) {
diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml
index 07839239c81..71a07926240 100644
--- a/config-model/src/test/schema-test-files/services-hosted.xml
+++ b/config-model/src/test/schema-test-files/services-hosted.xml
@@ -7,7 +7,7 @@
</admin>
<container id="container1" version="1.0">
- <nodes count="5" required="true">
+ <nodes count="[5,7]" required="true">
<resources vcpu="1.2" memory="10Gb" disk="0.3 TB"/>
</nodes>
</container>
@@ -27,4 +27,11 @@
</nodes>
</content>
+ <content id="ml" version="1.0">
+ <redundancy>2</redundancy>
+ <nodes count="[10,20]" flavor="large" groups="[1,3]">
+ <resources vcpu="[3.0, 4]" memory="[32000.0Mb, 33Gb]" disk="[300 Gb, 1Tb]"/>
+ </nodes>
+ </content>
+
</services>
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index 2bbd98f72ac..e7ea2683e3f 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -15,15 +15,29 @@
<slobrok hostalias="rtc-1" />
</slobroks>
<metrics>
- <consumer id="my-consumer">
+
+ <consumer id="cloudwatch-hosted">
<metric-set id="my-set" />
<metric id="my-metric"/>
<metric id="my-metric2" display-name="my-metric3"/>
<metric display-name="my-metric4" id="my-metric4.avg"/>
+ <cloudwatch region="us-east1" namespace="my-namespace">
+ <credentials access-key-name="my-access-key" secret-key-name="my-secret-key" />
+ </cloudwatch>
</consumer>
- <consumer id="my-consumer2">
- <metric-set id="my-set2" />
+
+ <consumer id="cloudwatch-self-hosted-with-default-auth">
+ <metric-set id="public" />
+ <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" />
</consumer>
+
+ <consumer id="cloudwatch-self-hosted-with-profile">
+ <metric id="my-custom-metric" />
+ <cloudwatch region="us-east1" namespace="another-namespace">
+ <shared-credentials file="/user/.aws/credentials" profile="profile-in-credentials-file" />
+ </cloudwatch>
+ </consumer>
+
</metrics>
<logforwarding>
<splunk deployment-server="foo:8989" client-name="foobar" splunk-home="/opt/splunk" phone-home-interval="900"/>
@@ -76,7 +90,6 @@
<exclude>
<binding>http//*/foo/*</binding>
</exclude>
- <application>my-app</application>
<vespa-domain>vespa.vespa.cd</vespa-domain>
</access-control>
@@ -119,6 +132,13 @@
<certificate-file>/foo/cert</certificate-file>
<ca-certificates-file>/foo/cacerts</ca-certificates-file>
<client-authentication>want</client-authentication>
+ <cipher-suites>
+ TLS_AES_128_GCM_SHA256,
+ TLS_AES_256_GCM_SHA384,
+ TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+ </cipher-suites>
+ <protocols>TLSv1.2,TLSv1.3</protocols>
</ssl>
</server>
<server port="4083" id="sslProvider">