summaryrefslogtreecommitdiffstats
path: root/config-model/src/main/java
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /config-model/src/main/java
Publish
Diffstat (limited to 'config-model/src/main/java')
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java279
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java34
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModel.java64
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java72
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java20
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java51
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java256
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java16
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java64
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java63
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java21
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java113
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java119
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java94
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java135
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java15
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java115
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java381
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java53
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java80
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java52
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java135
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java448
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java67
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java109
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/Host.java40
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java120
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java64
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java164
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java49
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java40
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java243
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java169
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java111
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java66
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/package-info.java6
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java15
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java58
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java13
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java39
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java366
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java39
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java74
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java140
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java415
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java31
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java21
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java48
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java65
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Index.java203
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java39
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java968
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java90
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java126
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Search.java555
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java429
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java40
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java31
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java28
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java157
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java136
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java184
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java82
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java26
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java79
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java54
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java64
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java559
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java231
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java146
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java61
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java44
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java93
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java72
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java62
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java374
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java105
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java37
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java144
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java110
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java111
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java275
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java107
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java50
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java12
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java23
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java320
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java73
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java42
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java151
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java87
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java45
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java68
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java316
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java770
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java64
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java72
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java13
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java19
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java20
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java35
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java26
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java40
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java153
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java14
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java24
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java11
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java13
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java14
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java33
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java114
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java53
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java12
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java53
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java32
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java23
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java35
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java41
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java91
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java24
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java50
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java81
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java44
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java30
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java39
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java23
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java63
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java16
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java78
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java70
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java47
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java46
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java49
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java201
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java31
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java43
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java58
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java98
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java94
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java70
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java222
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java38
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java43
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java47
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java33
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java110
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java141
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java149
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java69
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java86
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java79
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java60
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java47
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java67
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java92
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java82
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java76
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java33
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java138
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java84
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java131
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java37
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java28
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java55
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java30
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java40
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java40
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java102
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java45
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java73
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java57
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java45
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java63
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java122
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java61
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java67
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java32
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java69
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java42
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java38
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java62
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java33
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java85
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java46
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java48
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java156
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java158
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java79
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java61
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java126
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java71
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java27
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java350
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java99
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java516
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Affinity.java37
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Client.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java84
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java52
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java58
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java107
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Host.java121
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostResource.java251
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java203
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java196
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Logd.java34
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java28
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java168
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Service.java142
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java50
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java557
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java196
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java248
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java145
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java294
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java49
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java75
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java99
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java77
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java47
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java77
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java20
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java47
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java110
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java17
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java88
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java54
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java159
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java27
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java86
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java69
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java81
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java42
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java104
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java23
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java52
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java162
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java47
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java34
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java98
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java54
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java54
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java66
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java84
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java36
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java116
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java39
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java60
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java160
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java97
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java89
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java57
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java77
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java37
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java38
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java101
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java197
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java138
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java38
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java56
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java243
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java109
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java53
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java48
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java47
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java138
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java247
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java576
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java211
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java105
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java55
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java301
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java55
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java56
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java156
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java38
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java26
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java51
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java102
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java64
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java54
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java38
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java21
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java23
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java88
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java283
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java45
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java60
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java34
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java43
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java77
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java165
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java20
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java29
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java42
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/Container.java408
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java816
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java64
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java99
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java67
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java65
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java26
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java36
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java23
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java38
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java81
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java107
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java60
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java34
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java181
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java178
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java43
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java53
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java43
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java149
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java61
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java123
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java73
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java43
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java49
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java83
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java27
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java35
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java28
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java84
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java149
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java71
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java19
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java20
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java204
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java138
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java100
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java15
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java276
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java34
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java53
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java52
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java269
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java29
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java115
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java188
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java45
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java129
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java26
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java61
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java95
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java58
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java51
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java141
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java96
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java58
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java132
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java68
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java576
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java45
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java81
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java119
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java172
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java60
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java121
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/Content.java347
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java80
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java299
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java83
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java79
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java56
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java151
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java62
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java117
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java37
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java57
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java55
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java72
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java420
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java116
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java72
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java604
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java80
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java53
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java27
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java30
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java40
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java36
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java42
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java41
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java57
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java83
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java73
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java80
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java51
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java56
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java149
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java59
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java120
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java98
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java28
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java57
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java58
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/package-info.java188
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java385
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java29
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java201
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java129
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java226
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java131
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java110
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java90
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java54
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java45
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java390
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java44
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java89
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java33
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java153
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java27
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java46
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java32
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java288
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java29
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java123
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java62
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java377
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java69
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java149
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java91
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java114
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java5
518 files changed, 47126 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java
new file mode 100644
index 00000000000..1ef0ed6ec0a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java
@@ -0,0 +1,279 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.cloud.config.*;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Version;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.cloud.config.ModelConfig.Hosts;
+import com.yahoo.cloud.config.ModelConfig.Hosts.Services;
+import com.yahoo.cloud.config.ModelConfig.Hosts.Services.Ports;
+import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig;
+import com.yahoo.messagebus.MessagebusConfig;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.clients.Clients;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.routing.Routing;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * This is the parent of all ConfigProducers in the system resulting from configuring an application.
+ *
+ * @author gjoranv
+ */
+public class ApplicationConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>> implements CommonConfigsProducer {
+
+ private final DocumentModel documentModel;
+ private Routing routing = null;
+ // The ConfigProducers contained in this Vespa. (configId->producer)
+ Map<String, ConfigProducer> id2producer = new LinkedHashMap<>();
+ private Admin admin = null;
+ private HostSystem hostSystem = null;
+ private final Version vespaVersion;
+ private final ApplicationId applicationId;
+
+ /**
+ * Creates and initializes a new Vespa from the service config file
+ * in the given application directory.
+ *
+ * @param parent The parent, usually VespaModel
+ * @param name The name, used as configId
+ * @param documentModel DocumentModel to serve global document config from.
+ */
+ public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) {
+ super(parent, name);
+ this.documentModel = documentModel;
+ this.vespaVersion = vespaVersion;
+ this.applicationId = applicationId;
+ }
+
+ /**
+ * @return an unmodifiable copy of the set of configIds in this VespaModel.
+ */
+ public Set<String> getConfigIds() {
+ return Collections.unmodifiableSet(id2producer.keySet());
+ }
+
+ /**
+ * Returns the ConfigProducer with the given id, or null if no such
+ * configId exists.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return ConfigProducer with the given configId
+ */
+ public ConfigProducer getConfigProducer(String configId) {
+ return id2producer.get(configId);
+ }
+
+ /**
+ * Returns the Service with the given id, or null if no such
+ * configId exists or if it belongs to a non-Service ConfigProducer.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return Service with the given configId
+ */
+ public Service getService(String configId) {
+ ConfigProducer cp = getConfigProducer(configId);
+ if (cp == null || !(cp instanceof Service)) {
+ return null;
+ }
+ return (Service) cp;
+ }
+
+ /**
+ * Adds the descendant (at any depth level), so it can be looked up
+ * on configId in the Map.
+ *
+ * @param descendant The configProducer descendant to add
+ */
+ // TODO: Make protected if this moves to the same package as AbstractConfigProducer
+ public void addDescendant(AbstractConfigProducer descendant) {
+ id2producer.put(descendant.getConfigId(), descendant);
+ }
+
+ /**
+ * Prepares the model for start. The {@link VespaModel} calls
+ * this methods after it has loaded this and all plugins have been loaded and
+ * their initialize() methods have been called.
+ *
+ * @param plugins All initialized plugins of the vespa model.
+ */
+ public void prepare(ConfigModelRepo plugins) {
+ if (routing != null) {
+ routing.deriveCommonSettings(plugins);
+ }
+ }
+
+ public void setupAdmin(Admin admin) {
+ this.admin = admin;
+ }
+
+ public void setupRouting(ConfigModelRepo configModels) {
+ if (admin != null) {
+ Routing routing = configModels.getRouting();
+ if (routing == null) {
+ routing = new Routing(ConfigModelContext.createFromParentAndId(configModels, this, "routing"));
+ configModels.add(routing);
+ }
+ this.routing = routing;
+ }
+ }
+
+ @Override
+ public void getConfig(DocumentmanagerConfig.Builder builder) {
+ new DocumentManager().produce(documentModel, builder);
+ }
+
+ @Override
+ public void getConfig(DocumenttypesConfig.Builder builder) {
+ new DocumentTypes().produce(documentModel, builder);
+ }
+
+ @Override
+ public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) {
+ if (routing != null) {
+ routing.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(MessagebusConfig.Builder builder) {
+ if (routing != null) {
+ routing.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(LogdConfig.Builder builder) {
+ if (admin != null) {
+ admin.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(SlobroksConfig.Builder builder) {
+ if (admin != null) {
+ admin.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(ZookeepersConfig.Builder builder) {
+ if (admin != null) {
+ admin.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(LoadTypeConfig.Builder builder) {
+ VespaModel model = (VespaModel) getRoot();
+ Clients clients = model.getClients();
+ if (clients != null) {
+ clients.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(ClusterListConfig.Builder builder) {
+ VespaModel model = (VespaModel) getRoot();
+ for (ContentCluster cluster : model.getContentClusters().values()) {
+ ClusterListConfig.Storage.Builder storage = new ClusterListConfig.Storage.Builder();
+ storage.name(cluster.getName());
+ storage.configid(cluster.getConfigId());
+ builder.storage(storage);
+ }
+ }
+
+ @Override
+ public void getConfig(ModelConfig.Builder builder) {
+ builder.vespaVersion(vespaVersion.toSerializedForm());
+ for (HostResource modelHost : getHostSystem().getHosts()) {
+ builder.hosts(new Hosts.Builder()
+ .name(modelHost.getHostName())
+ .services(getServices(modelHost))
+ );
+ }
+ }
+
+ private List<Services.Builder> getServices(HostResource modelHost) {
+ List<Services.Builder> ret = new ArrayList<>();
+ for (Service modelService : modelHost.getServices()) {
+ ret.add(new Services.Builder()
+ .name(modelService.getServiceName())
+ .type(modelService.getServiceType())
+ .configid(modelService.getConfigId())
+ .clustertype(modelService.getServicePropertyString("clustertype", ""))
+ .clustername(modelService.getServicePropertyString("clustername", ""))
+ .index(Integer.parseInt(modelService.getServicePropertyString("index", "999999")))
+ .ports(getPorts(modelService))
+ );
+ }
+ return ret;
+ }
+
+ private List<Ports.Builder> getPorts(Service modelService) {
+ List<Ports.Builder> ret = new ArrayList<>();
+ PortsMeta portsMeta = modelService.getPortsMeta();
+ for (int i = 0; i < portsMeta.getNumPorts(); i++) {
+ ret.add(new Ports.Builder()
+ .number(modelService.getRelativePort(i))
+ .tags(getPortTags(portsMeta, i))
+ );
+ }
+ return ret;
+ }
+
+ public static String getPortTags(PortsMeta portsMeta, int portNumber) {
+ StringBuilder sb = new StringBuilder();
+ boolean firstTag = true;
+ for (String s : portsMeta.getTagsAt(portNumber)) {
+ if (!firstTag) {
+ sb.append(" ");
+ } else {
+ firstTag = false;
+ }
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
+ public void setHostSystem(HostSystem hostSystem) {
+ this.hostSystem = hostSystem;
+ }
+
+ @Override
+ public HostSystem getHostSystem() {
+ return hostSystem;
+ }
+
+ public FileDistributionConfigProducer getFileDistributionConfigProducer() {
+ return admin.getFileDistributionConfigProducer();
+ }
+
+ public Admin getAdmin() {
+ return admin;
+ }
+
+ @Override
+ public void getConfig(ApplicationIdConfig.Builder builder) {
+ builder.tenant(applicationId.tenant().value());
+ builder.application(applicationId.application().value());
+ builder.instance(applicationId.instance().value());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java
new file mode 100644
index 00000000000..228c74480c4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.cloud.config.ApplicationIdConfig;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.cloud.config.SlobroksConfig;
+import com.yahoo.cloud.config.ClusterListConfig;
+import com.yahoo.cloud.config.ZookeepersConfig;
+import com.yahoo.cloud.config.ModelConfig;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig;
+import com.yahoo.messagebus.MessagebusConfig;
+
+
+/**
+ * This interface describes the configs that are produced by the model producer root.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer,
+ DocumenttypesConfig.Producer,
+ MessagebusConfig.Producer,
+ DocumentrouteselectorpolicyConfig.Producer,
+ LogdConfig.Producer,
+ SlobroksConfig.Producer,
+ ZookeepersConfig.Producer,
+ LoadTypeConfig.Producer,
+ ClusterListConfig.Producer,
+ ModelConfig.Producer,
+ ApplicationIdConfig.Producer {
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java
new file mode 100644
index 00000000000..10342d44914
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+/**
+ * A config model is an abstract representation of a subsystem, which is used
+ * by the builder of that subsystem to derive a config producer tree for the subsystem, and by other
+ * builders to access information about the subsystem for config production at a suitable abstraction level.
+ *
+ * @author gjoranv
+ * @author bratseth
+ * @author lulf
+ */
+public abstract class ConfigModel {
+
+ private final String id;
+
+ /**
+ * Constructs a new config model given a context.
+ *
+ * @param modelContext The model context.
+ */
+ public ConfigModel(ConfigModelContext modelContext) {
+ super();
+ this.id = modelContext.getProducerId();
+ }
+
+ /** Returns the id of this model */
+ public final String getId() { return id; }
+
+ /**
+ * Initializes this model. All inter-model independent initialization
+ * is done by implementing this method.
+ * The model will be made available to dependent models by the framework when this returns.
+ * <p>
+ * TODO: Remove this method, as this is now done by the model builders.
+ *
+ * This default implementation does nothing.
+ *
+ * @param configModelRepo The ConfigModelRepo of the VespaModel
+ * @deprecated This will go away in the next Vespa major release. Instead, inject the models you depend on
+ * in your config model constructor.
+ */
+ public void initialize(ConfigModelRepo configModelRepo) { return; }
+
+ /**
+ * Prepares this model to start serving config requests, possibly using properties of other models.
+ * The framework will call this method after models have been built. The model
+ * should finalize its configurations that depend on other models in this step.
+ *
+ * This default implementation does nothing.
+ *
+ * @param configModelRepo The ConfigModelRepo of the system model
+ */
+ public void prepare(ConfigModelRepo configModelRepo) { return; }
+
+ /**
+ * <p>Returns whether this model must be maintained in memory for serving config requests.
+ * Models which are used to amend other models at build time should override this to return false.</p>
+ *
+ * <p>This default implementation returns true.</p>
+ */
+ public boolean isServing() { return true; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
new file mode 100644
index 00000000000..c375d8e28fa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * This class contains a context that is passed to a model builder, and can be used to retrieve the application package,
+ * logger etc.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ConfigModelContext {
+
+ private final AbstractConfigProducer producer;
+ private final String producerId;
+ private final DeployState deployState;
+ private final ConfigModelRepoAdder configModelRepoAdder;
+
+ private ConfigModelContext(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder,
+ AbstractConfigProducer producer, String producerId) {
+ this.deployState = deployState;
+ this.configModelRepoAdder = configModelRepoAdder;
+ this.producer = producer;
+ this.producerId = producerId;
+ }
+
+ public ApplicationPackage getApplicationPackage() { return deployState.getApplicationPackage(); }
+ public String getProducerId() { return producerId; }
+ public AbstractConfigProducer getParentProducer() { return producer; }
+ public DeployLogger getDeployLogger() { return deployState.getDeployLogger(); }
+ public DeployState getDeployState() { return deployState; }
+
+ /** Returns write access to the config model repo, or null (only) if this is improperly initialized during testing */
+ public ConfigModelRepoAdder getConfigModelRepoAdder() { return configModelRepoAdder; }
+
+ /**
+ * Create a new context with a different parent, but with the same id and application package.
+ *
+ * @param newParent The parent to use for the new context.
+ * @return A new context.
+ */
+ public ConfigModelContext modifyParent(AbstractConfigProducer newParent) {
+ return ConfigModelContext.create(deployState, configModelRepoAdder, newParent, producerId);
+ }
+
+ /**
+ * Create an application context from a parent producer and an id.
+ * @param deployState The global deploy state for this model.
+ * @param parent The parent to be used for the config model.
+ * @param id The id to be used for the config model.
+ * @return An model context that can be passed to a model.
+ */
+ public static ConfigModelContext create(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder,
+ AbstractConfigProducer parent, String id) {
+ return new ConfigModelContext(deployState, configModelRepoAdder, parent, id);
+ }
+
+ /**
+ * Create an application context from a parent producer and an id.
+ * @param parent The parent to be used for the config model.
+ * @param id The id to be used for the config model.
+ * @return An model context that can be passed to a model.
+ */
+ public static ConfigModelContext createFromParentAndId(ConfigModelRepoAdder configModelRepoAdder, AbstractConfigProducer parent, String id) {
+ return create(parent.getRoot().getDeployState(), configModelRepoAdder, parent, id);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java
new file mode 100644
index 00000000000..a9b3fc73eb3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+/**
+ * Interface for factories of config models.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public interface ConfigModelInstanceFactory<MODEL extends ConfigModel> {
+
+ /**
+ * Create an instance of {@link com.yahoo.config.model.ConfigModel} given the input context.
+ *
+ * @param context The {@link com.yahoo.config.model.ConfigModelContext} to use.
+ * @return an instance of {@link com.yahoo.config.model.ConfigModel}
+ */
+ MODEL createModel(ConfigModelContext context);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java
new file mode 100644
index 00000000000..2735b9f7fdf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * A resolver of implementations of named config models.
+ * Registries may be chained in a chain of command.
+ *
+ * @author bratseth
+ */
+public abstract class ConfigModelRegistry {
+
+ private final ConfigModelRegistry chained;
+
+ public ConfigModelRegistry() {
+ this(new EmptyTerminalRegistry());
+ }
+
+ /** Creates a config model class registry which forwards unresolved requests to the argument instance */
+ public ConfigModelRegistry(ConfigModelRegistry chained) {
+ this.chained=chained;
+ }
+
+ /**
+ * Returns the builders this id resolves to both in this and any chained registry.
+ *
+ * @return the resolved config model builders, or an empty list (never null) if none
+ */
+ public abstract Collection<ConfigModelBuilder> resolve(ConfigModelId id);
+
+ public ConfigModelRegistry chained() { return chained; }
+
+ /** An empty registry which does not support chaining */
+ private static class EmptyTerminalRegistry extends ConfigModelRegistry {
+
+ public EmptyTerminalRegistry() {
+ super(null);
+ }
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ return Collections.emptyList();
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java
new file mode 100644
index 00000000000..fa46c33a97f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java
@@ -0,0 +1,256 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.application.api.ApplicationFile;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.graph.ModelGraphBuilder;
+import com.yahoo.config.model.graph.ModelNode;
+import com.yahoo.config.model.provision.HostsXmlProvisioner;
+import com.yahoo.log.LogLevel;
+import com.yahoo.path.Path;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.VespaModelBuilder;
+import com.yahoo.vespa.model.clients.Clients;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.routing.Routing;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * A collection of config model instances owned by a system model
+ *
+ * @author gjoranv
+ */
+public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iterable<ConfigModel> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger log = Logger.getLogger(ConfigModelRepo.class.getPackage().toString());
+
+ private final Map<String,ConfigModel> configModelMap = new TreeMap<>();
+ private final List<ConfigModel> configModels = new ArrayList<>();
+
+ /**
+ * Returns a config model for a given id
+ *
+ * @param id the id of the model to return
+ * @return the model, or none if a model with this id is not present in this
+ */
+ public ConfigModel get(String id) {
+ return configModelMap.get(id);
+ }
+
+ /** Adds a new config model instance in this */
+ @Override
+ public void add(ConfigModel model) {
+ configModelMap.put(model.getId(), model);
+ configModels.add(model);
+ }
+
+ /** Returns the models in this as an iterator */
+ public Iterator<ConfigModel> iterator() {
+ return configModels.iterator();
+ }
+
+ /** Returns a read-only view of the config model instances of this */
+ public Map<String,ConfigModel> asMap() { return Collections.unmodifiableMap(configModelMap); }
+
+ /** Initialize part 1.: Reads the config models used in the application package. */
+ public void readConfigModels(DeployState deployState, VespaModelBuilder builder,
+ ApplicationConfigProducerRoot root, ConfigModelRegistry configModelRegistry) throws IOException, SAXException {
+ Element userServicesElement = getServicesFromApp(deployState.getApplicationPackage());
+ readConfigModels(root, userServicesElement, deployState, configModelRegistry);
+ builder.postProc(root, this);
+ }
+
+ private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException, SAXException {
+ try (Reader servicesFile = applicationPackage.getServices()) {
+ return getServicesFromReader(servicesFile);
+ }
+ }
+
+ /**
+ * If the top level is &lt;services&gt;, it contains a list of services elements,
+ * otherwise, the top level tag is a single service.
+ */
+ private List<Element> getServiceElements(Element servicesRoot) {
+ if (servicesRoot.getTagName().equals("services"))
+ return XML.getChildren(servicesRoot);
+
+ List<Element> singleServiceList = new ArrayList<>(1);
+ singleServiceList.add(servicesRoot);
+ return singleServiceList;
+ }
+
+ /**
+ * Creates all the config models specified in the given XML element and
+ * passes their respective XML node as parameter.
+ *
+ * @param root The Root to set as parent for all plugins
+ * @param servicesRoot XML root node of the services file
+ */
+ private void readConfigModels(ApplicationConfigProducerRoot root, Element servicesRoot, DeployState deployState, ConfigModelRegistry configModelRegistry) throws IOException, SAXException {
+ final Map<ConfigModelBuilder, List<Element>> model2Element = new LinkedHashMap<>();
+ ModelGraphBuilder graphBuilder = new ModelGraphBuilder();
+
+ final List<Element> children = getServiceElements(servicesRoot);
+
+ if (XML.getChild(servicesRoot, "admin") == null)
+ children.add(getImplicitAdmin(deployState));
+
+ children.addAll(getPermanentServices(deployState));
+
+ for (Element servicesElement : children) {
+ String tagName = servicesElement.getTagName();
+ if (tagName.equals("config")) continue; // TODO: Remove on Vespa 6
+ if (tagName.equals("cluster")) continue; // TODO: Remove on Vespa 6
+ if ((tagName.equals("clients")) && deployState.isHostedVespa())
+ throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment");
+
+ String tagVersion = servicesElement.getAttribute("version");
+ ConfigModelId xmlId = ConfigModelId.fromNameAndVersion(tagName, tagVersion);
+
+ Collection<ConfigModelBuilder> builders = configModelRegistry.resolve(xmlId);
+
+ if (builders.isEmpty())
+ throw new RuntimeException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component");
+
+ for (ConfigModelBuilder builder : builders) {
+ if ( ! model2Element.containsKey(builder)) {
+ model2Element.put(builder, new ArrayList<>());
+ graphBuilder.addBuilder(builder);
+ }
+ model2Element.get(builder).add(servicesElement);
+ }
+ }
+
+ for (ModelNode node : graphBuilder.build().topologicalSort())
+ buildModels(node, deployState, root, model2Element.get(node.builder));
+ for (ConfigModel model : configModels)
+ model.initialize(ConfigModelRepo.this);
+ }
+
+ private Collection<Element> getPermanentServices(DeployState deployState) throws IOException, SAXException {
+ List<Element> permanentServices = new ArrayList<>();
+ Optional<ApplicationPackage> applicationPackage = deployState.getPermanentApplicationPackage();
+ if (applicationPackage.isPresent()) {
+ ApplicationFile file = applicationPackage.get().getFile(Path.fromString(ApplicationPackage.PERMANENT_SERVICES));
+ if (file.exists()) {
+ try (Reader reader = file.createReader()) {
+ Element permanentServicesRoot = getServicesFromReader(reader);
+ permanentServices.addAll(getServiceElements(permanentServicesRoot));
+ }
+ }
+ }
+ return permanentServices;
+ }
+
+ private Element getServicesFromReader(Reader reader) throws IOException, SAXException {
+ Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(reader));
+ return doc.getDocumentElement();
+ }
+
+ private void buildModels(ModelNode node, DeployState deployState, AbstractConfigProducer parent, List<Element> elements) {
+ for (Element servicesElement : elements) {
+ ConfigModel model = buildModel(node, deployState, parent, servicesElement);
+ if (model.isServing())
+ add(model);
+ }
+ }
+
+ private ConfigModel buildModel(ModelNode node, DeployState deployState, AbstractConfigProducer parent, Element servicesElement) {
+ ConfigModelBuilder builder = node.builder;
+ ConfigModelContext context = ConfigModelContext.create(deployState, this, parent, getIdString(servicesElement));
+ return builder.build(node, servicesElement, context);
+ }
+
+ private static String getIdString(Element spec) {
+ String idString = XmlHelper.getIdString(spec);
+ if (idString == null || idString.isEmpty()) {
+ idString = spec.getTagName();
+ }
+ return idString;
+ }
+
+ /**
+ * Initialize part 2.:
+ * Prepare all config models for starting. Must be called after plugins are loaded and frozen.
+ */
+ public void prepareConfigModels() {
+ for (ConfigModel model : configModels) {
+ model.prepare(ConfigModelRepo.this);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends ConfigModel> List<T> getModels(Class<T> modelClass) {
+ List<T> modelsOfModelClass = new ArrayList<>();
+
+ for (ConfigModel model : asMap().values()) {
+ if (modelClass.isInstance(model))
+ modelsOfModelClass.add((T)model);
+ }
+ return modelsOfModelClass;
+ }
+
+ public Clients getClients() {
+ for (ConfigModel m : configModels) {
+ if (m instanceof Clients) {
+ return (Clients)m;
+ }
+ }
+ return null;
+ }
+
+ public Routing getRouting() {
+ for (ConfigModel m : configModels) {
+ if (m instanceof Routing) {
+ return (Routing)m;
+ }
+ }
+ return null;
+ }
+
+ public Content getContent() {
+ for (ConfigModel m : configModels) {
+ if (m instanceof Content) {
+ return (Content)m;
+ }
+ }
+ return null;
+ }
+
+ // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon
+ private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException {
+ final boolean hostedVespa = deployState.isHostedVespa();
+ String defaultAdminElement = hostedVespa ? getImplicitAdminV4() : getImplicitAdminV2();
+ log.log(LogLevel.DEBUG, "No <admin> defined, using " + defaultAdminElement);
+ return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement();
+ }
+
+ private static String getImplicitAdminV2() {
+ return "<admin version='2.0'>\n" +
+ " <adminserver hostalias='" + HostsXmlProvisioner.IMPLICIT_ADMIN_HOSTALIAS + "'/>\n" +
+ "</admin>\n";
+ }
+
+ private static String getImplicitAdminV4() {
+ return "<admin version='4.0'>\n" +
+ " <nodes count='1' />\n" +
+ "</admin>\n";
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java
new file mode 100644
index 00000000000..93c591b3a8e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+/**
+ * An interface which provides addition of new config models.
+ * This exists because some models need to add additional models during the build phase so *write* access
+ * to the config model repo is needed. *Read* access, on the other hand needs to happen through config model dependency
+ * inkection to avoid circular dependencies or undeclared dependencies working by accident.
+ *
+ * @author bratseth
+ */
+public interface ConfigModelRepoAdder {
+
+ void add(ConfigModel model);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java
new file mode 100644
index 00000000000..f64eee6e863
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import java.io.Serializable;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static javax.xml.xpath.XPathConstants.BOOLEAN;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Utilities for config models
+ *
+ * @author gjoranv
+ */
+// TODO: Split this into appropriate classes, or move to ConfigModel superclass
+public class ConfigModelUtils implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static Pattern hourNmin = Pattern.compile("(\\d\\d):(\\d\\d)");
+
+ public static Map<String, Integer> day2int;
+ static {
+ day2int = new HashMap<>();
+ day2int.put("sunday", 0);
+ day2int.put("monday", 1);
+ day2int.put("tuesday", 2);
+ day2int.put("wednesday", 3);
+ day2int.put("thursday", 4);
+ day2int.put("friday", 5);
+ day2int.put("saturday", 6);
+ }
+
+ /** Parses a 24 hour clock that must be the five characters ##:## to an int stating minutes after midnight. */
+ public static int getTimeOfDay(String time) {
+ Matcher m = ConfigModelUtils.hourNmin.matcher(time);
+ if (m.matches()) {
+ return Integer.parseInt(m.group(1)) * 60 + Integer.parseInt(m.group(2));
+ }
+ throw new IllegalArgumentException("The string '" + time + "' is not in ##:## format.");
+ }
+
+ /** Parses a day of week name in english to an int, where 0 is sunday, 6 saturday. */
+ public static int getDayOfWeek(String day) {
+ return ConfigModelUtils.day2int.get(toLowerCase(day));
+ }
+
+ /**
+ * Create a string with link to documentation for latest release.
+ *
+ * @param filePath Relative path of the file to link to, e.g. reference/services-jdisc.html
+ * @return a String with link to documentation
+ */
+ public static String createDocLink(String filePath) {
+ return filePath;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java
new file mode 100644
index 00000000000..d47a009928d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.google.inject.Inject;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.log.LogLevel;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.FrameworkUtil;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class MapConfigModelRegistry extends ConfigModelRegistry {
+
+ private static final Logger log = Logger.getLogger(MapConfigModelRegistry.class.getPackage().getName());
+ private final List<ConfigModelBuilder> builders;
+
+ /**
+ * Constructs a registry of config models, where the components are injected.
+ *
+ * @param registry a component registry
+ */
+ @Inject
+ public MapConfigModelRegistry(ComponentRegistry<? extends ConfigModelBuilder> registry) {
+ this(registry.allComponents());
+ }
+
+ /**
+ * Constructs a registry of config models.
+ *
+ * @param builderCollection A collection of builders used to populate the registry.
+ */
+ public MapConfigModelRegistry(Collection<? extends ConfigModelBuilder> builderCollection) {
+ super();
+ builders = new ArrayList<>(builderCollection);
+ }
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ Set<ConfigModelBuilder> matchingBuilders = new HashSet<>(chained().resolve(id));
+ for (ConfigModelBuilder builder : builders)
+ if (builder.handlesElements().contains(id))
+ matchingBuilders.add(builder);
+ return matchingBuilders;
+ }
+
+ /**
+ * Create a registry from a variable argument list of builders.
+ *
+ * @param builders A variable argument list of builders to use in this map
+ * @return a ConfigModelRegistry instance.
+ */
+ public static ConfigModelRegistry createFromList(ConfigModelBuilder<? extends ConfigModel> ... builders) {
+ return new MapConfigModelRegistry(Arrays.asList(builders));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java
new file mode 100644
index 00000000000..6930c8c792b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+
+import java.util.Collection;
+
+/**
+ * A config model class registry that only forwards to the chained registry.
+ *
+ * @author bratseth
+ */
+public class NullConfigModelRegistry extends ConfigModelRegistry {
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ return chained().resolve(id);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
new file mode 100644
index 00000000000..8b6ea27ccab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.admin;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.deploy.DeployProperties;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder;
+import com.yahoo.vespa.model.builder.xml.dom.DomAdminV4Builder;
+import com.yahoo.vespa.model.container.ContainerModel;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * Config model adaptor of the Admin class.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class AdminModel extends ConfigModel {
+
+ private Admin admin = null;
+ private final Collection<ContainerModel> containerModels;
+
+ /**
+ * Constructs a new config model given a context.
+ *
+ * @param modelContext the model context.
+ */
+ public AdminModel(ConfigModelContext modelContext, Collection<ContainerModel> containerModels) {
+ super(modelContext);
+ this.containerModels = containerModels;
+ }
+
+ public Admin getAdmin() { return admin; }
+
+ private Collection<ContainerModel> getContainerModels() { return containerModels; }
+
+ @Override
+ public void prepare(ConfigModelRepo configModelRepo) {
+ verifyClusterControllersOnlyDefinedForContent(configModelRepo);
+ if (admin == null || admin.getClusterControllers() == null) return;
+ admin.getClusterControllers().prepare();
+ }
+
+ private void verifyClusterControllersOnlyDefinedForContent(ConfigModelRepo configModelRepo) {
+ Admin admin = getAdmin();
+ if (admin == null || admin.getClusterControllers() == null) return;
+ if (configModelRepo.getContent() == null) {
+ throw new IllegalArgumentException("Declaring <clustercontrollers> in <admin> in services.xml will not work when <content> is not defined");
+ }
+ }
+
+ public static class BuilderV2 extends ConfigModelBuilder<AdminModel> {
+
+ public static final List<ConfigModelId> configModelIds =
+ ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "2.0"),
+ ConfigModelId.fromNameAndVersion("admin", "1.0"));
+
+ public BuilderV2() {
+ super(AdminModel.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() { return configModelIds; }
+
+ @Override
+ public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
+ AbstractConfigProducer parent = modelContext.getParentProducer();
+ DeployProperties properties = modelContext.getDeployState().getProperties();
+ DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs());
+ model.admin = domBuilder.build(parent, adminElement);
+ // TODO: Is required since other models depend on admin.
+ if (parent instanceof ApplicationConfigProducerRoot) {
+ ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin);
+ }
+ }
+ }
+
+ public static class BuilderV4 extends ConfigModelBuilder<AdminModel> {
+
+ public static final List<ConfigModelId> configModelIds =
+ ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "3.0"),
+ ConfigModelId.fromNameAndVersion("admin", "4.0"));
+
+ public BuilderV4() {
+ super(AdminModel.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() { return configModelIds; }
+
+ @Override
+ public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
+ AbstractConfigProducer parent = modelContext.getParentProducer();
+ DeployProperties properties = modelContext.getDeployState().getProperties();
+ DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels());
+ model.admin = domBuilder.build(parent, adminElement);
+ // TODO: Is required since other models depend on admin.
+ if (parent instanceof ApplicationConfigProducerRoot) {
+ ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java
new file mode 100644
index 00000000000..a0b1be20df6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelInstanceFactory;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.api.ConfigModelPlugin;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import org.w3c.dom.Element;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
+
+/**
+ * Builds a config model using DOM parsers
+ *
+ * @author vegardh
+ * @since 5.1.10
+ */
+public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends AbstractComponent implements ConfigModelPlugin {
+
+ private Class<MODEL> configModelClass;
+
+ public ConfigModelBuilder(Class<MODEL> configModelClass) {
+ this.configModelClass = configModelClass;
+ }
+
+ /**
+ * Method that must return the XML elements this builder handles. Subclasses must implement this in order to
+ * get called when one of the elements have been encountered when parsing.
+ *
+ * @return A list of elements that this builder handles.
+ */
+ public abstract List<ConfigModelId> handlesElements();
+
+ /**
+ * Convenience hook called from {@link #build}. Implement this method to build a config model.
+ *
+ * @param spec The XML element that this builder should handle.
+ * @param modelContext A model context that contains the application package and other data needed by the
+ * config model constructor.
+ */
+ public abstract void doBuild(MODEL model, Element spec, ConfigModelContext modelContext);
+
+ /**
+ * Builds an instance of this component model.
+ * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...).
+ *
+ * @param deployState a global deployment state used for this model.
+ * @param parent the root config producer this should be added to
+ * @param spec the XML element this is constructed from
+ */
+ public final MODEL build(DeployState deployState, ConfigModelRepo configModelRepo, AbstractConfigProducer parent, Element spec) {
+ ConfigModelContext context = ConfigModelContext.create(deployState, configModelRepo, parent, getIdString(spec));
+ return build(new DefaultModelInstanceFactory(), spec, context);
+ }
+
+ /**
+ * Builds an instance of this component model.
+ * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...).
+ *
+ * @param factory A factory capable of creating models.
+ * @param spec the XML element this is constructed from
+ * @param context A context object containing various data used by builders.
+ */
+ public MODEL build(ConfigModelInstanceFactory<MODEL> factory, Element spec, ConfigModelContext context) {
+ MODEL model = factory.createModel(context);
+ doBuild(model, spec, context);
+ return model;
+ }
+
+ public Class<MODEL> getModelClass() {
+ return configModelClass;
+ }
+
+ private static String getIdString(Element spec) {
+ String idString = XmlHelper.getIdString(spec);
+ if (idString == null || idString.isEmpty()) {
+ idString = spec.getTagName();
+ }
+ return idString;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof ConfigModelBuilder)) {
+ return false;
+ }
+ ConfigModelBuilder otherBuilder = (ConfigModelBuilder) other;
+ List<ConfigModelId> thisIds = this.handlesElements();
+ List<ConfigModelId> otherIds = otherBuilder.handlesElements();
+ if (thisIds.size() != otherIds.size()) {
+ return false;
+ }
+ for (int i = 0; i < thisIds.size(); i++) {
+ if (!thisIds.get(i).equals(otherIds.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ private class DefaultModelInstanceFactory implements ConfigModelInstanceFactory<MODEL> {
+ @Override
+ public MODEL createModel(ConfigModelContext context) {
+ try {
+ Constructor<MODEL> constructor = configModelClass.getConstructor(ConfigModelContext.class);
+ return constructor.newInstance(context);
+ } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Error constructing model '" + configModelClass.getName() + "'", e);
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java
new file mode 100644
index 00000000000..fca114757ec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.component.Version;
+
+/**
+ * A {@link ConfigModelId} describes an element handled by a {@link ConfigModelBuilder}.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ConfigModelId implements Comparable<ConfigModelId> {
+
+ private final String name;
+ private final Version version;
+ private final String stringValue;
+
+ private ConfigModelId(String name, Version version) {
+ this.name = name;
+ this.version = version;
+ this.stringValue = toStringValue();
+ }
+
+ /**
+ * Create id with a name and version
+ * @param tagName Name of the id
+ * @param tagVersion Version of the id
+ * @return A ConfigModelId instance
+ */
+ public static ConfigModelId fromNameAndVersion(String tagName, String tagVersion) {
+ return new ConfigModelId(tagName, Version.fromString(tagVersion));
+ }
+
+ /**
+ * Create id with given name, using default version 1.
+ *
+ * @param tagName Name of the id
+ * @return A ConfigModelId instance
+ */
+ public static ConfigModelId fromName(String tagName) {
+ return new ConfigModelId(tagName, new Version(1));
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (!(object instanceof ConfigModelId)) return false;
+ ConfigModelId other = (ConfigModelId)object;
+ return this.name.equals(other.name) && this.version.equals(other.version);
+ }
+
+ @Override
+ public int compareTo(ConfigModelId other) {
+ if (other == this) return 0;
+ int cmp = this.name.compareTo(other.name);
+ if (cmp == 0) {
+ cmp = this.version.compareTo(other.version);
+ }
+ return cmp;
+ }
+
+ @Override
+ public String toString() {
+ return stringValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return stringValue.hashCode();
+ }
+
+ /**
+ * Return the XML element name.
+ * @return the name of the config model
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Return the XML element version.
+ * @return the version of the config model
+ */
+ Version getVersion() {
+ return version;
+ }
+
+ private String toStringValue() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(name);
+ sb.append(".");
+ sb.append(version);
+ return sb.toString();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
new file mode 100644
index 00000000000..924d888b0d0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.XML;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+
+/**
+ * Static methods for helping dom building
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public final class XmlHelper {
+ private static final Logger log = Logger.getLogger(XmlHelper.class.getPackage().toString());
+
+
+ private static final String idReference = "idref";
+ // Access to this needs to be synchronized (as it is in getDocumentBuilder() below)
+ public static final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+
+ static {
+ XmlHelper.factory.setNamespaceAware(true);
+ // if use jdom and jaxen this will fail badly:
+ XmlHelper.factory.setXIncludeAware(true);
+ }
+
+ private XmlHelper() {}
+
+ public static String nullIfEmpty(String attribute) {
+ if (attribute.isEmpty())
+ return null;
+ else
+ return attribute;
+ }
+
+ /**
+ * For searchers inside search chains, the id may be both a reference and an id at once, or just a reference.
+ * In other cases, it is clear which one it is from context, so I think the difference is not worth bothering users
+ * with, unless they are XML purists in which case they will have the option of writing this correctly.
+ * - Jon
+ */
+ public static String getIdString(Element element) {
+ String idString = element.getAttribute("id");
+ if (idString == null || idString.trim().equals(""))
+ idString = element.getAttribute(idReference);
+ if (idString == null || idString.trim().equals(""))
+ idString = element.getAttribute("ident");
+ return idString;
+ }
+
+ public static ComponentId getId(Element element) {
+ return new ComponentId(getIdString(element));
+ }
+
+ public static ComponentSpecification getIdRef(Element element) {
+ return new ComponentSpecification(getIdString(element));
+ }
+
+ public static Document getDocument(Reader reader) {
+ Document doc;
+ try {
+ doc = getDocumentBuilder().parse(new InputSource(reader));
+ } catch (SAXException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ return doc;
+ }
+
+ public static List<String> splitAndDiscardEmpty(String field, String regex) {
+ List<String> ret = new ArrayList<>();
+ for (String t : field.split(regex)) {
+ if (!t.isEmpty()) {
+ ret.add(t);
+ }
+ }
+ return ret;
+ }
+
+ public static List<String> spaceSeparatedSymbols(String field) {
+ return splitAndDiscardEmpty(field, " ");
+ }
+
+ public static Collection<String> spaceSeparatedSymbolsFromAttribute(Element spec, String name) {
+ return spaceSeparatedSymbols(spec.getAttribute(name));
+ }
+
+ public static Collection<String> valuesFromElements(Element parent, String elementName) {
+ List<String> symbols = new ArrayList<>();
+ for (Element symbol : XML.getChildren(parent, elementName)) {
+ symbols.add(XML.getValue(symbol).trim());
+ }
+ return symbols;
+ }
+
+ public static boolean isReference(Element element) {
+ return element.hasAttribute(idReference);
+ }
+
+ /**
+ * Creates a new XML document builder.
+ *
+ * @return A new DocumentBuilder instance, or null if we fail to get one.
+ */
+ public static synchronized DocumentBuilder getDocumentBuilder() {
+ try {
+ DocumentBuilder docBuilder = factory.newDocumentBuilder();
+ log.log(LogLevel.DEBUG, "XML parser now operational!");
+ return docBuilder;
+ } catch (ParserConfigurationException e) {
+ log.log(LogLevel.WARNING, "No XML parser available - " + e);
+ return null;
+ }
+ }
+
+ public static Optional<String> getOptionalAttribute(Element element, String name) {
+ return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java
new file mode 100644
index 00000000000..6ce400dc922
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
new file mode 100644
index 00000000000..22ea054a27d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.vespa.config.ConfigDefinition;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public interface ConfigDefinitionStore {
+
+ ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
new file mode 100644
index 00000000000..3f6ef494a27
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.provision.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Collection of properties for a deployment.
+ *
+ * @author lulf
+ * @since 5.17
+ */
+public class DeployProperties {
+ private final boolean multitenant;
+ private final ApplicationId applicationId;
+ private final List<ConfigServerSpec> serverSpecs = new ArrayList<>();
+ private final boolean hostedVespa;
+ private final Version vespaVersion;
+ private final Zone zone;
+
+ private DeployProperties(boolean multitenant,
+ ApplicationId applicationId,
+ List<ConfigServerSpec> configServerSpecs,
+ boolean hostedVespa, Version vespaVersion, Zone zone) {
+ this.vespaVersion = vespaVersion;
+ this.zone = zone;
+ this.multitenant = multitenant || hostedVespa || Boolean.getBoolean("multitenant");
+ this.applicationId = applicationId;
+ this.serverSpecs.addAll(configServerSpecs);
+ this.hostedVespa = hostedVespa;
+ }
+
+
+ public boolean multitenant() {
+ return multitenant;
+ }
+
+ public ApplicationId applicationId() {
+ return applicationId;
+ }
+
+ public List<ConfigServerSpec> configServerSpecs() {
+ return serverSpecs;
+ }
+
+ public Quota quota() {
+ return new Quota(Integer.MAX_VALUE);
+ }
+
+ public boolean hostedVespa() {
+ return hostedVespa;
+ }
+
+ public Version vespaVersion() {
+ return vespaVersion;
+ }
+
+ public Zone zone() {
+ return zone;
+ }
+
+ public static class Builder {
+
+ private ApplicationId applicationId = ApplicationId.defaultId();
+ private boolean multitenant = false;
+ private List<ConfigServerSpec> configServerSpecs = new ArrayList<>();
+ private Quota quota = new Quota(Integer.MAX_VALUE);
+ private boolean hostedVespa = false;
+ private Version vespaVersion = Version.fromIntValues(1, 0, 0);
+ private Zone zone = Zone.defaultZone();
+
+ public Builder applicationId(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ return this;
+ }
+
+ public Builder multitenant(boolean multitenant) {
+ this.multitenant = multitenant;
+ return this;
+ }
+
+ public Builder configServerSpecs(List<ConfigServerSpec> configServerSpecs) {
+ this.configServerSpecs = configServerSpecs;
+ return this;
+ }
+
+ public Builder quota(Quota quota) {
+ this.quota = quota;
+ return this;
+ }
+
+ public Builder vespaVersion(Version version) {
+ this.vespaVersion = version;
+ return this;
+ }
+
+ public Builder hostedVespa(boolean hostedVespa) {
+ this.hostedVespa = hostedVespa;
+ return this;
+ }
+
+ public Builder zone(Zone zone) {
+ this.zone = zone;
+ return this;
+ }
+
+ public DeployProperties build() {
+ return new DeployProperties(multitenant, applicationId, configServerSpecs, hostedVespa, vespaVersion, zone);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
new file mode 100644
index 00000000000..5896dc59df2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -0,0 +1,381 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.application.api.UnparsedConfigDefinition;
+import com.yahoo.config.codegen.CNode;
+import com.yahoo.config.model.api.ConfigDefinitionRepo;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.application.provider.MockFileRegistry;
+import com.yahoo.config.model.provision.HostsXmlProvisioner;
+import com.yahoo.config.model.provision.SingleNodeProvisioner;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.Rotation;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.log.LogLevel;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.config.ConfigDefinition;
+import com.yahoo.vespa.config.ConfigDefinitionBuilder;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.xml.ValidationOverridesXMLReader;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+import com.yahoo.vespa.model.container.search.QueryProfilesBuilder;
+import com.yahoo.vespa.model.container.search.SemanticRuleBuilder;
+import com.yahoo.vespa.model.container.search.SemanticRules;
+import com.yahoo.vespa.model.search.SearchDefinition;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+
+/**
+ * Contains various state during deploy that should be reachable to all builders of a {@link com.yahoo.config.model.ConfigModel}
+ *
+ * @author lulf
+ * @since 5.8
+ */
+public class DeployState implements ConfigDefinitionStore {
+
+ private static final Logger log = Logger.getLogger(DeployState.class.getName());
+
+ private final DeployLogger logger;
+ private final FileRegistry fileRegistry;
+ private final DocumentModel documentModel;
+ private final List<SearchDefinition> searchDefinitions;
+ private final ApplicationPackage applicationPackage;
+ private final Optional<ConfigDefinitionRepo> configDefinitionRepo;
+ private final Optional<ApplicationPackage> permanentApplicationPackage;
+ private final Optional<Model> previousModel;
+ private final DeployProperties properties;
+ private final Set<Rotation> rotations;
+ private final Zone zone;
+ private final QueryProfiles queryProfiles;
+ private final SemanticRules semanticRules;
+ private final ValidationOverrides validationOverrides;
+
+ private final HostProvisioner provisioner;
+
+ public static DeployState createTestState() {
+ return new Builder().build();
+ }
+
+ public static DeployState createTestState(ApplicationPackage applicationPackage) {
+ return new Builder().applicationPackage(applicationPackage).build();
+ }
+
+ private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry,
+ FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties,
+ Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo,
+ java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now) {
+ this.logger = deployLogger;
+ this.fileRegistry = fileRegistry;
+ this.rankProfileRegistry = rankProfileRegistry;
+ this.applicationPackage = applicationPackage;
+ this.properties = properties;
+ this.previousModel = previousModel;
+ if (hostProvisioner.isPresent()) {
+ this.provisioner = hostProvisioner.get();
+ } else {
+ this.provisioner = getDefaultModelHostProvisioner(applicationPackage);
+ }
+ this.searchDefinitions = searchDocumentModel.getSearchDefinitions();
+ this.documentModel = searchDocumentModel.getDocumentModel();
+ this.permanentApplicationPackage = permanentApplicationPackage;
+ this.configDefinitionRepo = configDefinitionRepo;
+ this.rotations = rotations;
+ this.zone = zone;
+ this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated
+ this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated
+ this.validationOverrides = new ValidationOverridesXMLReader().read(applicationPackage.getValidationOverrides(), now);
+
+ }
+
+ public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) {
+ if (applicationPackage.getHosts() == null) {
+ return new SingleNodeProvisioner();
+ } else {
+ return new HostsXmlProvisioner(applicationPackage.getHosts());
+ }
+ }
+
+ /** Get the global rank profile registry for this application. */
+ public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; }
+
+ /** Returns the validation overrides of this. This is never null */
+ public ValidationOverrides validationOverrides() { return validationOverrides; }
+
+ /**
+ * Returns the config def with the given name and namespace.
+ *
+ * @param defKey The {@link ConfigDefinitionKey} that will uniquely identify a config definition.
+ * @return The definition with a matching name and namespace
+ * @throws java.lang.IllegalArgumentException if def is not found.
+ */
+ public final ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey) {
+ if (existingConfigDefs == null) {
+ existingConfigDefs = new LinkedHashMap<>();
+ if (configDefinitionRepo.isPresent()) {
+ existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get()));
+ }
+ existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs());
+ }
+ log.log(LogLevel.DEBUG, "Getting config definition " + defKey);
+ // Fall back to default namespace if not found.
+ if (defKey.getNamespace() == null || defKey.getNamespace().isEmpty()) {
+ defKey = new ConfigDefinitionKey(defKey.getName(), CNode.DEFAULT_NAMESPACE);
+ }
+ ConfigDefinitionKey lookupKey = defKey;
+ // Fall back to just using name
+ if (!existingConfigDefs.containsKey(lookupKey)) {
+
+ int count = 0;
+ for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) {
+ if (entry.getName().equals(defKey.getName())) {
+ count++;
+ }
+ }
+ if (count > 1) {
+ throw new IllegalArgumentException("Using config definition '" + defKey.getName() + "' is ambiguous, there are more than one config definitions with this name, please specify namespace");
+ }
+
+ lookupKey = null;
+ log.log(LogLevel.DEBUG, "Could not find config definition '" + defKey + "', trying with same name in all namespaces");
+ for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) {
+ if (entry.getName().equals(defKey.getName()) && defKey.getNamespace().equals(CNode.DEFAULT_NAMESPACE)) {
+ log.log(LogLevel.INFO, "Could not find config definition '" + defKey + "'" +
+ ", using config definition '" + entry + "' with same name instead (please use new namespace when specifying this config)");
+ lookupKey = entry;
+ break;
+ }
+ }
+ }
+
+ if (lookupKey == null) {
+ throw new IllegalArgumentException("Could not find a config definition with name '" + defKey + "'.");
+ }
+ if (defArchive.get(defKey) != null) {
+ log.log(LogLevel.DEBUG, "Found in archive: " + defKey);
+ return defArchive.get(defKey);
+ }
+
+ log.log(LogLevel.DEBUG, "Retrieving config definition: " + defKey);
+ ConfigDefinition def = existingConfigDefs.get(lookupKey).parse();
+
+ log.log(LogLevel.DEBUG, "Adding " + def + " to archive");
+ defArchive.put(defKey, def);
+ return def;
+ }
+
+ private static Map<ConfigDefinitionKey, UnparsedConfigDefinition> createLazyMapping(final ConfigDefinitionRepo configDefinitionRepo) {
+ Map<ConfigDefinitionKey, UnparsedConfigDefinition> keyToRepo = new LinkedHashMap<>();
+ for (final Map.Entry<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defEntry : configDefinitionRepo.getConfigDefinitions().entrySet()) {
+ keyToRepo.put(defEntry.getKey(), new UnparsedConfigDefinition() {
+ @Override
+ public ConfigDefinition parse() {
+ return ConfigDefinitionBuilder.createConfigDefinition(configDefinitionRepo.getConfigDefinitions().get(defEntry.getKey()).getCNode());
+ }
+
+ @Override
+ public String getUnparsedContent() {
+ throw new UnsupportedOperationException("Cannot get unparsed content from " + defEntry.getKey());
+ }
+ });
+ }
+ return keyToRepo;
+ }
+
+ // Global registry of rank profiles.
+ // TODO: I think this can be removed when we remove "<search version=2.0>" and only support content.
+ private final RankProfileRegistry rankProfileRegistry;
+
+ // Mapping from key to something that can create a config definition.
+ private Map<ConfigDefinitionKey, UnparsedConfigDefinition> existingConfigDefs = null;
+
+ // Cache of config defs for all [def,version] combinations looked up so far.
+ private final Map<ConfigDefinitionKey, ConfigDefinition> defArchive = new LinkedHashMap<>();
+
+ public ApplicationPackage getApplicationPackage() {
+ return applicationPackage;
+ }
+ public List<SearchDefinition> getSearchDefinitions() {
+ return searchDefinitions;
+ }
+
+ public DocumentModel getDocumentModel() {
+ return documentModel;
+ }
+
+ public DeployLogger getDeployLogger() {
+ return logger;
+ }
+
+ public FileRegistry getFileRegistry() {
+ return fileRegistry;
+ }
+
+ public HostProvisioner getProvisioner() { return provisioner; }
+
+ public Optional<ApplicationPackage> getPermanentApplicationPackage() {
+ return permanentApplicationPackage;
+ }
+
+ public DeployProperties getProperties() { return properties; }
+
+ public Optional<Model> getPreviousModel() { return previousModel; }
+
+ public boolean isHostedVespa() {
+ return properties.hostedVespa();
+ }
+
+ public Set<Rotation> getRotations() {
+ return this.rotations; // todo: consider returning a copy or immutable view
+ }
+
+ /** Returns the zone in which this is currently running */
+ public Zone zone() { return zone; }
+
+ public QueryProfiles getQueryProfiles() { return queryProfiles; }
+
+ public SemanticRules getSemanticRules() { return semanticRules; }
+
+ public static class Builder {
+
+ private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty();
+ private FileRegistry fileRegistry = new MockFileRegistry();
+ private DeployLogger logger = new BaseDeployLogger();
+ private Optional<HostProvisioner> hostProvisioner = Optional.empty();
+ private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty();
+ private DeployProperties properties = new DeployProperties.Builder().build();
+ private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty();
+ private Optional<Model> previousModel = Optional.empty();
+ private Set<Rotation> rotations = new HashSet<>();
+ private Zone zone = Zone.defaultZone();
+ private Instant now = Instant.now();
+
+ public Builder applicationPackage(ApplicationPackage applicationPackage) {
+ this.applicationPackage = applicationPackage;
+ return this;
+ }
+
+ public Builder fileRegistry(FileRegistry fileRegistry) {
+ this.fileRegistry = fileRegistry;
+ return this;
+ }
+
+ public Builder deployLogger(DeployLogger logger) {
+ this.logger = logger;
+ return this;
+ }
+
+ public Builder modelHostProvisioner(HostProvisioner modelProvisioner) {
+ this.hostProvisioner = Optional.of(modelProvisioner);
+ return this;
+ }
+
+ public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) {
+ this.permanentApplicationPackage = permanentApplicationPackage;
+ return this;
+ }
+
+ public Builder properties(DeployProperties properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ public Builder configDefinitionRepo(ConfigDefinitionRepo configDefinitionRepo) {
+ this.configDefinitionRepo = Optional.of(configDefinitionRepo);
+ return this;
+ }
+
+ public Builder previousModel(Model previousModel) {
+ this.previousModel = Optional.of(previousModel);
+ return this;
+ }
+
+ public Builder rotations(Set<Rotation> rotations) {
+ this.rotations = rotations;
+ return this;
+ }
+
+ public Builder zone(Zone zone) {
+ this.zone = zone;
+ return this;
+ }
+
+ public Builder now(Instant now) {
+ this.now = now;
+ return this;
+ }
+
+ public DeployState build() {
+ RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+ QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage);
+ SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage);
+ SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles);
+ return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner,
+ properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now);
+ }
+
+ private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) {
+ Collection<NamedReader> readers = applicationPackage.getSearchDefinitions();
+ Map<String, String> names = new LinkedHashMap<>();
+ SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry);
+ for (NamedReader reader : readers) {
+ try {
+ String readerName = reader.getName();
+ String searchName = builder.importReader(reader, readerName, logger);
+ String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX);
+ names.put(searchName, sdName);
+ if (!sdName.equals(searchName)) {
+ throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " +
+ "search element ('" + searchName + "') are not equal for file '" + readerName + "'");
+ }
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not read search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+ } finally {
+ closeIgnoreException(reader.getReader());
+ }
+ }
+ builder.build(logger, queryProfiles);
+ return SearchDocumentModel.fromBuilderAndNames(builder, names);
+ }
+
+ private String getSearchDefinitionRelativePath(String name) {
+ return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name;
+ }
+
+ private static String stripSuffix(String nodeName, String postfix) {
+ assert (nodeName.endsWith(postfix));
+ return nodeName.substring(0, nodeName.length() - postfix.length());
+ }
+
+ @SuppressWarnings("EmptyCatchBlock")
+ private static void closeIgnoreException(Reader reader) {
+ try {
+ reader.close();
+ } catch(Exception e) {}
+ }
+ }
+
+}
+
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
new file mode 100644
index 00000000000..1fbd72599fe
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.search.SearchDefinition;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Internal helper class to retrieve document model and search definitions.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class SearchDocumentModel {
+
+ private final DocumentModel documentModel;
+ private final List<SearchDefinition> searchDefinitions;
+
+ public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) {
+ this.documentModel = documentModel;
+ this.searchDefinitions = searchDefinitions;
+
+ }
+
+ public DocumentModel getDocumentModel() {
+ return documentModel;
+ }
+
+ public List<SearchDefinition> getSearchDefinitions() {
+ return searchDefinitions;
+ }
+
+ public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) {
+ List<SearchDefinition> ret = new ArrayList<>();
+ for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
+ ret.add(new SearchDefinition(names.get(search.getName()), search));
+ }
+ return new SearchDocumentModel(builder.getModel(), ret);
+ }
+
+ public static SearchDocumentModel fromBuilder(SearchBuilder builder) {
+ List<SearchDefinition> ret = new ArrayList<>();
+ for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
+ ret.add(new SearchDefinition(search.getName(), search));
+ }
+ return new SearchDocumentModel(builder.getModel(), ret);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java
new file mode 100644
index 00000000000..3feb934615f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java
new file mode 100644
index 00000000000..a435733e60f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.graph;
+
+import com.yahoo.component.ComponentId;
+
+import java.util.*;
+
+/**
+ * A model graph contains the dependency graph of config models.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ModelGraph {
+
+ private final List<ModelNode> modelNodes;
+ private final List<ModelNode> roots;
+ public ModelGraph(List<ModelNode> modelNodes, List<ModelNode> roots) {
+ this.modelNodes = modelNodes;
+ this.roots = roots;
+ }
+
+ /**
+ * Performs a topological sort ot the models stored in this graph. The algorithm is based on Kahn topological sort.
+ *
+ * @return a sorted list of {@link com.yahoo.config.model.graph.ModelNode} in dependency order.
+ */
+ public List<ModelNode> topologicalSort() {
+ DependencyMap dependencyMap = new DependencyMap(modelNodes);
+ Queue<ModelNode> unprocessed = new LinkedList<>();
+ unprocessed.addAll(roots);
+ List<ModelNode> sortedList = new ArrayList<>();
+ while (!unprocessed.isEmpty()) {
+ ModelNode sortedNode = unprocessed.remove();
+ sortedList.add(sortedNode);
+ for (ModelNode node : modelNodes) {
+ if (dependencyMap.dependsOn(node, sortedNode)) {
+ dependencyMap.removeDependency(node, sortedNode);
+ if (!dependencyMap.hasDependencies(node)) {
+ unprocessed.add(node);
+ }
+ }
+ }
+ }
+ for (ModelNode node : modelNodes) {
+ if (dependencyMap.hasDependencies(node)) {
+ throw new IllegalArgumentException("Unable to sort graph because it contains cycles");
+ }
+ }
+ return sortedList;
+ }
+
+ List<ModelNode> getNodes() {
+ return modelNodes;
+ }
+
+ private static class DependencyMap {
+ private final Map<ComponentId, Set<ComponentId>> map = new LinkedHashMap<>();
+ DependencyMap(List<ModelNode> modelNodes) {
+ for (ModelNode node : modelNodes) {
+ Set<ComponentId> ids = new LinkedHashSet<>();
+ ids.addAll(node.listDependencyIds());
+ map.put(node.id, ids);
+ }
+ }
+
+ public boolean dependsOn(ModelNode node, ModelNode sortedNode) {
+ return map.get(node.id).contains(sortedNode.id);
+ }
+
+ public void removeDependency(ModelNode node, ModelNode sortedNode) {
+ map.get(node.id).remove(sortedNode.id);
+ }
+
+ public boolean hasDependencies(ModelNode node) {
+ return !map.get(node.id).isEmpty();
+ }
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java
new file mode 100644
index 00000000000..97d914c6012
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.graph;
+
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Used to add builders and elements in addBuilder, and then build a dependency graph based on the
+ * constructor arguments.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ModelGraphBuilder {
+
+ private final List<ConfigModelBuilder<? extends ConfigModel>> builders = new ArrayList<>();
+
+ /**
+ * Add a {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to this graph.
+ *
+ * @param builder The {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to add.
+ * @return this for convenience
+ */
+ public ModelGraphBuilder addBuilder(ConfigModelBuilder<? extends ConfigModel> builder) {
+ builders.add(builder);
+ return this;
+ }
+
+ /**
+ * Build a {@link com.yahoo.config.model.graph.ModelGraph} based on the {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder}s
+ * added to this.
+ *
+ * @return A {@link com.yahoo.config.model.graph.ModelGraph} representing the dependency graph.
+ */
+ public ModelGraph build() {
+ List<ModelNode> modelNodes = new ArrayList<>();
+ for (ConfigModelBuilder<? extends ConfigModel> builder : builders) {
+ modelNodes.add(new ModelNode(builder));
+ }
+ List<ModelNode> roots = new ArrayList<>();
+ for (ModelNode modelNode : modelNodes) {
+ int numDependencies = modelNode.addDependenciesFrom(modelNodes);
+ if (numDependencies == 0) {
+ roots.add(modelNode);
+ }
+ }
+ return new ModelGraph(modelNodes, roots);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java
new file mode 100644
index 00000000000..0cd045b09c8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.graph;
+
+import com.google.inject.Inject;
+import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.ConfigModelInstanceFactory;
+
+import java.lang.reflect.*;
+import java.util.*;
+
+/**
+ * Represents a node in the dependency graph, and contains information about a builders dependencies.
+ * Constructor signatures of model classes must have ConfigModelContext as the first argument and
+ * ConfigModel subclasses or Collection of ConfigModels as subsequent arguments.
+ * Only Collection, not Collection subtypes can be used.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ModelNode<MODEL extends ConfigModel> implements ConfigModelInstanceFactory<MODEL> {
+
+ final ComponentId id;
+ public final ConfigModelBuilder<MODEL> builder;
+ final Class<MODEL> clazz;
+ final Constructor<MODEL> constructor;
+ final List<MODEL> instances = new ArrayList<>();
+ private final Map<ComponentId, ModelNode> dependencies = new HashMap<>();
+
+ public ModelNode(ConfigModelBuilder<MODEL> builder) {
+ this.id = builder.getId();
+ this.builder = builder;
+ this.clazz = builder.getModelClass();
+ this.constructor = findConstructor(clazz);
+ }
+
+ private Constructor<MODEL> findConstructor(Class<MODEL> clazz) {
+ for (Constructor<?> ctor : clazz.getDeclaredConstructors()) {
+ if (ctor.getAnnotation(Inject.class) != null) {
+ return (Constructor<MODEL>) ctor;
+ }
+ }
+ return (Constructor<MODEL>) clazz.getDeclaredConstructors()[0];
+ }
+
+ boolean hasDependencies() {
+ return !dependencies.isEmpty();
+ }
+
+ boolean dependsOn(ModelNode node) {
+ return dependencies.containsKey(node.id);
+ }
+
+ /**
+ * This adds dependencies base on constructor arguments in the model classes themselves.
+ * These then have to be created by this mini-di framework and then handed to the builders
+ * that will fill them.
+ *
+ * TODO: This should be changed to model dependencies between model builders instead, such
+ * that they can create their model objects, and eventually make them immutable.
+ */
+ int addDependenciesFrom(List<ModelNode> modelNodes) {
+ int numDependencies = 0;
+ for (Type param : constructor.getGenericParameterTypes()) {
+ for (ModelNode node : modelNodes) {
+ if (param.equals(node.clazz) || isCollectionOf(param, node.clazz)) {
+ addDependency(node);
+ numDependencies++;
+ }
+ }
+ }
+ return numDependencies;
+ }
+
+ private boolean isCollectionOf(Type type, Class<?> nodeClazz) {
+ if (type instanceof ParameterizedType) {
+ ParameterizedType t = (ParameterizedType) type;
+ // Note: IntelliJ says the following cannot be equal but that is wrong
+ return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1 && t.getActualTypeArguments()[0].equals(nodeClazz));
+ }
+ return false;
+ }
+
+ private boolean isCollection(Type type) {
+ if (type instanceof ParameterizedType) {
+ ParameterizedType t = (ParameterizedType) type;
+ // Note: IntelliJ says the following cannot be equal but that is wrong
+ return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1);
+ }
+ return false;
+ }
+
+ private void addDependency(ModelNode node) {
+ dependencies.put(node.id, node);
+ }
+
+ Collection<ComponentId> listDependencyIds() {
+ return dependencies.keySet();
+ }
+
+ @Override
+ public MODEL createModel(ConfigModelContext context) {
+ try {
+ Type [] params = constructor.getGenericParameterTypes();
+ if (params.length < 1 || ! params[0].equals(ConfigModelContext.class)) {
+ throw new IllegalArgumentException("Constructor for " + clazz.getName() + " must have as its first argument a " + ConfigModelContext.class.getName());
+ }
+ Object arguments[] = new Object[params.length];
+ arguments[0] = context;
+ for (int i = 1; i < params.length; i++)
+ arguments[i] = findArgument(params[i]);
+ MODEL instance = constructor.newInstance(arguments);
+ instances.add(instance);
+ return instance;
+ } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Error constructing model '" + clazz.getName() + "'", e);
+ }
+ }
+
+ private Object findArgument(Type param) {
+ for (ModelNode dependency : dependencies.values()) {
+ if (param.equals(dependency.clazz))
+ return dependency.instances.get(0);
+ if (isCollectionOf(param, dependency.clazz))
+ return Collections.unmodifiableCollection(dependency.instances);
+ }
+ // For collections, we don't require that dependency has been added, we just give an empty collection
+ if (isCollection(param))
+ return Collections.emptyList();
+ throw new IllegalArgumentException("Unable to find constructor argument " + param + " for " + clazz.getName());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/package-info.java b/config-model/src/main/java/com/yahoo/config/model/package-info.java
new file mode 100644
index 00000000000..594f52c0d14
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
new file mode 100644
index 00000000000..41927bc09a9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
@@ -0,0 +1,448 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.producer;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.config.*;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.admin.MonitoringSystem;
+import com.yahoo.vespa.model.utils.FreezableMap;
+
+import java.io.*;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * Superclass for all config producers.
+ * Config producers constructs and returns config instances on request.
+ *
+ * @author gjoranv
+ */
+public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProducer<?>>
+ implements ConfigProducer, ConfigInstance.Producer, Serializable {
+
+ private static final long serialVersionUID = 1L;
+ public static final Logger log = Logger.getLogger(AbstractConfigProducer.class.getPackage().toString());
+ private final String subId;
+ private final boolean hostedVespa;
+ private String configId = null;
+
+ private List<Service> descendantServices = new ArrayList<>();
+
+ private AbstractConfigProducer parent = null;
+
+ private UserConfigRepo userConfigs = new UserConfigRepo();
+
+ private final FreezableMap<String, CHILD> childrenBySubId = new FreezableMap<>(LinkedHashMap.class);
+
+ private static boolean isHostedVespa(AbstractConfigProducer parent) {
+ return (parent != null)
+ && (parent.getRoot() != null)
+ && (parent.getRoot().getDeployState() != null)
+ && parent.getRoot().getDeployState().isHostedVespa();
+ }
+
+ /**
+ * Creates a new AbstractConfigProducer with the given parent and subId.
+ * This constructor will add the resulting producer to the children of parent.
+ *
+ * @param parent The parent of this ConfigProducer
+ * @param subId The fragment of the config id for the producer
+ */
+ public AbstractConfigProducer(AbstractConfigProducer parent, String subId) {
+ this(subId, isHostedVespa(parent));
+
+ if (parent != null) {
+ parent.addChild(this);
+ }
+ }
+
+ protected final void setParent(AbstractConfigProducer parent) { this.parent = parent; }
+ public final String getSubId() { return subId; }
+ public final boolean isHostedVespa() { return hostedVespa; }
+
+ /**
+ * Create an config producer with a configId only. Used e.g. to create root nodes, and producers
+ * that are given children after construction using {@link #addChild(AbstractConfigProducer)}.
+ *
+ * @param subId The sub configId. Note that this can be prefixed when calling addChild with this producer as arg.
+ */
+ public AbstractConfigProducer(String subId) {
+ this(subId, false);
+ }
+
+ private AbstractConfigProducer(String subId, boolean hostedVespa) {
+ if (subId.indexOf('/') != -1) {
+ throw new IllegalArgumentException("A subId might not contain '/' : '" + subId + "'");
+ }
+ this.subId = subId;
+ this.hostedVespa = hostedVespa;
+ }
+
+ /**
+ * Adds a child to this config producer.
+ *
+ * @param child The child config producer to add.
+ */
+ protected void addChild(CHILD child) {
+ if (child == null) {
+ throw new IllegalArgumentException("Trying to add null child for: " + this);
+ }
+ if (child instanceof AbstractConfigProducerRoot) {
+ throw new IllegalArgumentException("Child cannot be a root node: " + child);
+ }
+
+ child.setParent(this);
+ if (childrenBySubId.get(child.getSubId()) != null) {
+ throw new IllegalArgumentException("Multiple services/instances of the id '" + child.getSubId() + "' under the service/instance " +
+ errorMsgClassName() + " '" + subId + "'. (This is commonly caused by service/node index " +
+ "collisions in the config.)." +
+ "\nExisting instance: " + childrenBySubId.get(child.getSubId()) +
+ "\nAttempted to add: " + child);
+ }
+ childrenBySubId.put(child.getSubId(), child);
+
+ if (child instanceof Service) {
+ addDescendantService((Service)child);
+ }
+ }
+
+ public void removeChild(CHILD child) {
+ if (child.getParent() != this)
+ throw new IllegalArgumentException("Could not remove " + child + ": Expected its parent to be " +
+ this + ", but was " + child.getParent());
+
+ if (child instanceof Service)
+ descendantServices.remove(child);
+
+ childrenBySubId.remove(child.getSubId());
+ child.setParent(null);
+ }
+
+ /**
+ * Helper to provide an error message on collisions of sub ids (ignore SimpleConfigProducer, use the parent in that case)
+ */
+ private String errorMsgClassName() {
+ if (getClass().equals(SimpleConfigProducer.class)) return parent.getClass().getSimpleName();
+ return getClass().getSimpleName();
+ }
+
+ /**
+ * Sets the user configs for this producer.
+ *
+ * @param repo User configs repo.
+ */
+ public void setUserConfigs(UserConfigRepo repo) { this.userConfigs = repo; }
+
+ /** Returns the user configs of this */
+ @Override
+ public UserConfigRepo getUserConfigs() { return userConfigs; }
+
+ /**
+ * ConfigProducers that must have a special config id should use
+ * setConfigId() instead of overloading this method. This is
+ * because config IDs must be registered through setConfigId().
+ */
+ public final String getConfigId() {
+ if (configId == null) throw new RuntimeException("The system topology must be frozen first.");
+ return configId;
+ }
+
+ /**
+ * Sets the config id for this producer. Will also add this
+ * service to the root node, so the new config id will be picked
+ * up. Note that this producer will be known with both the old
+ * and the new config id in the root node after using this method.
+ */
+ protected void addConfigId(String id) {
+ if (id == null) throw new NullPointerException("Config ID cannot be null.");
+ getRoot().addDescendant(id, this);
+ if (!isVespa() && (getVespa() != null))
+ getVespa().addDescendant(this);
+ }
+
+ /** Returns this ConfigProducer's children (only 1st level) */
+ public Map<String, CHILD> getChildren() { return Collections.unmodifiableMap(childrenBySubId); }
+
+ @Beta
+ public <J extends AbstractConfigProducer<?>> List<J> getChildrenByTypeRecursive(Class<J> type) {
+ List<J> validChildren = new ArrayList<>();
+
+ if (this.getClass().equals(type)) {
+ validChildren.add(type.cast(this));
+ }
+
+ Map<String, ? extends AbstractConfigProducer<?>> children = this.getChildren();
+ for (AbstractConfigProducer<?> child : children.values()) {
+ validChildren.addAll(child.getChildrenByTypeRecursive(type));
+ }
+
+ return Collections.unmodifiableList(validChildren);
+ }
+
+ /** Returns a list of all the children of this who are instances of Service */
+ public List<Service> getDescendantServices() { return Collections.unmodifiableList(descendantServices); }
+
+ protected void addDescendantService(Service s) { descendantServices.add(s); }
+
+ @Override
+ public final boolean cascadeConfig(ConfigInstance.Builder builder) {
+ boolean found=false;
+ if (parent != null)
+ found = parent.cascadeConfig(builder);
+
+ boolean foundHere = builder.dispatchGetConfig(this);
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "cascadeconfig in " + this + ", getting config " +
+ builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + "' found here=" + foundHere);
+ }
+ found = found || foundHere;
+ return found;
+ }
+
+ @Override
+ public final boolean addUserConfig(ConfigInstance.Builder builder) {
+ boolean didApply = false;
+ if (parent != null) {
+ didApply = parent.addUserConfig(builder);
+ }
+
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "User configs is: " + userConfigs.toString());
+ }
+ // TODO: What do we do with md5. Currently ignored for user configs?
+ ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace());
+ if (userConfigs.get(key) != null) {
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "Apply in " + configId);
+ }
+ applyUserConfig(builder, userConfigs.get(key));
+ didApply = true;
+ }
+ return didApply;
+ }
+
+ private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) {
+ ConfigInstance.Builder override;
+ if (builder instanceof GenericConfig.GenericConfigBuilder) {
+ // Means that the builder is unknown and that we should try to apply the payload without
+ // the real builder
+ override = getGenericConfigBuilderOverride((GenericConfig.GenericConfigBuilder) builder, payloadBuilder);
+ } else {
+ override = getConfigInstanceBuilderOverride(builder, ConfigPayload.fromBuilder(payloadBuilder));
+ }
+ ConfigInstanceUtil.setValues(builder, override);
+ }
+
+ private ConfigInstance.Builder getGenericConfigBuilderOverride(GenericConfig.GenericConfigBuilder builder, ConfigPayloadBuilder payloadBuilder) {
+ ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace());
+ return new GenericConfig.GenericConfigBuilder(key, payloadBuilder);
+ }
+
+ private ConfigInstance.Builder getConfigInstanceBuilderOverride(ConfigInstance.Builder builder, ConfigPayload payload) {
+ try {
+ ConfigTransformer transformer = new ConfigTransformer(builder.getClass().getEnclosingClass());
+ return transformer.toConfigBuilder(payload);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Error applying override to builder", e);
+ }
+ }
+
+ /**
+ * Returns the one and only HostSystem of the root node
+ * Must be overridden by root node.
+ */
+ public HostSystem getHostSystem() {
+ return getRoot().getHostSystem();
+ }
+
+ public AbstractConfigProducerRoot getRoot() {
+ return parent == null ? null : parent.getRoot();
+ }
+
+ /**
+ * Returns the {@link ApplicationConfigProducerRoot} that is the parent of this sub-tree, or null
+ * if this sub-tree has no Vespa parent.
+ */
+ private ApplicationConfigProducerRoot getVespa() {
+ if (isRoot()) return null;
+ return isVespa() ? (ApplicationConfigProducerRoot)this : parent.getVespa();
+ }
+
+ private boolean isRoot() {
+ return parent == null;
+ }
+
+ private boolean isVespa() {
+ return ((this instanceof ApplicationConfigProducerRoot) && parent.isRoot());
+ }
+
+ public AbstractConfigProducer getParent() { return parent; }
+
+ /**
+ * Writes files that need to be written. The files will usually only be
+ * written when the Vespa model is generated through the deploy-application
+ * script.
+ *
+ * TODO: Make sure all implemented ConfigProducers call createConfig()
+ * instead of getConfig() when implementing this method.
+ */
+ public void writeFiles(File directory) throws java.io.IOException {
+ if (!directory.isDirectory() && !directory.mkdirs()) {
+ throw new java.io.IOException("Cannot create directory: "+ directory);
+ }
+ for (Method m : getClass().getMethods()) {
+ try {
+ ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m);
+ if (builder!=null) {
+ writeBuilder(directory, m, builder);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private void writeBuilder(File directory, Method m,
+ ConfigInstance.Builder builder) throws IllegalAccessException,
+ InvocationTargetException, InstantiationException,
+ NoSuchMethodException, IOException {
+ m.invoke(this, builder);
+ Class<?> configInstClass = builder.getClass().getEnclosingClass();
+ ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder);
+ List<String> payloadList = ConfigInstance.serialize(inst);
+ File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg");
+ FileOutputStream out = new FileOutputStream(outfn);
+ for (String s : payloadList) {
+ out.write(Utf8.toBytes(s));
+ out.write('\n');
+ }
+ }
+
+ /**
+ * New Builder instance if m is getConfig(SomeConfig.Builder), or null
+ */
+ private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws InstantiationException, IllegalAccessException {
+ if (!"getConfig".equals(m.getName())) return null;
+ Type[] params = m.getParameterTypes();
+ if (params.length!=1) return null;
+ Type param = params[0];
+ if (!(param instanceof Class)) return null;
+ Class<?> paramClass = (Class<?>) param;
+ if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null;
+ return (ConfigInstance.Builder) paramClass.newInstance();
+ }
+
+ public void dump(PrintStream out) {
+ for (ConfigProducer c : getChildren().values()) {
+ out.println("id: " + c.getConfigId());
+ if (c.getChildren().size() > 0) {
+ c.dump(out);
+ }
+ }
+ }
+
+ void setupConfigId(String parentConfigId) {
+ if (this instanceof AbstractConfigProducerRoot) {
+ configId = "";
+ } else {
+ configId = parentConfigId + subId;
+ addConfigId(configId);
+ }
+
+ if (this instanceof AbstractConfigProducerRoot || this instanceof ApplicationConfigProducerRoot) {
+ setupChildConfigIds("");
+ } else {
+ setupChildConfigIds(configId + '/');
+ }
+ }
+
+ private static ClassLoader findInheritedClassLoader(Class clazz, String producerName) {
+ Class<?>[] interfazes = clazz.getInterfaces();
+ for (Class interfaze : interfazes) {
+ if (producerName.equals(interfaze.getName())) {
+ return interfaze.getClassLoader();
+ }
+ }
+ if (clazz.getSuperclass() == null)
+ return null;
+ return findInheritedClassLoader(clazz.getSuperclass(), producerName);
+ }
+
+ public ClassLoader getConfigClassLoader(String producerName) {
+ ClassLoader classLoader = findInheritedClassLoader(getClass(), producerName);
+ if (classLoader != null)
+ return classLoader;
+
+ // TODO: Make logic correct, so that the deepest child will be the one winning.
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ ClassLoader loader = child.getConfigClassLoader(producerName);
+ if (loader != null) {
+ return loader;
+ }
+ }
+ return null;
+ }
+
+ private void setupChildConfigIds(String currentConfigId) {
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.setupConfigId(currentConfigId);
+ }
+ }
+
+ void aggregateDescendantServices() {
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.aggregateDescendantServices();
+ descendantServices.addAll(child.descendantServices);
+ }
+ }
+
+ void freeze() {
+ childrenBySubId.freeze();
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.freeze();
+ }
+ }
+
+ public void mergeUserConfigs(UserConfigRepo newRepo) {
+ userConfigs.merge(newRepo);
+ }
+
+ @Override
+ public void validate() throws Exception {
+ assert (childrenBySubId.isFrozen());
+
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.validate();
+ }
+ }
+
+ /** Returns a logger to be used for warnings and messages during initialization, never null */
+ public DeployLogger deployLogger() {
+ return parent.deployLogger();
+ }
+
+ // TODO: Make producers depend on AdminModel instead
+ /** Returns a monitoring service (yamas if that is configured, null otherwise) */
+ protected MonitoringSystem getMonitoringService() {
+ AbstractConfigProducerRoot root = getRoot();
+ Admin admin = (root == null? null : root.getAdmin());
+ if (admin == null) {
+ return null;
+ }
+ if (admin.getYamas() != null) {
+ return admin.getYamas();
+ }
+ return null;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java
new file mode 100644
index 00000000000..b85e2b8fa97
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.producer;
+
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.vespa.model.ConfigProducer;
+import com.yahoo.vespa.model.ConfigProducerRoot;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The parent class of classes having the role as the root of a config producer tree.
+ *
+ * @author tonytv
+ */
+public abstract class AbstractConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>>
+ implements ConfigProducerRoot {
+
+ /** The ConfigProducers contained in this model indexed by config id */
+ protected final Map<String, ConfigProducer> id2producer = new LinkedHashMap<>();
+
+ public AbstractConfigProducerRoot(String rootConfigId) {
+ super(rootConfigId);
+ }
+
+ public AbstractConfigProducerRoot getRoot() {
+ return this;
+ }
+
+ public abstract FileDistributionConfigProducer getFileDistributionConfigProducer();
+
+ /**
+ * Freezes the parent - child connections of the model
+ * and sets information derived from the topology.
+ */
+ public void freezeModelTopology() {
+ freeze();
+ setupConfigId("");
+ aggregateDescendantServices();
+ }
+
+ public abstract ConfigModelRepo configModelRepo();
+
+ /**
+ * Returns the ConfigProducer with the given id if such configId exists.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return ConfigProducer with the given configId
+ */
+ public Optional<ConfigProducer> getConfigProducer(String configId) {
+ return Optional.ofNullable(id2producer.get(configId));
+ }
+
+ /**
+ * Returns the Service with the given id if such configId exists and it belongs to a Service ConfigProducer.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return Service with the given configId
+ */
+ public Optional<Service> getService(String configId) {
+ return getConfigProducer(configId)
+ .filter(Service.class::isInstance)
+ .map(Service.class::cast);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java
new file mode 100644
index 00000000000..d32b0c75e69
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.producer;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.*;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A UserConfigRepo is a repository for user configs, typically for a particular config producer. The repo encapsulates
+ * how the user configs are stored, and defines the methods to retrieve user configs and merge the repo with others.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class UserConfigRepo {
+ private final Map<ConfigDefinitionKey, ConfigPayloadBuilder> userConfigsMap;
+
+ public UserConfigRepo() {
+ this.userConfigsMap = new LinkedHashMap<>();
+ }
+
+ @Override
+ public UserConfigRepo clone() {
+ return new UserConfigRepo(copyBuilders(userConfigsMap));
+ }
+
+ /**
+ * Must copy the builder, because the merge method on {@link AbstractConfigProducer} might override the row's builders otherwise
+ */
+ private Map<ConfigDefinitionKey, ConfigPayloadBuilder> copyBuilders(Map<ConfigDefinitionKey, ConfigPayloadBuilder> source) {
+ Map<ConfigDefinitionKey, ConfigPayloadBuilder> ret = new LinkedHashMap<>();
+ for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> e : source.entrySet()) {
+ ConfigDefinitionKey key = e.getKey();
+ ConfigPayloadBuilder sourceVal = e.getValue();
+ ConfigPayloadBuilder destVal = new ConfigPayloadBuilder(ConfigPayload.fromBuilder(sourceVal));
+ ret.put(key, destVal);
+ }
+ return ret;
+ }
+
+ public UserConfigRepo(Map<ConfigDefinitionKey, ConfigPayloadBuilder> map) {
+ this.userConfigsMap = map;
+ }
+
+ public UserConfigRepo(UserConfigRepo userConfigRepo) {
+ this.userConfigsMap = userConfigRepo.userConfigsMap;
+ }
+
+ public ConfigPayloadBuilder get(ConfigDefinitionKey key) {
+ return userConfigsMap.get(key);
+ }
+
+ public void merge(UserConfigRepo newRepo) {
+ for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> entry : newRepo.userConfigsMap.entrySet()) {
+ if (entry.getValue() == null) continue;
+
+ ConfigDefinitionKey key = entry.getKey();
+ if (userConfigsMap.containsKey(key)) {
+ ConfigPayloadBuilder lhsBuilder = userConfigsMap.get(key);
+ ConfigPayloadBuilder rhsBuilder = entry.getValue();
+ lhsBuilder.override(rhsBuilder);
+ } else {
+ userConfigsMap.put(key, entry.getValue());
+ }
+ }
+ }
+
+ public boolean isEmpty() {
+ return userConfigsMap.isEmpty();
+ }
+
+ public int size() {
+ return userConfigsMap.size();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (ConfigDefinitionKey key : userConfigsMap.keySet()) {
+ sb.append(key.toString());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * The keys of all the configs contained in this.
+ * @return a set of ConfigDefinitionsKey
+ */
+ public Set<ConfigDefinitionKey> configsProduced() {
+ return userConfigsMap.keySet();
+ }
+
+ /**
+ * Will take the warning messages stored on the payload builders, and apply them to the producer's {@link DeployLogger}
+ * @param producer the producer to apply warnings to
+ */
+ public void applyWarnings(AbstractConfigProducer<?> producer) {
+ for (ConfigPayloadBuilder b : userConfigsMap.values()) {
+ for (String warning : b.warnings()) {
+ producer.deployLogger().log(LogLevel.WARNING, warning);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java
new file mode 100644
index 00000000000..c58a116b196
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.producer;
+
+import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java
new file mode 100644
index 00000000000..9d97308fc76
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A hostname with zero or more aliases.
+ *
+ * @author musum
+ */
+public class Host {
+
+ private final String hostname;
+ private final List<String> hostAliases;
+
+ public Host(String hostname) {
+ this.hostname = hostname;
+ this.hostAliases = new ArrayList<>();
+ }
+
+ public Host(String hostname, List<String> hostAliases) {
+ this.hostname = hostname;
+ this.hostAliases = hostAliases;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public List<String> getHostAliases() {
+ return hostAliases;
+ }
+
+ @Override
+ public String toString() {
+ return hostname + " (aliases: " + hostAliases + ")";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
new file mode 100644
index 00000000000..ece781dd5bd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.log.LogLevel;
+import com.yahoo.net.HostName;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * TODO: What is this?
+ *
+ * @author musum
+ */
+public class Hosts {
+
+ public static final Logger log = Logger.getLogger(Hosts.class.getPackage().toString());
+
+ private final HashMap<String, Host> hosts = new LinkedHashMap<>();
+ private final Map<String, String> alias2hostname = new LinkedHashMap<>();
+ private final Map<String, Host> alias2host = new LinkedHashMap<>();
+
+ /**
+ * Builds host system from a hosts.xml file
+ *
+ * @param hostsFile a reader for host from application package
+ * @return the HostSystem for this application package
+ */
+ public static Hosts getHosts(Reader hostsFile) {
+ Hosts hosts = new Hosts();
+ Document doc;
+ try {
+ doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile));
+ } catch (SAXException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) {
+ String name = hostE.getAttribute("name");
+ if (name.equals("")) {
+ throw new RuntimeException("Missing 'name' attribute for host.");
+ }
+ if ("localhost".equals(name)) {
+ name = HostName.getLocalhost();
+ }
+ final List<String> hostAliases = VespaDomBuilder.getHostAliases(hostE.getChildNodes());
+ if (hostAliases.isEmpty()) {
+ throw new IllegalArgumentException("No host aliases defined for host '" + name + "'");
+ }
+ Host host = new Host(name, hostAliases);
+ hosts.addHost(host, hostAliases);
+ }
+ log.log(LogLevel.DEBUG, "Created hosts:" + hosts);
+ return hosts;
+ }
+
+ public Collection<Host> getHosts() {
+ return hosts.values();
+ }
+
+ /**
+ * Adds one host to this host system.
+ *
+ * @param host The host to add
+ * @param aliases The aliases for this host.
+ */
+ public void addHost(Host host, List<String> aliases) {
+ hosts.put(host.getHostname(), host);
+ if ((aliases != null) && (aliases.size() > 0)) {
+ addHostAliases(aliases, host);
+ }
+ }
+
+ /**
+ * Add all aliases for one host
+ *
+ * @param hostAliases A list of host aliases
+ * @param host The Host instance to add the alias for
+ */
+ private void addHostAliases(List<String> hostAliases, Host host) {
+ if (hostAliases.size() < 1) {
+ throw new RuntimeException("Host '" + host.getHostname() + "' must have at least one <alias> tag.");
+ }
+ for (String alias : hostAliases) {
+ addHostAlias(alias, host);
+ }
+ }
+
+ /**
+ * Adds an alias for the given host
+ *
+ * @param alias alias (string) for a Host
+ * @param host the {@link Host} to add the alias for
+ */
+ protected void addHostAlias(String alias, Host host) {
+ if (alias2hostname.containsKey(alias)) {
+ throw new RuntimeException("Alias '" + alias + "' must be used for only one host!");
+ }
+ alias2hostname.put(alias, host.getHostname());
+ alias2host.put(alias, host);
+ }
+
+ public Map<String, Host> getAlias2host() {
+ return alias2host;
+ }
+
+ @Override
+ public String toString() {
+ return "Hosts: " + hosts.keySet();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
new file mode 100644
index 00000000000..20dc190d8e7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.provision.*;
+import com.yahoo.vespa.model.container.Container;
+
+import java.io.Reader;
+import java.util.List;
+
+/**
+ * A host provisioner based on a hosts.xml file.
+ * No state in this provisioner, i.e it does not know anything about the active
+ * application if one exists. Pre-condition: A valid hosts file.
+ *
+ * @author musum
+ * @since 5.11
+ */
+public class HostsXmlProvisioner implements HostProvisioner {
+
+ private final Hosts hosts;
+ public static final String IMPLICIT_ADMIN_HOSTALIAS = "INTERNAL_VESPA_IMPLICIT_ADMIN";
+
+ public HostsXmlProvisioner(Reader hosts) {
+ this.hosts = Hosts.getHosts(hosts);
+ }
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ /**
+ * Some special rules to allow no admin elements as well
+ * as jdisc element without nodes.
+ */
+ if (alias.equals(IMPLICIT_ADMIN_HOSTALIAS)) {
+ if (hosts.getHosts().size() > 1) {
+ throw new IllegalArgumentException("More than 1 host specified (" + hosts.getHosts().size() + ") and <admin> not specified");
+ } else {
+ return host2HostSpec(getFirstHost());
+ }
+ } else if (alias.equals(Container.SINGLENODE_CONTAINER_SERVICESPEC)) {
+ return host2HostSpec(getFirstHost());
+ }
+ for (Host host : hosts.getHosts()) {
+ if (host.getHostAliases().contains(alias)) {
+ return new HostSpec(host.getHostname(), host.getHostAliases());
+ }
+ }
+ throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'");
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) {
+ throw new UnsupportedOperationException();
+ }
+
+ private HostSpec host2HostSpec(Host host) {
+ return new HostSpec(host.getHostname(), host.getHostAliases());
+ }
+
+ private Host getFirstHost() {
+ return hosts.getHosts().iterator().next();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
new file mode 100644
index 00000000000..c06efece329
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -0,0 +1,164 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.collections.ListMap;
+import com.yahoo.collections.Pair;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.provision.*;
+
+import java.util.*;
+
+/**
+ * In memory host provisioner. NB! ATM cannot be reused after allocate has been called.
+ *
+ * @author musum
+ * @author bratseth
+ */
+public class InMemoryProvisioner implements HostProvisioner {
+
+ /**
+ * If this is true an exception is thrown when all nodes are used.
+ * If false this will simply return nodes best effort, preferring to satisfy the
+ * number of groups requested when possible.
+ */
+ private final boolean failOnOutOfCapacity;
+
+ /** Hosts which should be returned as retired */
+ private final Set<String> retiredHostNames;
+
+ /** Free hosts of each flavor */
+ private final ListMap<String, Host> freeNodes = new ListMap<>();
+ private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>();
+ private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>();
+
+ /** Indexes must be unique across all groups in a cluster */
+ private final Map<Pair<ClusterSpec.Type,ClusterSpec.Id>, Integer> nextIndexInCluster = new HashMap<>();
+
+ /** Use this index as start index for all clusters */
+ private final int startIndexForClusters;
+
+ /** Creates this with a number of nodes of the flavor 'default' */
+ public InMemoryProvisioner(int nodeCount) {
+ this(Collections.singletonMap("default", createHostInstances(nodeCount)), true, 0);
+ }
+
+ /** Creates this with a set of host names of the flavor 'default' */
+ public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) {
+ this(Collections.singletonMap("default", toHostInstances(hosts)), failOnOutOfCapacity, 0);
+ }
+
+ /** Creates this with a set of hosts of the flavor 'default' */
+ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) {
+ this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, 0, retiredHostNames);
+ }
+
+ /** Creates this with a set of hosts of the flavor 'default' */
+ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
+ this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames);
+ }
+
+ public InMemoryProvisioner(Map<String, Collection<Host>> hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
+ this.failOnOutOfCapacity = failOnOutOfCapacity;
+ for (Map.Entry<String, Collection<Host>> hostsOfFlavor : hosts.entrySet())
+ for (Host host : hostsOfFlavor.getValue())
+ freeNodes.put(hostsOfFlavor.getKey(), host);
+ this.retiredHostNames = new HashSet<>(Arrays.asList(retiredHostNames));
+ this.startIndexForClusters = startIndexForClusters;
+ }
+
+ private static Collection<Host> toHostInstances(String[] hostnames) {
+ List<Host> hosts = new ArrayList<>();
+ for (String hostname : hostnames) {
+ hosts.add(new Host(hostname));
+ }
+ return hosts;
+ }
+
+ private static Collection<Host> createHostInstances(int hostCount) {
+ List<Host> hosts = new ArrayList<>();
+ for (int i = 1; i <= hostCount; i++) {
+ hosts.add(new Host("host" + i));
+ }
+ return hosts;
+ }
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias);
+ List<Host> defaultHosts = freeNodes.get("default");
+ if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts of default flavor available");
+ Host newHost = freeNodes.removeValue("default", 0);
+ HostSpec hostSpec = new HostSpec(newHost.getHostname(), newHost.getHostAliases());
+ legacyMapping.put(alias, hostSpec);
+ return hostSpec;
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) {
+ if (cluster.group().isPresent() && groups > 1)
+ throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
+ if (requestedCapacity.nodeCount() % groups != 0)
+ throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " +
+ groups + " groups, but the node count is not divisible into this number of groups");
+
+ int capacity = failOnOutOfCapacity ? requestedCapacity.nodeCount() :
+ Math.min(requestedCapacity.nodeCount(), freeNodes.get("default").size() + totalAllocatedTo(cluster));
+ if (groups > capacity)
+ groups = capacity;
+
+ String flavor = requestedCapacity.flavor().orElse("default");
+
+ List<HostSpec> allocation = new ArrayList<>();
+ if (groups == 1) {
+ allocation.addAll(allocateHostGroup(cluster, flavor, capacity, startIndexForClusters));
+ }
+ else {
+ for (int i = 0; i < groups; i++) {
+ allocation.addAll(allocateHostGroup(cluster.changeGroup(Optional.of(ClusterSpec.Group.from(String.valueOf(i)))),
+ flavor,
+ capacity / groups,
+ allocation.size()));
+ }
+ }
+ for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) {
+ HostSpec host = i.next();
+ if (retiredHostNames.contains(host.hostname()))
+ i.set(retire(host));
+ }
+ return allocation;
+ }
+
+ private HostSpec retire(HostSpec host) {
+ return new HostSpec(host.hostname(), host.aliases(), host.membership().get().retire());
+ }
+
+ private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex) {
+ List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>());
+ allocations.put(clusterGroup, allocation);
+
+ int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex);
+ while (allocation.size() < nodesInGroup) {
+ if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("No nodes of flavor '" + flavor + "' available");
+ Host newHost = freeNodes.removeValue(flavor, 0);
+ ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
+ allocation.add(new HostSpec(newHost.getHostname(), newHost.getHostAliases(), membership));
+ }
+ nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
+
+ while (allocation.size() > nodesInGroup)
+ allocation.remove(0);
+
+ return allocation;
+ }
+
+ private int totalAllocatedTo(ClusterSpec cluster) {
+ int count = 0;
+ for (Map.Entry<ClusterSpec, List<HostSpec>> allocation : allocations.entrySet()) {
+ if ( ! allocation.getKey().type().equals(cluster.type())) continue;
+ if ( ! allocation.getKey().id().equals(cluster.id())) continue;
+ count += allocation.getValue().size();
+ }
+ return count;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
new file mode 100644
index 00000000000..38ed728e4e8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.provision.*;
+import com.yahoo.net.HostName;
+import com.yahoo.vespa.model.HostSystem;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.net.UnknownHostException;
+
+/**
+ * A host provisioner used when there is no hosts.xml file (using localhost as the only host)
+ * No state in this provisioner, i.e it does not know anything about the active
+ * application if one exists.
+ *
+ * @author musum
+ * @since 5.11
+ */
+public class SingleNodeProvisioner implements HostProvisioner {
+
+ private final Host host; // the only host in this system
+ private final HostSpec hostSpec;
+ private int counter = 0;
+
+ public SingleNodeProvisioner() {
+ try {
+ host = new Host(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()));
+ } catch (UnknownHostException e) {
+ throw new RuntimeException(e);
+ }
+ this.hostSpec = new HostSpec(host.getHostname(), host.getHostAliases());
+ }
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ return hostSpec;
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1
+ List<HostSpec> hosts = new ArrayList<>();
+ hosts.add(new HostSpec(host.getHostname(), host.getHostAliases(), ClusterMembership.from(cluster, counter++)));
+ return hosts;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java
new file mode 100644
index 00000000000..a39e4025efa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.provision;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java
new file mode 100644
index 00000000000..b75147774ab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.yahoo.collections.CollectionUtil;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class ConfigModelTestUtil {
+ /**
+ * @param xmlLines XML with " replaced with '
+ */
+ public static Element parse(String... xmlLines) {
+ List<String> lines = new ArrayList<>();
+ lines.add("<?xml version='1.0' encoding='utf-8' ?>");
+ lines.addAll(Arrays.asList(xmlLines));
+
+ try {
+ return XmlHelper.getDocumentBuilder().parse(
+ inputSource((CollectionUtil.mkString(lines, "\n").replace("'", "\""))))
+ .getDocumentElement();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static InputSource inputSource(String str) {
+ return new InputSource(new StringReader(str));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
new file mode 100644
index 00000000000..731410c9bf3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
@@ -0,0 +1,243 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.yahoo.config.application.api.ComponentInfo;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.UnparsedConfigDefinition;
+import com.yahoo.config.application.api.ApplicationFile;
+import com.yahoo.config.provision.Version;
+import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.searchdefinition.*;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.config.application.api.ApplicationPackage;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.*;
+
+/**
+ * For testing purposes only
+ *
+ * @author tonytv
+ */
+public class MockApplicationPackage implements ApplicationPackage {
+ public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo");
+ public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar");
+
+ private final String hostsS;
+ private final String servicesS;
+ private final List<String> searchDefinitions;
+ private final String searchDefinitionDir;
+ private final Optional<String> deploymentInfo;
+ private final Optional<String> validationOverrides;
+ private final boolean failOnValidateXml;
+
+ private MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir,
+ String deploymentInfo, String validationOverrides, boolean failOnValidateXml) {
+ this.hostsS = hosts;
+ this.servicesS = services;
+ this.searchDefinitions = searchDefinitions;
+ this.searchDefinitionDir = searchDefinitionDir;
+ this.deploymentInfo = Optional.ofNullable(deploymentInfo);
+ this.validationOverrides = Optional.ofNullable(validationOverrides);
+ this.failOnValidateXml = failOnValidateXml;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "mock application";
+ }
+
+ @Override
+ public Reader getServices() {
+ return new StringReader(servicesS);
+ }
+
+ @Override
+ public Reader getHosts() {
+ if (hostsS==null) return null;
+ return new StringReader(hostsS);
+ }
+
+ @Override
+ public List<NamedReader> getSearchDefinitions() {
+ ArrayList<NamedReader> readers = new ArrayList<>();
+ SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry());
+ for (String sd : searchDefinitions) {
+ try {
+ String name = searchBuilder.importString(sd);
+ readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd)));
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return readers;
+ }
+
+ @Override
+ public List<NamedReader> searchDefinitionContents() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public List<NamedReader> getFiles(Path dir,String fileSuffix,boolean recurse) {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public ApplicationFile getFile(Path file) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getHostSource() {
+ return "mock source";
+ }
+
+ @Override
+ public String getServicesSource() {
+ return "mock source";
+ }
+
+ @Override
+ public Optional<Reader> getDeployment() {
+ return deploymentInfo.map(StringReader::new);
+ }
+
+ @Override
+ public Optional<Reader> getValidationOverrides() {
+ return validationOverrides.map(StringReader::new);
+ }
+
+ public List<ComponentInfo> getComponentsInfo(Version vespaVersion) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Reader getRankingExpression(String name) {
+ File expressionFile = new File(searchDefinitionDir, name);
+ try {
+ return IOUtils.createReader(expressionFile, "utf-8");
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("Could not read ranking expression file '" +
+ expressionFile.getAbsolutePath() + "'", e);
+ }
+ }
+
+ public static ApplicationPackage createEmpty() {
+ return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build();
+ }
+
+ public static ApplicationPackage fromSearchDefinitionDirectory(String dir) {
+ return new MockApplicationPackage.Builder()
+ .withEmptyHosts()
+ .withEmptyServices()
+ .withSearchDefinitionDir(dir).build();
+ }
+
+ public static class Builder {
+ private String hosts = null;
+ private String services = null;
+ private List<String> searchDefinitions = Collections.emptyList();
+ private String searchDefinitionDir = null;
+ private String deploymentInfo = null;
+ private String validationOverrides = null;
+ private boolean failOnValidateXml = false;
+
+ public Builder() {
+ }
+
+ public Builder withEmptyHosts() {
+ return this.withHosts(emptyHosts);
+ }
+
+ public Builder withHosts(String hosts) {
+ this.hosts = hosts;
+ return this;
+ }
+
+ public Builder withEmptyServices() {
+ return this.withServices(emptyServices);
+ }
+
+ public Builder withServices(String services) {
+ this.services = services;
+ return this;
+ }
+
+ public Builder withSearchDefinition(String searchDefinition) {
+ this.searchDefinitions = Collections.singletonList(searchDefinition);
+ return this;
+ }
+
+ public Builder withSearchDefinitions(List<String> searchDefinition) {
+ this.searchDefinitions = Collections.unmodifiableList(searchDefinition);
+ return this;
+ }
+
+ public Builder withSearchDefinitionDir(String searchDefinitionDir) {
+ this.searchDefinitionDir = searchDefinitionDir;
+ return this;
+ }
+
+ public Builder withDeploymentInfo(String deploymentInfo) {
+ this.deploymentInfo = deploymentInfo;
+ return this;
+ }
+
+ public Builder withValidationOverrides(String validationOverrides) {
+ this.validationOverrides = validationOverrides;
+ return this;
+ }
+
+ public Builder failOnValidateXml() {
+ this.failOnValidateXml = true;
+ return this;
+ }
+
+ public ApplicationPackage build() {
+ return new MockApplicationPackage(hosts, services, searchDefinitions, searchDefinitionDir,
+ deploymentInfo, validationOverrides, failOnValidateXml);
+ }
+ }
+
+ public static String createSearchDefinition(String name, String fieldName) {
+ return "search " + name + " {" +
+ " document " + name + " {" +
+ " field " + fieldName + " type string {}" +
+ " }" +
+ "}";
+ }
+
+ private static final String emptyServices = "<services version=\"1.0\">" +
+ " <admin version=\"2.0\">" +
+ " <adminserver hostalias=\"node1\" />" +
+ " </admin>" +
+ "</services>";
+
+ private static final String emptyHosts = "<hosts>" +
+ " <host name=\"localhost\">" +
+ " <alias>node1</alias>" +
+ " </host>" +
+ "</hosts>";
+
+ @Override
+ public void validateXML(DeployLogger logger) throws IOException {
+ if (failOnValidateXml) {
+ throw new IllegalArgumentException("Error in application package");
+ } else {
+ throw new UnsupportedOperationException("This application package cannot validate XML");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
new file mode 100644
index 00000000000..fa84cf1c7eb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.ConfigProducer;
+import com.yahoo.vespa.model.HostSystem;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.filedistribution.FileDistributor;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+
+
+/**
+ * Use for testing. Use as parent for the config producer(s) you want to test, to test
+ * only a subtree of the producers.
+ *
+ * @author gjoranv
+ */
+public class MockRoot extends AbstractConfigProducerRoot {
+ private static final long serialVersionUID = 1L;
+ public static final String MOCKHOST = "mockhost";
+
+ private HostSystem hostSystem;
+
+ private final DeployState deployState;
+ private FileDistributor fileDistributor;
+ private Admin admin;
+
+ public MockRoot() {
+ this("");
+ }
+
+ public MockRoot(String rootConfigId) {
+ this(rootConfigId, new MockApplicationPackage.Builder().build());
+ }
+
+ public MockRoot(String rootConfigId, ApplicationPackage applicationPackage) {
+ this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build());
+ }
+
+ public MockRoot(String rootConfigId, DeployState deployState) {
+ super(rootConfigId);
+ hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner());
+ this.deployState = deployState;
+ fileDistributor = new FileDistributor(deployState.getFileRegistry());
+ }
+
+ public FileDistributionConfigProducer getFileDistributionConfigProducer() {
+ return null;
+ }
+
+ @Override
+ public ConfigModelRepo configModelRepo() {
+ return new ConfigModelRepo();
+ }
+
+ public Set<String> getConfigIds() {
+ return Collections.unmodifiableSet(id2producer.keySet());
+ }
+
+ @Override
+ public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) {
+ ConfigProducer cp = id2producer.get(configId);
+ if (cp == null) return null;
+
+ cp.cascadeConfig(builder);
+ cp.addUserConfig(builder);
+ return builder;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends ConfigInstance> T getConfig(Class<T> configClass, String configId) {
+ try {
+ ConfigInstance.Builder builder = getConfig(getBuilder(configClass).newInstance(), configId);
+ return configClass.getConstructor(builder.getClass()).newInstance(builder);
+ } catch (InstantiationException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public ConfigProducer getProducer(String configId) {
+ return id2producer.get(configId);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T extends ConfigInstance> Class<? extends ConfigInstance.Builder> getBuilder(Class<T> configClass) {
+ for (Class<?> memberClass : configClass.getClasses()) {
+ if (memberClass.getSimpleName().equals("Builder"))
+ return (Class<? extends ConfigInstance.Builder>) memberClass;
+ }
+ throw new RuntimeException("Missing builder");
+ }
+
+ @Override
+ public DeployState getDeployState() {
+ return deployState;
+ }
+
+ public FileDistributor getFileDistributor() {
+ return fileDistributor;
+ }
+
+ public HostSystem getHostSystem() {
+ return hostSystem;
+ }
+
+ public void addDescendant(String configId, AbstractConfigProducer descendant) {
+ if (id2producer.containsKey(configId)) {
+ throw new RuntimeException
+ ("Config ID '" + configId + "' cannot be reserved by an instance of class '" +
+ descendant.getClass().getName() + "' since it is already used by an instance of class '" +
+ id2producer.get(configId).getClass().getName() + "'");
+ }
+ id2producer.put(configId, descendant);
+ }
+
+ @Override
+ public void addChild(AbstractConfigProducer abstractConfigProducer) {
+ super.addChild(abstractConfigProducer);
+ }
+
+ public final void setAdmin(String xml) {
+ String servicesXml =
+ "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<services>" + xml + "</services>";
+
+ try {
+ Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml)));
+ setAdmin(new DomAdminV2Builder(deployState.getFileRegistry(), false, new ArrayList<>()).
+ build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0)));
+ } catch (SAXException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public final void setAdmin(Admin admin) {
+ this.admin = admin;
+ }
+
+ @Override
+ public final Admin getAdmin() {
+ return admin;
+ }
+
+ @Override
+ public DeployLogger deployLogger() {
+ return new BaseDeployLogger();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java
new file mode 100644
index 00000000000..4518e44d441
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.config.model.MapConfigModelRegistry;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.application.provider.SchemaValidator;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.vespa.model.VespaModel;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test driver for testing config models. Add custom builders for plugins to be tested. Builds a model from the
+ * xml string and returns a config producer that can be use to test getConfig.
+ *
+ * @author lulf
+ * @since 5.1.20
+ */
+@Beta
+public class TestDriver {
+
+ private final List<ConfigModelBuilder> builders = new ArrayList<>();
+ private final boolean validate;
+
+ public TestDriver(boolean validate) {
+ this.validate = validate;
+ }
+
+ public TestDriver() {
+ this(false);
+ }
+
+ /**
+ * Add a new builder to the tester.
+ *
+ * @param builder builder to add.
+ * @return this for chaining
+ */
+ public TestDriver addBuilder(ConfigModelBuilder builder) {
+ builders.add(builder);
+ return this;
+ }
+
+ /**
+ * Build a model from an XML string. The hosts referenced in services must be set to 'mockhost' when using
+ * this method, as it automatically generates a hosts file for you.
+ *
+ * @param servicesXml The xml for services.xml
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(String servicesXml) {
+ return buildModel(servicesXml, "<hosts><host name='localhost'><alias>mockhost</alias></host></hosts>");
+ }
+
+ /**
+ * Build a model from an XML string of services and one of hosts.
+ *
+ * @param servicesXml The xml for services.xml
+ * @param hostsXml The xml for hosts.xml
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(String servicesXml, String hostsXml) {
+ if (!servicesXml.contains("<services")) {
+ servicesXml = "<services version='1.0'>" + servicesXml + "</services>";
+ }
+ return buildModel(new MockApplicationPackage.Builder().withHosts(hostsXml).withServices(servicesXml).build());
+ }
+
+ /**
+ * Build a model from an application package.
+ *
+ * @param applicationPackage Any type of application package.
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(ApplicationPackage applicationPackage) {
+ return buildModel(new DeployState.Builder().applicationPackage(applicationPackage).build());
+ }
+
+ /**
+ * Build a model given a deploy state.
+ *
+ * @param deployState An instance of {@link com.yahoo.config.model.deploy.DeployState}
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(DeployState deployState) {
+ MapConfigModelRegistry registry = new MapConfigModelRegistry(builders);
+ try {
+ validate(deployState.getApplicationPackage());
+ return new TestRoot(new VespaModel(registry, deployState));
+ } catch (IOException | SAXException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void validate(ApplicationPackage appPkg) throws IOException {
+ if (!validate) {
+ return;
+ }
+ SchemaValidator validator = SchemaValidator.createTestValidatorHosts();
+ if (appPkg.getHosts() != null) {
+ validator.validate(appPkg.getHosts());
+ }
+ validator = SchemaValidator.createTestValidatorServices();
+ validator.validate(appPkg.getServices());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java
new file mode 100644
index 00000000000..44491e33479
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.util.List;
+
+/**
+ * Test utility class that provides many methods for inspecting the state of a completely built model
+ *
+ * @author lulf
+ * @since 5.1
+ */
+@Beta
+public class TestRoot {
+ private final VespaModel model;
+ TestRoot(VespaModel model) {
+ this.model = model;
+ }
+
+ /**
+ * Get a list of all config models of a particular type.
+ *
+ * @param clazz The class of the models to find.
+ * @return A list of models of given type.
+ */
+ public <MODEL extends ConfigModel> List<MODEL> getConfigModels(Class<MODEL> clazz) {
+ return model.configModelRepo().getModels(clazz);
+ }
+
+ /**
+ * Ask model to populate builder with config for a given config id. This method gives the same config as the
+ * configserver would return to its clients.
+ * @param builder The builder to populate
+ * @param configId The config id of the producer to ask for config.
+ * @return the same builder.
+ */
+ public <BUILDER extends ConfigInstance.Builder> BUILDER getConfig(BUILDER builder, String configId) {
+ return (BUILDER) model.getConfig(builder, configId);
+ }
+
+ /**
+ * Request config of a given type and id. This method gives the same config as the configserver would return to
+ * its clients.
+ *
+ * @param clazz Type of config to request.
+ * @param configId The config id of the producer to ask for config.
+ * @return A config object of the appropriate type with config values set.
+ */
+ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) {
+ return model.getConfig(clazz, configId);
+ }
+
+ /**
+ * Retrieve the hosts available in this model. Useful to verify that hostnames are set correctly etc.
+ *
+ * @return A list of hosts.
+ */
+ public List<HostResource> getHosts() {
+ return model.getHostSystem().getHosts();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/package-info.java b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java
new file mode 100644
index 00000000000..6c33505df83
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java
@@ -0,0 +1,6 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+//TODO: Temporary export due to standalone container package, remove later.
+@ExportPackage
+package com.yahoo.config.model.test;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
new file mode 100644
index 00000000000..822ddcd0da7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import com.yahoo.document.DataType;
+
+import java.util.Collection;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public interface DataTypeCollection {
+ public DataType getDataType(String name);
+ public DataType getDataType(int id);
+ public Collection<DataType> getTypes();
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
new file mode 100644
index 00000000000..6d332ba16fb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import com.yahoo.document.DataType;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class DataTypeRepo implements DataTypeCollection {
+ Map<Integer, DataType> typeById = new LinkedHashMap<>();
+ Map<String, DataType> typeByName = new LinkedHashMap<>();
+
+ public DataType getDataType(String name) {
+ return typeByName.get(name);
+ }
+
+ public DataType getDataType(int id) {
+ return typeById.get(id);
+ }
+
+ public Collection<DataType> getTypes() { return typeById.values(); }
+
+ public DataTypeRepo add(DataType type) {
+ if (typeByName.containsKey(type.getName()) ||
+ typeById.containsKey(type.getId()))
+ {
+ throw new IllegalStateException("Data type '" + type.getName() + "', id '" + type.getId() + "' is already registered.");
+ }
+ typeByName.put(type.getName(), type);
+ typeById.put(type.getId(), type);
+ return this;
+ }
+
+ public DataTypeRepo addAll(DataTypeCollection repo) {
+ for (DataType dataType : repo.getTypes()) {
+ add(dataType);
+ }
+ return this;
+ }
+
+ public DataTypeRepo replace(DataType type) {
+ if (!typeByName.containsKey(type.getName()) ||
+ !typeById.containsKey(type.getId()))
+ {
+ throw new IllegalStateException("Data type '" + type.getName() + "' is not registered.");
+ }
+ typeByName.remove(type.getName());
+ typeByName.put(type.getName(), type);
+ typeById.remove(type.getId());
+ typeById.put(type.getId(), type);
+ return this;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java
new file mode 100644
index 00000000000..6c71410c048
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import java.util.Collection;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public interface DocumentTypeCollection {
+ public NewDocumentType getDocumentType(NewDocumentType.Name name);
+ public NewDocumentType getDocumentType(int id);
+ public Collection<NewDocumentType> getTypes();
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
new file mode 100644
index 00000000000..3585a12ac2f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class DocumentTypeRepo implements DocumentTypeCollection {
+ final Map<Integer, NewDocumentType> typeById = new LinkedHashMap<>();
+ final Map<NewDocumentType.Name, NewDocumentType> typeByName = new LinkedHashMap<>();
+
+ public final NewDocumentType getDocumentType(String name) {
+ return typeByName.get(new NewDocumentType.Name(name));
+ }
+ public NewDocumentType getDocumentType(NewDocumentType.Name name) {
+ return typeByName.get(name);
+ }
+
+ public NewDocumentType getDocumentType(int id) {
+ return typeById.get(id);
+ }
+
+ public Collection<NewDocumentType> getTypes() { return typeById.values(); }
+
+ public DocumentTypeRepo add(NewDocumentType type) {
+ if (typeByName.containsKey(type.getFullName())) {
+ throw new IllegalStateException("Document type " + type.toString() + " is already registered");
+ }
+ if (typeById.containsKey(type.getFullName().getId())) {
+ throw new IllegalStateException("Document type " + type.toString() + " is already registered");
+ }
+ typeByName.put(type.getFullName(), type);
+ typeById.put(type.getFullName().getId(), type);
+ return this;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
new file mode 100644
index 00000000000..51171e97704
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -0,0 +1,366 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.AnnotationTypeRegistry;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.searchdefinition.FieldSets;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.processing.BuiltInFieldSets;
+
+import java.util.*;
+
+/**
+ * TODO: What is this and why?
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public final class NewDocumentType extends StructuredDataType implements DataTypeCollection {
+
+ /**
+ * TODO: What is this and why?
+ */
+ public static final class Name {
+
+ // TODO: privatize
+ final String name;
+ final int id;
+
+ public Name(String name) {
+ this(name.hashCode(),name);
+ }
+
+ public Name(int id,String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ public String toString() { return name; }
+
+ public final String getName() { return name; }
+
+ public final int getId() { return id; }
+
+ public int hashCode() { return name.hashCode(); }
+
+ public boolean equals(Object other) {
+ if ( ! (other instanceof Name)) return false;
+ return name.equals(((Name)other).getName());
+ }
+ }
+
+ private final Name name;
+ private final DataTypeRepo dataTypes = new DataTypeRepo();
+ private final Map<Integer, NewDocumentType> inherits = new LinkedHashMap<>();
+ private final AnnotationTypeRegistry annotations = new AnnotationTypeRegistry();
+ private final StructDataType header;
+ private final StructDataType body;
+ private final Set<FieldSet> fieldSets = new LinkedHashSet<>();
+
+ public NewDocumentType(Name name) {
+ this(name,
+ new StructDataType(name.getName() + ".header"),
+ new StructDataType(name.getName() + ".body"), new FieldSets());
+ }
+ public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs) {
+ super(name.getName());
+ this.name = name;
+ this.header = header;
+ this.body = body;
+ if (fs != null) {
+ this.fieldSets.addAll(fs.userFieldSets().values());
+ for (FieldSet f : fs.builtInFieldSets().values()) {
+ if ((f.getName() != BuiltInFieldSets.INTERNAL_FIELDSET_NAME) &&
+ (f.getName() != BuiltInFieldSets.SEARCH_FIELDSET_NAME)) {
+ fieldSets.add(f);
+ }
+ }
+ }
+ }
+
+ public Name getFullName() {
+ return name;
+ }
+
+ public DataType getHeader() { return header; }
+ public DataType getBody() { return body; }
+ public Collection<NewDocumentType> getInherited() { return inherits.values(); }
+ public NewDocumentType getInherited(Name inherited) { return inherits.get(inherited.getId()); }
+ public NewDocumentType removeInherited(Name inherited) { return inherits.remove(inherited.getId()); }
+
+ /**
+ * Data type of the header fields of this and all inherited document types
+ * @return merged {@link StructDataType}
+ */
+ public StructDataType allHeader() {
+ StructDataType ret = new StructDataType(header.getName());
+ for (Field f : header.getFields()) {
+ ret.addField(f);
+ }
+ for (NewDocumentType inherited : getInherited()) {
+ for (Field f : ((StructDataType) inherited.getHeader()).getFields()) {
+ ret.addField(f);
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Data type of the body fields of this and all inherited document types
+ * @return merged {@link StructDataType}
+ */
+ public StructDataType allBody() {
+ StructDataType ret = new StructDataType(body.getName());
+ for (Field f : body.getFields()) {
+ ret.addField(f);
+ }
+ for (NewDocumentType inherited : getInherited()) {
+ for (Field f : ((StructDataType) inherited.getBody()).getFields()) {
+ ret.addField(f);
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public Class getValueClass() {
+ return Document.class;
+ }
+
+ @Override
+ public boolean isValueCompatible(FieldValue value) {
+ if (!(value instanceof Document)) {
+ return false;
+ }
+ /** Temporary disabled due to clash with document and covariant return type
+ Document doc = (Document) value;
+ if (((NewDocumentType) doc.getDataType()).inherits(this)) {
+ //the value is of this type; or the supertype of the value is of this type, etc....
+ return true;
+ }
+ */
+ return false;
+ }
+
+ private boolean verifyInheritance(NewDocumentType inherited) {
+ for (Field f : getFields()) {
+ Field inhF = inherited.getField(f.getName());
+ if (inhF != null && !inhF.equals(f)) {
+ throw new IllegalArgumentException("Inherited document '" + inherited.toString() + "' already contains field '" +
+ inhF.getName() + "'. Can not override with '" + f.getName() + "'.");
+ }
+ }
+ for (Field f : inherited.getAllFields()) {
+ for (NewDocumentType side : inherits.values()) {
+ Field sideF = side.getField(f.getName());
+ if (sideF != null && !sideF.equals(f)) {
+ throw new IllegalArgumentException("Inherited document '" + side.toString() + "' already contains field '" +
+ sideF.getName() + "'. Document '" + inherited.toString() + "' also defines field '" + f.getName() +
+ "'.Multiple inheritance must be disjunctive.");
+ }
+ }
+ }
+ return true;
+ }
+ public void inherit(NewDocumentType inherited) {
+ if ( ! inherits.containsKey(inherited.getId())) {
+ verifyInheritance(inherited);
+ inherits.put(inherited.getId(), inherited);
+ }
+ }
+ public boolean inherits(NewDocumentType superType) {
+ if (getId() == superType.getId()) return true;
+ for (NewDocumentType type : inherits.values()) {
+ if (type.inherits(superType)) return true;
+ }
+ return false;
+ }
+
+ @Override
+ public Field getField(String name) {
+ Field field = header.getField(name);
+ if (field == null) {
+ field = body.getField(name);
+ }
+ if (field == null) {
+ for (NewDocumentType inheritedType : inherits.values()) {
+ field = inheritedType.getField(name);
+ if (field != null) {
+ return field;
+ }
+ }
+ }
+ return field;
+ }
+
+ public boolean containsField(String fieldName) {
+ return getField(fieldName) != null;
+ }
+
+ @Override
+ public Field getField(int id) {
+ Field field = header.getField(id);
+ if (field == null) {
+ field = body.getField(id);
+ }
+ if (field == null) {
+ for (NewDocumentType inheritedType : inherits.values()) {
+ field = inheritedType.getField(id);
+ if (field != null) {
+ return field;
+ }
+ }
+ }
+ return field;
+ }
+
+ public Collection<Field> getAllFields() {
+ Collection<Field> collection = new LinkedList<>();
+
+ for (NewDocumentType type : inherits.values()) {
+ collection.addAll(type.getAllFields());
+ }
+
+ collection.addAll(header.getFields());
+ collection.addAll(body.getFields());
+ return Collections.unmodifiableCollection(collection);
+ }
+
+ public Collection<Field> getFields() {
+ Collection<Field> collection = new LinkedList<>();
+ collection.addAll(header.getFields());
+ collection.addAll(body.getFields());
+ return Collections.unmodifiableCollection(collection);
+ }
+
+ @Override
+ public Document createFieldValue() {
+ return new Document(null, (DocumentId)null);
+ }
+
+ @Override
+ public Collection<DataType> getTypes() {
+ return dataTypes.getTypes();
+ }
+
+ public DataTypeCollection getAllTypes() {
+ DataTypeRepo repo = new DataTypeRepo();
+ Set<Name> seen = new HashSet<>();
+ Deque<NewDocumentType> stack = new LinkedList<>();
+ stack.push(this);
+ while (!stack.isEmpty()) {
+ NewDocumentType docType = stack.pop();
+ if (seen.contains(docType.name)) {
+ continue; // base type
+ }
+ seen.add(docType.name);
+ for (DataType dataType : docType.getTypes()) {
+ if (repo.getDataType(dataType.getId()) == null) {
+ repo.add(dataType);
+ }
+ }
+ stack.addAll(docType.inherits.values());
+ }
+ return repo;
+ }
+
+ public Collection<AnnotationType> getAnnotations() { return annotations.getTypes().values(); }
+ public Collection<AnnotationType> getAllAnnotations() {
+ Collection<AnnotationType> collection = new LinkedList<>();
+
+ for (NewDocumentType type : inherits.values()) {
+ collection.addAll(type.getAllAnnotations());
+ }
+ collection.addAll(getAnnotations());
+
+ return Collections.unmodifiableCollection(collection);
+ }
+
+ public DataType getDataType(String name) {
+ return dataTypes.getDataType(name);
+ }
+ public DataType getDataType(int id) {
+ return dataTypes.getDataType(id);
+ }
+ public DataType getDataTypeRecursive(String name) {
+ DataType a = dataTypes.getDataType(name);
+ if (a != null) {
+ return a;
+ } else {
+ for (NewDocumentType dt : getInherited()) {
+ a = dt.getDataTypeRecursive(name);
+ if (a != null) {
+ return a;
+ }
+ }
+ }
+ return null;
+ }
+
+ public DataType getDataTypeRecursive(int id) {
+ DataType a = dataTypes.getDataType(id);
+ if (a != null) {
+ return a;
+ } else {
+ for (NewDocumentType dt : getInherited()) {
+ a = dt.getDataTypeRecursive(id);
+ if (a != null) {
+ return a;
+ }
+ }
+ }
+ return null;
+ }
+
+ public AnnotationType getAnnotationType(String name) {
+ AnnotationType a = annotations.getType(name);
+ if (a != null) {
+ return a;
+ } else {
+ for (NewDocumentType dt : getInherited()) {
+ a = dt.getAnnotationType(name);
+ if (a != null) {
+ return a;
+ }
+ }
+ }
+ return null;
+ }
+ public AnnotationType getAnnotationType(int id) {
+ AnnotationType a = annotations.getType(id);
+ if (a != null) {
+ return a;
+ } else {
+ for (NewDocumentType dt : getInherited()) {
+ a = dt.getAnnotationType(id);
+ if (a != null) {
+ return a;
+ }
+ }
+ }
+ return null;
+ }
+
+ public NewDocumentType add(AnnotationType type) {
+ annotations.register(type);
+ return this;
+ }
+ public NewDocumentType add(DataType type) {
+ dataTypes.add(type);
+ return this;
+ }
+ public NewDocumentType replace(DataType type) {
+ dataTypes.replace(type);
+ return this;
+ }
+
+ /**
+ * The field sets defined for this type and its {@link Search}
+ * @return fieldsets
+ */
+ public Set<FieldSet> getFieldSets() {
+ return Collections.unmodifiableSet(fieldSets);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
new file mode 100644
index 00000000000..bf8ec8d3da7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentmodel;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DataTypeName;
+import com.yahoo.document.PositionDataType;
+
+/**
+ * This class represents the builtin 'doument' document type that all other documenttypes inherits.
+ * Remember that changes here must be compatible. Changes to types of fields can not be done here.
+ * This must also match the mirroring class in c++.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class VespaDocumentType {
+
+ public static NewDocumentType INSTANCE = newInstance();
+
+ public static DataTypeName NAME = new DataTypeName("document");
+
+ private static NewDocumentType newInstance() {
+ NewDocumentType vespa = new NewDocumentType(new NewDocumentType.Name(8, "document"));
+ vespa.add(DataType.BYTE);
+ vespa.add(DataType.INT);
+ vespa.add(DataType.LONG);
+ vespa.add(DataType.STRING);
+ vespa.add(DataType.RAW);
+ vespa.add(DataType.TAG);
+ vespa.add(DataType.FLOAT);
+ vespa.add(DataType.DOUBLE);
+ vespa.add(DataType.DOCUMENT);
+ vespa.add(PositionDataType.INSTANCE);
+ vespa.add(DataType.URI);
+ vespa.add(DataType.PREDICATE);
+ vespa.add(DataType.TENSOR);
+ return vespa;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java
new file mode 100644
index 00000000000..98eb0a4b77c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.NameNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Transforms named references to constant tensors with the rank feature 'constant'.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+class ConstantTensorTransformer extends ExpressionTransformer {
+
+ private final Map<String, Value> constants;
+ private final Map<String, String> rankPropertiesOutput;
+
+ public ConstantTensorTransformer(Map<String, Value> constants,
+ Map<String, String> rankPropertiesOutput) {
+ this.constants = constants;
+ this.rankPropertiesOutput = rankPropertiesOutput;
+ }
+
+ @Override
+ public ExpressionNode transform(ExpressionNode node) {
+ if (node instanceof ReferenceNode) {
+ return transformFeature((ReferenceNode) node);
+ } else if (node instanceof CompositeNode) {
+ return transformChildren((CompositeNode) node);
+ } else {
+ return node;
+ }
+ }
+
+ private ExpressionNode transformFeature(ReferenceNode node) {
+ if (!node.getArguments().isEmpty()) {
+ return transformArguments(node);
+ } else {
+ return transformConstantReference(node);
+ }
+ }
+
+ private ExpressionNode transformArguments(ReferenceNode node) {
+ List<ExpressionNode> arguments = node.getArguments().expressions();
+ List<ExpressionNode> transformedArguments = new ArrayList<>(arguments.size());
+ for (ExpressionNode argument : arguments) {
+ transformedArguments.add(transform(argument));
+ }
+ return node.setArguments(transformedArguments);
+ }
+
+ private ExpressionNode transformConstantReference(ReferenceNode node) {
+ Value value = constants.get(node.getName());
+ if (value == null || !(value instanceof TensorValue)) {
+ return node;
+ }
+ TensorValue tensorValue = (TensorValue)value;
+ String featureName = "constant(" + node.getName() + ")";
+ String tensorType = (tensorValue.getType().isPresent() ? tensorValue.getType().get().toString() : "tensor");
+ rankPropertiesOutput.put(featureName + ".value", tensorValue.toString());
+ rankPropertiesOutput.put(featureName + ".type", tensorType);
+ return new ReferenceNode("constant", Arrays.asList(new NameNode(node.getName())), null);
+ }
+
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java
new file mode 100644
index 00000000000..c0dcf27c88d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * The rank profile containing default settings. This is derived from the fields
+ * whenever this is accessed.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class DefaultRankProfile extends RankProfile {
+
+ /**
+ * Creates a new rank profile
+ *
+ * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles.
+ */
+ public DefaultRankProfile(Search search, RankProfileRegistry rankProfileRegistry) {
+ super("default", search, rankProfileRegistry);
+ }
+
+ /**
+ * Does nothing, the default rank profile can not inherit anything
+ */
+ // TODO: Why not? If that's the case, then fail attempts at it
+ public void setInherited(String inheritedName) {
+ }
+
+ /**
+ * Returns null, the default rank profile can not inherit anything
+ */
+ public String getInheritedName() {
+ return null;
+ }
+
+ /**
+ * Returns the rank boost value of the given field
+ */
+ public RankSetting getRankSetting(String fieldOrIndex,RankSetting.Type type) {
+ RankSetting setting=super.getRankSetting(fieldOrIndex,type);
+ if (setting!=null) return setting;
+
+ SDField field=getSearch().getField(fieldOrIndex);
+ if (field!=null) {
+ setting=toRankSetting(field,type);
+ if (setting!=null)
+ return setting;
+ }
+
+ Index index=getSearch().getIndex(fieldOrIndex);
+ if (index!=null) {
+ setting=toRankSetting(index,type);
+ if (setting!=null)
+ return setting;
+ }
+
+ return null;
+ }
+
+ private RankSetting toRankSetting(SDField field,RankSetting.Type type) {
+ if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight()>0 && field.getWeight()!=100)
+ return new RankSetting(field.getName(),type,field.getWeight());
+ if (type.equals(RankSetting.Type.RANKTYPE))
+ return new RankSetting(field.getName(),type,field.getRankType());
+ if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost()>0)
+ return new RankSetting(field.getName(),type,field.getLiteralBoost());
+
+ // Index level setting really
+ if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) {
+ return new RankSetting(field.getName(), type, true);
+ }
+
+ return null;
+ }
+
+ private RankSetting toRankSetting(Index index, RankSetting.Type type) {
+ /* TODO: Add support for indexes by adding a ranking object to the index
+ if (type.equals(RankSetting.Type.PREFERBITVECTOR) && index.isPreferBitVector()) {
+ return new RankSetting(index.getName(), type, new Boolean(true));
+ }
+ */
+ return null;
+ }
+
+ /**
+ * Returns the names of the fields which have a rank boost setting
+ * explicitly in this profile or in fields
+ */
+ public Set<RankSetting> rankSettings() {
+ Set<RankSetting> settings=new LinkedHashSet<>(20);
+ settings.addAll(this.rankSettings);
+ for (SDField field : getSearch().allFieldsList() ) {
+ addSetting(field,RankSetting.Type.WEIGHT,settings);
+ addSetting(field,RankSetting.Type.RANKTYPE,settings);
+ addSetting(field,RankSetting.Type.LITERALBOOST,settings);
+ addSetting(field,RankSetting.Type.PREFERBITVECTOR,settings);
+ }
+
+ // Foer settings that really pertains to indexes do the explicit indexes too
+ for (Index index : getSearch().getExplicitIndices()) {
+ addSetting(index,RankSetting.Type.PREFERBITVECTOR,settings);
+ }
+ return settings;
+ }
+
+ private void addSetting(SDField field,RankSetting.Type type,Set<RankSetting> settings) {
+ if (type.isIndexLevel()) {
+ addIndexSettings(field,type,settings);
+ }
+ else {
+ RankSetting setting=toRankSetting(field,type);
+ if (setting==null) return;
+ settings.add(setting);
+ }
+ }
+
+ private void addIndexSettings(SDField field,RankSetting.Type type,Set<RankSetting> settings) {
+ for (Iterator i = field.getFieldNameAsIterator(); i.hasNext(); ) {
+ String indexName=(String)i.next();
+ Index explicitIndex=field.getIndex(indexName);
+
+ // TODO: Make a ranking object in the index override the field level ranking object
+ if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) {
+ settings.add(new RankSetting(indexName, type, true));
+ }
+ }
+ }
+
+ private void addSetting(Index index,RankSetting.Type type,Set<RankSetting> settings) {
+ RankSetting setting=toRankSetting(index,type);
+ if (setting==null) return;
+ settings.add(setting);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
new file mode 100644
index 00000000000..4fd7048159e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -0,0 +1,415 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.documentmodel.DataTypeCollection;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.documentmodel.FieldView;
+import com.yahoo.vespa.documentmodel.SearchDef;
+import com.yahoo.vespa.documentmodel.SearchField;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class DocumentModelBuilder {
+
+ public static class RetryLaterException extends IllegalArgumentException {
+ public RetryLaterException(String message) {
+ super(message);
+ }
+ }
+ private DocumentModel model;
+ private final Map<NewDocumentType, List<SDDocumentType>> scratchInheritsMap = new HashMap<>();
+ public DocumentModelBuilder(DocumentModel model) {
+ this.model = model;
+ model.getDocumentManager().add(VespaDocumentType.INSTANCE);
+ }
+ public boolean valid() {
+ return scratchInheritsMap.isEmpty();
+ }
+ public void addToModel(Collection<Search> searchList) {
+ List<SDDocumentType> docList = new LinkedList<>();
+ for (Search search : searchList) {
+ docList.add(search.getDocument());
+ }
+ docList = sortDocumentTypes(docList);
+ addDocumentTypes(docList);
+ for (Collection<Search> toAdd = tryAdd(searchList);
+ !toAdd.isEmpty() && (toAdd.size() < searchList.size()); toAdd = tryAdd(searchList)) {
+ searchList = toAdd;
+ }
+ }
+
+ private List<SDDocumentType> sortDocumentTypes(List<SDDocumentType> docList) {
+ Set<String> doneNames = new HashSet<>();
+ doneNames.add(SDDocumentType.VESPA_DOCUMENT.getName());
+ List<SDDocumentType> doneList = new LinkedList<>();
+ List<SDDocumentType> prevList = null;
+ List<SDDocumentType> nextList = docList;
+ while (prevList == null || nextList.size() < prevList.size()) {
+ prevList = nextList;
+ nextList = new LinkedList<>();
+ for (SDDocumentType doc : prevList) {
+ boolean isDone = true;
+ for (SDDocumentType inherited : doc.getInheritedTypes()) {
+ if (!doneNames.contains(inherited.getName())) {
+ isDone = false;
+ break;
+ }
+ }
+ if (isDone) {
+ doneNames.add(doc.getName());
+ doneList.add(doc);
+ } else {
+ nextList.add(doc);
+ }
+ }
+ }
+ if (!nextList.isEmpty()) {
+ throw new IllegalArgumentException("Could not resolve inheritance of document types " +
+ toString(prevList) + ".");
+ }
+ return doneList;
+ }
+
+ private static String toString(List<SDDocumentType> lst) {
+ StringBuilder out = new StringBuilder();
+ for (int i = 0, len = lst.size(); i < len; ++i) {
+ out.append("'").append(lst.get(i).getName()).append("'");
+ if (i < len - 2) {
+ out.append(", ");
+ } else if (i < len - 1) {
+ out.append(" and ");
+ }
+ }
+ return out.toString();
+ }
+
+ private Collection<Search> tryAdd(Collection<Search> searchList) {
+ Collection<Search> left = new ArrayList<>();
+ for (Search search : searchList) {
+ try {
+ addToModel(search);
+ } catch (RetryLaterException e) {
+ left.add(search);
+ }
+ }
+ return left;
+ }
+ public void addToModel(Search search) {
+ // Then we add the search specific stuff
+ SearchDef searchDef = new SearchDef(search.getName());
+ addSearchFields(search.extraFieldList(), searchDef);
+ for (Field f : search.getDocument().fieldSet()) {
+ addSearchField((SDField) f, searchDef);
+ }
+ for(SDField field : search.allFieldsList()) {
+ for(Attribute attribute : field.getAttributes().values()) {
+ if (!searchDef.getFields().containsKey(attribute.getName())) {
+ searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true));
+ }
+ }
+ }
+
+ for (Field f : search.getDocument().fieldSet()) {
+ addAlias((SDField) f, searchDef);
+ }
+ model.getSearchManager().add(searchDef);
+ }
+ private static void addSearchFields(Collection<SDField> fields, SearchDef searchDef) {
+ for (SDField field : fields) {
+ addSearchField(field, searchDef);
+ }
+ }
+ private static void addSearchField(SDField field, SearchDef searchDef) {
+
+ SearchField searchField =
+ new SearchField(field,
+ field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA),
+ field.getAttributes().containsKey(field.getName()));
+ searchDef.add(searchField);
+
+ // Add field to views
+ addToView(field.getIndices().keySet(), searchField, searchDef);
+ }
+
+ private static void addAlias(SDField field, SearchDef searchDef) {
+ for (Map.Entry<String, String> entry : field.getAliasToName().entrySet()) {
+ searchDef.addAlias(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private static void addToView(Collection<String> views, Field field, SearchDef searchDef) {
+ for (String viewName : views) {
+ addToView(viewName, field, searchDef);
+ }
+ }
+
+ private static void addToView(String viewName, Field field, SearchDef searchDef) {
+ if (searchDef.getViews().containsKey(viewName)) {
+ searchDef.getViews().get(viewName).add(field);
+ } else {
+ if (!searchDef.getFields().containsKey(viewName)) {
+ FieldView view = new FieldView(viewName);
+ view.add(field);
+ searchDef.add(view);
+ }
+ }
+ }
+ private void addDocumentTypes(List<SDDocumentType> docList) {
+ LinkedList<NewDocumentType> lst = new LinkedList<>();
+ for (SDDocumentType doc : docList) {
+ lst.add(convert(doc));
+ model.getDocumentManager().add(lst.getLast());
+ }
+ for(NewDocumentType doc : lst) {
+ resolveTemporaries(doc.getAllTypes(), lst);
+ }
+ }
+ private static void resolveTemporaries(DataTypeCollection dtc, Collection<NewDocumentType> docs) {
+ for (DataType type : dtc.getTypes()) {
+ resolveTemporariesRecurse(type, dtc, docs);
+ }
+ }
+ private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo,
+ Collection<NewDocumentType> docs) {
+ if (type instanceof TemporaryStructuredDataType) {
+ NewDocumentType docType = getDocumentType(docs, type.getId());
+ if (docType != null) {
+ type = docType;
+ return type;
+ }
+ DataType real = repo.getDataType(type.getId());
+ if (real == null) {
+ throw new NullPointerException("Can not find type '" + type.toString() + "', impossible.");
+ }
+ type = real;
+ } else if (type instanceof StructDataType) {
+ StructDataType dt = (StructDataType) type;
+ for (com.yahoo.document.Field field : dt.getFields()) {
+ if (field.getDataType() != type) {
+ field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs));
+ }
+ }
+ } else if (type instanceof MapDataType) {
+ MapDataType t = (MapDataType) type;
+ t.setKeyType(resolveTemporariesRecurse(t.getKeyType(), repo, docs));
+ t.setValueType(resolveTemporariesRecurse(t.getValueType(), repo, docs));
+ } else if (type instanceof CollectionDataType) {
+ CollectionDataType t = (CollectionDataType) type;
+ t.setNestedType(resolveTemporariesRecurse(t.getNestedType(), repo, docs));
+ }
+ return type;
+ }
+
+ private static NewDocumentType getDocumentType(Collection<NewDocumentType> docs, int id) {
+ for (NewDocumentType doc : docs) {
+ if (doc.getId() == id) {
+ return doc;
+ }
+ }
+ return null;
+ }
+
+ private static void specialHandleAnnotationReference(NewDocumentType docType, Field field) {
+ DataType fieldType = specialHandleAnnotationReferenceRecurse(docType, field.getName(), field.getDataType());
+ if (fieldType == null) {
+ return;
+ }
+ field.setDataType(fieldType);
+ }
+
+ private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName,
+ DataType dataType)
+ {
+ if (dataType instanceof TemporaryAnnotationReferenceDataType) {
+ TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType;
+ if (refType.getId() != 0) {
+ return null;
+ }
+ AnnotationType target = docType.getAnnotationType(refType.getTarget());
+ if (target == null) {
+ throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName +
+ "' does not exist.");
+ }
+ dataType = new AnnotationReferenceDataType(target);
+ addType(docType, dataType);
+ return dataType;
+ } else if (dataType instanceof MapDataType) {
+ MapDataType mapType = (MapDataType)dataType;
+ DataType valueType = specialHandleAnnotationReferenceRecurse(docType, fieldName, mapType.getValueType());
+ if (valueType == null) {
+ return null;
+ }
+ mapType = mapType.clone();
+ mapType.setValueType(valueType);
+ addType(docType, mapType);
+ return mapType;
+ } else if (dataType instanceof CollectionDataType) {
+ CollectionDataType lstType = (CollectionDataType)dataType;
+ DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, lstType.getNestedType());
+ if (nestedType == null) {
+ return null;
+ }
+ lstType = lstType.clone();
+ lstType.setNestedType(nestedType);
+ addType(docType, lstType);
+ return lstType;
+ }
+ return null;
+ }
+
+ private static StructDataType handleStruct(NewDocumentType dt, SDDocumentType type) {
+ StructDataType s = new StructDataType(type.getName());
+ for (Field f : type.getDocumentType().getHeaderType().getFieldsThisTypeOnly()) {
+ specialHandleAnnotationReference(dt, f);
+ s.addField(f);
+ }
+ for (StructDataType inherited : type.getDocumentType().getHeaderType().getInheritedTypes()) {
+ s.inherit(inherited);
+ }
+ extractNestedTypes(dt, s);
+ addType(dt, s);
+ return s;
+ }
+
+ private static StructDataType handleStruct(NewDocumentType dt, StructDataType s) {
+ for (Field f : s.getFieldsThisTypeOnly()) {
+ specialHandleAnnotationReference(dt, f);
+ }
+ extractNestedTypes(dt, s);
+ addType(dt, s);
+ return s;
+ }
+ private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) {
+ if (sa.getInherits() != null) {
+ AnnotationType tmp = sdoc.findAnnotation(sa.getInherits());
+ SDAnnotationType inherited = (SDAnnotationType) tmp;
+ return ((inherited.getSdDocType() != null) || anyParentsHavePayLoad(inherited, sdoc));
+ }
+ return false;
+ }
+ private NewDocumentType convert(SDDocumentType sdoc) {
+ Map<AnnotationType, String> annotationInheritance = new HashMap<>();
+ Map<StructDataType, String> structInheritance = new HashMap<>();
+ NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()),
+ sdoc.getDocumentType().getHeaderType(),
+ sdoc.getDocumentType().getBodyType(),
+ sdoc.getFieldSets());
+ for (SDDocumentType n : sdoc.getInheritedTypes()) {
+ NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
+ NewDocumentType inherited = model.getDocumentManager().getDocumentType(name);
+ if (inherited != null) {
+ dt.inherit(inherited);
+ }
+ }
+ for (SDDocumentType type : sdoc.getTypes()) {
+ if (type.isStruct()) {
+ handleStruct(dt, type);
+ } else {
+ throw new IllegalArgumentException("Data type '" + sdoc.getName() + "' is not a struct => tostring='" + sdoc.toString() + "'.");
+ }
+ }
+ for (AnnotationType annotation : sdoc.getAnnotations()) {
+ dt.add(annotation);
+ }
+ for (AnnotationType annotation : sdoc.getAnnotations()) {
+ SDAnnotationType sa = (SDAnnotationType) annotation;
+ if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) {
+ annotationInheritance.put(annotation, sa.getInherits());
+ }
+ if (annotation.getDataType() == null) {
+ if (sa.getSdDocType() != null) {
+ StructDataType s = handleStruct(dt, sa.getSdDocType());
+ annotation.setDataType(s);
+ if ((sa.getInherits() != null)) {
+ structInheritance.put(s, "annotation."+sa.getInherits());
+ }
+ } else if (sa.getInherits() != null) {
+ StructDataType s = new StructDataType("annotation."+annotation.getName());
+ if (anyParentsHavePayLoad(sa, sdoc)) {
+ annotation.setDataType(s);
+ addType(dt, s);
+ }
+ structInheritance.put(s, "annotation."+sa.getInherits());
+ }
+ }
+ }
+ for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) {
+ e.getKey().inherit(dt.getAnnotationType(e.getValue()));
+ }
+ for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) {
+ StructDataType s = (StructDataType)dt.getDataType(e.getValue());
+ if (s != null) {
+ e.getKey().inherit(s);
+ }
+ }
+ handleStruct(dt, sdoc.getDocumentType().getHeaderType());
+ handleStruct(dt, sdoc.getDocumentType().getBodyType());
+
+ extractDataTypesFromFields(dt, sdoc.fieldSet());
+ return dt;
+ }
+ private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
+ for (Field f : fields) {
+ DataType type = f.getDataType();
+ if (testAddType(dt, type)) {
+ extractNestedTypes(dt, type);
+ addType(dt, type);
+ }
+ }
+ }
+ private static void extractNestedTypes(NewDocumentType dt, DataType type) {
+ if (type instanceof StructDataType) {
+ StructDataType tmp = (StructDataType) type;
+ extractDataTypesFromFields(dt, tmp.getFieldsThisTypeOnly());
+ } else if (type instanceof DocumentType) {
+ throw new IllegalArgumentException("Can not handle nested document definitions. In document type '" + dt.getName().toString() +
+ "', we can not define document type '" + type.toString());
+ } else if (type instanceof CollectionDataType) {
+ CollectionDataType tmp = (CollectionDataType) type;
+ extractNestedTypes(dt, tmp.getNestedType());
+ addType(dt, tmp.getNestedType());
+ } else if (type instanceof MapDataType) {
+ MapDataType tmp = (MapDataType) type;
+ extractNestedTypes(dt, tmp.getKeyType());
+ extractNestedTypes(dt, tmp.getValueType());
+ addType(dt, tmp.getKeyType());
+ addType(dt, tmp.getValueType());
+ } else if (type instanceof TemporaryAnnotationReferenceDataType) {
+ throw new IllegalArgumentException(type.toString());
+ }
+ }
+ private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); }
+ private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); }
+ private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) {
+ DataType oldType = dt.getDataTypeRecursive(type.getId());
+ if (oldType == null) {
+ if ( ! dryRun) {
+ dt.add(type);
+ }
+ return true;
+ } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) {
+ StructDataType s = (StructDataType) type;
+ StructDataType os = (StructDataType) oldType;
+ if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) {
+ if ( ! dryRun) {
+ dt.replace(type);
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java
new file mode 100644
index 00000000000..9cb3dea36a8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldOperationApplier {
+ public void process(SDDocumentType sdoc) {
+ if (!sdoc.isStruct()) {
+ apply(sdoc);
+ }
+ }
+
+ protected void apply(SDDocumentType type) {
+ for (Field field : type.fieldSet()) {
+ apply(field);
+ }
+ }
+
+ protected void apply(Field field) {
+ if (field instanceof SDField) {
+ SDField sdField = (SDField) field;
+ sdField.applyOperations();
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java
new file mode 100644
index 00000000000..301d78bbaec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldOperationApplierForSearch extends FieldOperationApplier {
+ @Override
+ public void process(SDDocumentType sdoc) {
+ //Do nothing
+ }
+
+ public void process(Search search) {
+ for (Field field : search.extraFieldList()) {
+ apply(field);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
new file mode 100644
index 00000000000..5bef71ac920
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldOperationApplierForStructs extends FieldOperationApplier {
+ @Override
+ public void process(SDDocumentType sdoc) {
+ for (SDDocumentType type : sdoc.getAllTypes()) {
+ if (type.isStruct()) {
+ apply(type);
+ copyFields(type, sdoc);
+ }
+ }
+ }
+
+ private void copyFields(SDDocumentType structType, SDDocumentType sdoc) {
+ //find all fields in OTHER types that have this type:
+ List<SDDocumentType> list = new ArrayList<>();
+ list.add(sdoc);
+ list.addAll(sdoc.getTypes());
+ for (SDDocumentType anyType : list) {
+ Iterator<Field> fields = anyType.fieldIterator();
+ while (fields.hasNext()) {
+ SDField field = (SDField) fields.next();
+ DataType structUsedByField = field.getFirstStructRecursive();
+ if (structUsedByField == null) {
+ continue;
+ }
+ if (structUsedByField.getName().equals(structType.getName())) {
+ //this field is using this type!!
+ field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0);
+ field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching());
+ }
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java
new file mode 100644
index 00000000000..503977a8f49
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.yahoo.searchdefinition.document.FieldSet;
+
+/**
+ * The field sets owned by a {@link Search}
+ * Both built in and user defined.
+ * @author vegardh
+ *
+ */
+public class FieldSets {
+
+ private final Map<String, FieldSet> userFieldSets = new LinkedHashMap<>();
+ private final Map<String, FieldSet> builtInFieldSets = new LinkedHashMap<>();
+
+ /**
+ * Adds an entry to user field sets, creating entries as needed
+ *
+ * @param setName name of a field set
+ * @param field field to add to field set
+ */
+ public void addUserFieldSetItem(String setName, String field) {
+ if (userFieldSets.get(setName) == null) {
+ // First entry in this set
+ userFieldSets.put(setName, new FieldSet(setName));
+ }
+ userFieldSets.get(setName).addFieldName(field);
+ }
+
+ /**
+ * Adds an entry to built in field sets, creating entries as needed
+ *
+ * @param setName name of a field set
+ * @param field field to add to field set
+ */
+ public void addBuiltInFieldSetItem(String setName, String field) {
+ if (builtInFieldSets.get(setName) == null) {
+ // First entry in this set
+ builtInFieldSets.put(setName, new FieldSet(setName));
+ }
+ builtInFieldSets.get(setName).addFieldName(field);
+ }
+
+ /**
+ * The built in field sets, unmodifiable
+ * @return built in field sets
+ */
+ public Map<String, FieldSet> builtInFieldSets() {
+ return Collections.unmodifiableMap(builtInFieldSets);
+ }
+
+ /**
+ * The user defined field sets, unmodifiable
+ * @return user field sets
+ */
+ public Map<String, FieldSet> userFieldSets() {
+ return Collections.unmodifiableMap(userFieldSets);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
new file mode 100644
index 00000000000..9ed7d8677b9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
@@ -0,0 +1,203 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.Stemming;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * An index definition in a search definition.
+ * Two indices are equal if they have the same name and the same settings, except
+ * alias settings (which are excluded).
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class Index implements Cloneable, Serializable {
+
+ public static enum Type {
+ VESPA("vespa");
+ private String name;
+ private Type(String name) { this.name = name; }
+ public String getName() { return name; }
+
+ }
+
+ // Please see hashCode, equals and copy when adding attributes to this
+
+ /** The search definition-unique name of this index */
+ private String name;
+
+ /** The rank type of this index */
+ private RankType rankType=null;
+
+ /** Whether this index supports prefix search */
+ private boolean prefix;
+
+ /** The list of aliases (Strings) to this index name */
+ private Set<String> aliases=new java.util.LinkedHashSet<>(1);
+
+ /**
+ * The stemming setting of this field, or null to use the default.
+ * Default is determined by the owning search definition.
+ */
+ private Stemming stemming=null;
+
+ /** Whether the content of this index is normalized */
+ private boolean normalized=true;
+
+ /** The set of all searchable fields which should be searched with this index. May not null */
+ public Set<String> matchGroup=new LinkedHashSet<>();
+
+ private Type type = Type.VESPA;
+
+ /** The boolean index definition, if set */
+ private BooleanIndexDefinition boolIndex;
+
+ public Index(String name) {
+ this(name, false);
+ }
+
+ public Index(String name, boolean prefix) {
+ this.name=name;
+ this.prefix=prefix;
+ }
+
+ public void setName(String name) { this.name=name; }
+
+ public String getName() { return name; }
+
+ /** Sets the rank type of this field */
+ public void setRankType(RankType rankType) { this.rankType=rankType; }
+
+ /** Returns the rank type of this field, or null if nothing is set */
+ public RankType getRankType() { return rankType; }
+
+ /** Return the stemming setting of this index, may be null */
+ public Stemming getStemming() { return stemming; }
+
+ /**
+ * Returns the (unmodifiable) set of searchable fields which should be searched
+ * when this index is searched. This is useful to specify that some attributes should be
+ * searched as well when an index is searched.
+ * This set is either empty, or if set contains both the name of this index, and some other
+ * indexes.
+ */
+ public Set<String> getMatchGroup() { return Collections.unmodifiableSet(matchGroup); }
+
+ /** Adds a searchable field name to be searched when this index is searched */
+ public void addToMatchGroup(String name) {
+ if (name.equals(this.name)) return;
+ if (matchGroup.size()==0) matchGroup.add(this.name);
+ matchGroup.add(name);
+ }
+
+ /**
+ * Whether this field should be stemmed in this search definition,
+ * this is never null
+ */
+ public Stemming getStemming(Search search) {
+ if (stemming!=null)
+ return stemming;
+ else
+ return search.getStemming();
+ }
+
+ /**
+ * Sets how this field should be stemmed, or set to null to use the default.
+ */
+ public void setStemming(Stemming stemming) { this.stemming=stemming; }
+
+ /** Returns whether this index supports prefix search, default is false */
+ public boolean isPrefix() { return prefix; }
+
+ /** Sets whether this index supports prefix search */
+ public void setPrefix(boolean prefix) { this.prefix=prefix; }
+
+ /** Adds an alias to this index name */
+ public void addAlias(String alias) {
+ aliases.add(alias);
+ }
+
+ /** Returns a read-only iterator of the aliases (Strings) to this index name */
+ public Iterator<String> aliasIterator() {
+ return Collections.unmodifiableSet(aliases).iterator();
+ }
+
+ public int hashCode() {
+ return name.hashCode() + ( prefix ? 17 : 0 );
+ }
+
+ public boolean equals(Object object) {
+ if ( ! (object instanceof Index)) return false;
+
+ Index other=(Index)object;
+ return
+ this.name.equals(other.name) &&
+ this.prefix==other.prefix &&
+ this.stemming==other.stemming &&
+ this.normalized==other.normalized;
+ }
+
+ public String toString() {
+ String rankTypeName=rankType==null ? "(none)" : rankType.name();
+ return
+ "index '" + name +
+ "' [ranktype: " + rankTypeName +
+ ", prefix: " + prefix + "]";
+ }
+
+ /** Makes a deep copy of this index */
+ public Object clone() {
+ try {
+ Index copy=(Index)super.clone();
+ copy.aliases=new LinkedHashSet<>(this.aliases);
+ return copy;
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Programming error",e);
+ }
+ }
+
+ public Index copy() {
+ return (Index)clone();
+ }
+
+ /**
+ * The index engine type
+ * @return the type
+ */
+ public Type getType() {
+ return type;
+ }
+
+ /**
+ * Sets the index engine type
+ * @param type a index engine type
+ */
+ public void setType(Type type) {
+ this.type = type;
+ }
+
+ /**
+ * The boolean index definition
+ * @return the boolean index definition
+ */
+ public BooleanIndexDefinition getBooleanIndexDefiniton() {
+ return boolIndex;
+ }
+
+ /**
+ * Sets the boolean index definition
+ * @param def boolean index definition
+ */
+ public void setBooleanIndexDefiniton(BooleanIndexDefinition def) {
+ boolIndex = def;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java
new file mode 100644
index 00000000000..a248ca30a8e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+
+import java.util.Map;
+
+/**
+ * Inlines macros in ranking expressions
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class MacroInliner extends ExpressionTransformer {
+
+ private final Map<String, RankProfile.Macro> macros;
+
+ public MacroInliner(Map<String, RankProfile.Macro> macros) {
+ this.macros = macros;
+ }
+
+ @Override
+ public ExpressionNode transform(ExpressionNode node) {
+ if (node instanceof ReferenceNode)
+ return transformFeatureNode((ReferenceNode)node);
+ if (node instanceof CompositeNode)
+ return transformChildren((CompositeNode)node);
+ return node;
+ }
+
+ private ExpressionNode transformFeatureNode(ReferenceNode feature) {
+ RankProfile.Macro macro = macros.get(feature.getName());
+ if (macro == null) return feature;
+ return transform(macro.getRankingExpression().getRoot()); // inline recursively and return
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
new file mode 100644
index 00000000000..3e943377611
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -0,0 +1,968 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.search.query.ranking.Diversity;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.FeatureList;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer;
+import com.yahoo.searchlib.rankingexpression.transform.Simplifier;
+import com.yahoo.config.application.api.ApplicationPackage;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Represents a rank profile - a named set of ranking settings
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class RankProfile implements Serializable, Cloneable {
+
+ /** The search definition-unique name of this rank profile */
+ private final String name;
+
+ /** The search definition owning this profile, or null if none */
+ private Search search=null;
+
+ /** The name of the rank profile inherited by this */
+ private String inheritedName = null;
+
+ /** The match settings of this profile */
+ protected MatchPhaseSettings matchPhaseSettings = null;
+
+ /** The rank settings of this profile */
+ protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>();
+
+ /** The ranking expression to be used for first phase */
+ private RankingExpression firstPhaseRanking= null;
+
+ /** The ranking expression to be used for second phase */
+ private RankingExpression secondPhaseRanking = null;
+
+ /** Number of hits to be reranked in second phase, -1 means use default */
+ private int rerankCount = -1;
+
+ /** Mysterious attribute */
+ private int keepRankCount = -1;
+
+ private int numThreadsPerSearch = -1;
+ private int numSearchPartitions = -1;
+
+ private double termwiseLimit = 1.0;
+
+ /** The drop limit used to drop hits with rank score less than or equal to this value */
+ private double rankScoreDropLimit = -Double.MAX_VALUE;
+
+ private Set<ReferenceNode> summaryFeatures;
+
+ private Set<ReferenceNode> rankFeatures;
+
+ /** The properties of this - a multimap */
+ private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>();
+
+ private Boolean ignoreDefaultRankFeatures=null;
+
+ private String secondPhaseRankingString=null;
+
+ private String firstPhaseRankingString=null;
+
+ private Map<String, Macro> macros= new LinkedHashMap<>();
+
+ private Set<String> filterFields = new HashSet<>();
+
+ private final RankProfileRegistry rankProfileRegistry;
+
+ /** Constants in ranking expressions */
+ private Map<String, Value> constants = new HashMap<>();
+
+ private final TypeSettings attributeTypes = new TypeSettings();
+
+ private final TypeSettings queryFeatureTypes = new TypeSettings();
+
+ /**
+ * Creates a new rank profile
+ *
+ * @param name the name of the new profile
+ * @param search the search definition owning this profile
+ * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles.
+ */
+ public RankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) {
+ this.name = name;
+ this.search = search;
+ this.rankProfileRegistry = rankProfileRegistry;
+ }
+
+ public String getName() { return name; }
+
+ /**
+ * Returns the search definition owning this, or null if none
+ *
+ * @return The search definition.
+ */
+ public Search getSearch() {
+ return search;
+ }
+
+ /**
+ * Sets the name of the rank profile this inherits. Both rank profiles must be present in the same search
+ * definition
+ *
+ * @param inheritedName The name of the profile that this inherits from.
+ */
+ public void setInherited(String inheritedName) {
+ this.inheritedName = inheritedName;
+ }
+
+ /**
+ * Returns the name of the profile this one inherits, or null if none is inherited
+ *
+ * @return The inherited name.
+ */
+ public String getInheritedName() {
+ return inheritedName;
+ }
+
+ /**
+ * Returns the inherited rank profile, or null if there is none
+ *
+ * @return The inherited profile.
+ */
+ public RankProfile getInherited() {
+ if (getSearch()==null) return getInheritedFromRegistry(inheritedName);
+ RankProfile inheritedInThisSearch = rankProfileRegistry.getRankProfile(search, inheritedName);
+ if (inheritedInThisSearch!=null) return inheritedInThisSearch;
+ return getInheritedFromRegistry(inheritedName);
+ }
+
+ private RankProfile getInheritedFromRegistry(String inheritedName) {
+ for (RankProfile r : rankProfileRegistry.allRankProfiles()) {
+ if (r.getName().equals(inheritedName)) {
+ return r;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns whether this profile inherits (directly or indirectly) the given profile
+ *
+ * @param name The profile name to compare this to.
+ * @return Whether or not this inherits from the named profile.
+ */
+ public boolean inherits(String name) {
+ RankProfile parent = getInherited();
+ while (parent != null) {
+ if (parent.getName().equals(name))
+ return true;
+ parent = parent.getInherited();
+ }
+ return false;
+ }
+
+ /**
+ * change match settings
+ * @param settings The new match settings
+ **/
+ public void setMatchPhaseSettings(MatchPhaseSettings settings) {
+ settings.checkValid();
+ this.matchPhaseSettings = settings;
+ }
+
+ public MatchPhaseSettings getMatchPhaseSettings() {
+ MatchPhaseSettings settings = this.matchPhaseSettings;
+ if (settings != null ) return settings;
+ if (getInherited() != null) return getInherited().getMatchPhaseSettings();
+ return null;
+ }
+
+ public void addRankSetting(RankSetting rankSetting) {
+ rankSettings.add(rankSetting);
+ }
+
+ public void addRankSetting(String fieldName, RankSetting.Type type, Object value) {
+ addRankSetting(new RankSetting(fieldName, type, value));
+ }
+
+ /**
+ * Returns the a rank setting of a field, or null if there is no such rank setting in this profile
+ *
+ * @param field The field whose settings to return.
+ * @param type The type that the field is required to be.
+ * @return The rank setting found, or null.
+ */
+ public RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) {
+ for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) {
+ RankSetting setting = i.next();
+ if (setting.getFieldName().equals(field) &&
+ setting.getType().equals(type)) {
+ return setting;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns a rank setting of field or index, or null if there is no such rank setting in this profile or one it
+ * inherits
+ *
+ * @param field The field whose settings to return.
+ * @param type The type that the field is required to be.
+ * @return The rank setting found, or null.
+ */
+ public RankSetting getRankSetting(String field, RankSetting.Type type) {
+ RankSetting rankSetting = getDeclaredRankSetting(field, type);
+ if (rankSetting != null) return rankSetting;
+
+ if (getInherited() != null) return getInherited().getRankSetting(field, type);
+
+ return null;
+ }
+
+ /**
+ * Returns the rank settings in this rank profile
+ *
+ * @return An iterator for the declared rank setting.
+ */
+ public Iterator<RankSetting> declaredRankSettingIterator() {
+ return Collections.unmodifiableSet(rankSettings).iterator();
+ }
+
+ /**
+ * Returns all settings in this profile or any profile it inherits
+ *
+ * @return An iterator for all rank settings of this.
+ */
+ public Iterator<RankSetting> rankSettingIterator() {
+ return rankSettings().iterator();
+ }
+
+ /**
+ * Returns a snapshot of the rank settings of this and everything it inherits
+ * Changes to the returned set will not be reflected in this rank profile.
+ */
+ public Set<RankSetting> rankSettings() {
+ Set<RankSetting> allSettings = new LinkedHashSet<>(rankSettings);
+ RankProfile parent = getInherited();
+ if (parent != null)
+ allSettings.addAll(parent.rankSettings());
+
+ return allSettings;
+ }
+
+ public void addConstant(String name, Value value) {
+ constants.put(name, value.freeze());
+ }
+
+ public void addConstantTensor(String name, TensorValue value) {
+ addConstant(name, value);
+ }
+
+ /**
+ * Returns an unmodifiable view of the constants to use in this.
+ */
+ public Map<String, Value> getConstants() {
+ if (constants.isEmpty())
+ return getInherited() != null ? getInherited().getConstants() : Collections.<String,Value>emptyMap();
+ if (getInherited() == null || getInherited().getConstants().isEmpty())
+ return Collections.unmodifiableMap(constants);
+
+ Map<String, Value> combinedConstants = new HashMap<>(getInherited().getConstants());
+ combinedConstants.putAll(constants);
+ return combinedConstants;
+ }
+
+ public void addAttributeType(String attributeName, String attributeType) {
+ attributeTypes.addType(attributeName, attributeType);
+ }
+
+ public Map<String, String> getAttributeTypes() {
+ return attributeTypes.getTypes();
+ }
+
+ public void addQueryFeatureType(String queryFeature, String queryFeatureType) {
+ queryFeatureTypes.addType(queryFeature, queryFeatureType);
+ }
+
+ public Map<String, String> getQueryFeatureTypes() {
+ return queryFeatureTypes.getTypes();
+ }
+
+ /**
+ * Returns the ranking expression to use by this. This expression must not be edited.
+ * Returns null if no expression is set.
+ */
+ public RankingExpression getFirstPhaseRanking() {
+ if (firstPhaseRanking!=null) return firstPhaseRanking;
+ if (getInherited()!=null) return getInherited().getFirstPhaseRanking();
+ return null;
+ }
+
+ public void setFirstPhaseRanking(RankingExpression rankingExpression) {
+ this.firstPhaseRanking=rankingExpression;
+ }
+
+ /**
+ * Returns the ranking expression to use by this. This expression must not be edited.
+ * Returns null if no expression is set.
+ */
+ public RankingExpression getSecondPhaseRanking() {
+ if (secondPhaseRanking!=null) return secondPhaseRanking;
+ if (getInherited()!=null) return getInherited().getSecondPhaseRanking();
+ return null;
+ }
+
+ public void setSecondPhaseRanking(RankingExpression rankingExpression) {
+ this.secondPhaseRanking=rankingExpression;
+ }
+
+ /**
+ * Called by parser to store the expression string, for delayed evaluation
+ * @param exp ranking expression for second phase
+ */
+ public void setSecondPhaseRankingString(String exp) {
+ this.secondPhaseRankingString = exp;
+ }
+
+ /**
+ * Called by parser to store the expression string, for delayed evaluation
+ * @param exp ranking expression for first phase
+ */
+ public void setFirstPhaseRankingString(String exp) {
+ this.firstPhaseRankingString = exp;
+ }
+
+ /** Returns a read-only view of the summary features to use in this profile. This is never null */
+ public Set<ReferenceNode> getSummaryFeatures() {
+ if (summaryFeatures!=null) return Collections.unmodifiableSet(summaryFeatures);
+ if (getInherited()!=null) return getInherited().getSummaryFeatures();
+ return Collections.emptySet();
+ }
+
+ public void addSummaryFeature(ReferenceNode feature) {
+ if (summaryFeatures==null)
+ summaryFeatures=new LinkedHashSet<>();
+ summaryFeatures.add(feature);
+ }
+
+ /**
+ * Adds the content of the given feature list to the internal list of summary features.
+ *
+ * @param features The features to add.
+ */
+ public void addSummaryFeatures(FeatureList features) {
+ for (ReferenceNode feature : features) {
+ addSummaryFeature(feature);
+ }
+ }
+
+ /** Returns a read-only view of the rank features to use in this profile. This is never null */
+ public Set<ReferenceNode> getRankFeatures() {
+ if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures);
+ if (getInherited() != null) return getInherited().getRankFeatures();
+ return Collections.emptySet();
+ }
+
+ public void addRankFeature(ReferenceNode feature) {
+ if (rankFeatures==null)
+ rankFeatures=new LinkedHashSet<>();
+ rankFeatures.add(feature);
+ }
+
+ /**
+ * Adds the content of the given feature list to the internal list of rank features.
+ *
+ * @param features The features to add.
+ */
+ public void addRankFeatures(FeatureList features) {
+ for (ReferenceNode feature : features) {
+ addRankFeature(feature);
+ }
+ }
+
+ /** Returns a read only flattened list view of the rank properties to use in this profile. This is never null. */
+ public List<RankProperty> getRankProperties() {
+ List<RankProperty> properties = new ArrayList<>();
+ for (List<RankProperty> propertyList : getRankPropertyMap().values()) {
+ properties.addAll(propertyList);
+ }
+ return Collections.unmodifiableList(properties);
+ }
+
+ /** Returns a read only map view of the rank properties to use in this profile. This is never null. */
+ public Map<String, List<RankProperty>> getRankPropertyMap() {
+ if (rankProperties.size() == 0 && getInherited() == null) return Collections.emptyMap();
+ if (rankProperties.size() == 0) return getInherited().getRankPropertyMap();
+ if (getInherited() == null) return Collections.unmodifiableMap(rankProperties);
+
+ // Neither is null
+ Map<String, List<RankProperty>> combined = new LinkedHashMap<>(getInherited().getRankPropertyMap());
+ combined.putAll(rankProperties); // Don't combine values across inherited properties
+ return Collections.unmodifiableMap(combined);
+ }
+
+ public void addRankProperty(String name, String parameter) {
+ addRankProperty(new RankProperty(name, parameter));
+ }
+
+ public void addRankProperty(RankProperty rankProperty) {
+ // Just the usual multimap semantics here
+ List<RankProperty> properties = rankProperties.get(rankProperty.getName());
+ if (properties == null) {
+ properties = new ArrayList<>(1);
+ rankProperties.put(rankProperty.getName(), properties);
+ }
+ properties.add(rankProperty);
+ }
+
+ public String toString() {
+ return "rank profile " + getName();
+ }
+
+ public int getRerankCount() {
+ if (rerankCount>=0) return rerankCount;
+ if (getInherited()!=null) return getInherited().getRerankCount();
+ return -1;
+ }
+
+ public int getNumThreadsPerSearch() {
+ return numThreadsPerSearch;
+ }
+
+ public void setNumThreadsPerSearch(int numThreads) {
+ this.numThreadsPerSearch = numThreads;
+ }
+
+ public void setNumSearchPartitions(int numSearchPartitions) {
+ this.numSearchPartitions = numSearchPartitions;
+ }
+
+ public int getNumSearchPartitions() { return numSearchPartitions; }
+
+ public double getTermwiseLimit() { return termwiseLimit; }
+ public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; }
+
+ /** Sets the rerank count. Set to -1 to use inherited */
+ public void setRerankCount(int rerankCount) {
+ this.rerankCount = rerankCount;
+ }
+
+ /** Whether we should ignore the default rank features. Set to null to use inherited */
+ public void setIgnoreDefaultRankFeatures(Boolean ignoreDefaultRankFeatures) {
+ this.ignoreDefaultRankFeatures = ignoreDefaultRankFeatures;
+ }
+
+ public boolean getIgnoreDefaultRankFeatures() {
+ if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures;
+ return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures();
+ }
+
+ /**
+ * Returns the string form of the second phase ranking expression.
+ * @return string form of second phase ranking expression
+ */
+ public String getSecondPhaseRankingString() {
+ if (secondPhaseRankingString != null) return secondPhaseRankingString;
+ if (getInherited() != null) return getInherited().getSecondPhaseRankingString();
+ return null;
+ }
+
+ /**
+ * Returns the string form of the first phase ranking expression.
+ * @return string form of first phase ranking expression
+ */
+ public String getFirstPhaseRankingString() {
+ if (firstPhaseRankingString != null) return firstPhaseRankingString;
+ if (getInherited() != null) return getInherited().getFirstPhaseRankingString();
+ return null;
+ }
+
+ public void addMacro(String name, boolean inline) {
+ macros.put(name, new Macro(name, inline));
+ }
+
+ /** Returns an unmodifiable view of the macros in this */
+ public Map<String, Macro> getMacros() {
+ if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap();
+ if (macros.size() == 0) return getInherited().getMacros();
+ if (getInherited() == null) return Collections.unmodifiableMap(macros);
+
+ // Neither is null
+ Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros());
+ allMacros.putAll(macros);
+ return Collections.unmodifiableMap(allMacros);
+
+ }
+
+ public int getKeepRankCount() {
+ if (keepRankCount>=0) return keepRankCount;
+ if (getInherited()!=null) return getInherited().getKeepRankCount();
+ return -1;
+ }
+
+ public void setKeepRankCount(int rerankArraySize) {
+ this.keepRankCount = rerankArraySize;
+ }
+
+ public double getRankScoreDropLimit() {
+ if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit;
+ if (getInherited()!=null) return getInherited().getRankScoreDropLimit();
+ return rankScoreDropLimit;
+ }
+
+ public void setRankScoreDropLimit(double rankScoreDropLimit) {
+ this.rankScoreDropLimit = rankScoreDropLimit;
+ }
+
+ public Set<String> filterFields() {
+ return filterFields;
+ }
+
+ /**
+ * Returns all filter fields in this profile and any profile it inherits.
+ * @return the set of all filter fields
+ */
+ public Set<String> allFilterFields() {
+ RankProfile parent = getInherited();
+ Set<String> retval = new LinkedHashSet<>();
+ if (parent != null) {
+ retval.addAll(parent.allFilterFields());
+ }
+ retval.addAll(filterFields());
+ return retval;
+ }
+
+ /**
+ * Will take the parser-set textual ranking expressions and turn into objects
+ */
+ public void parseExpressions() {
+ try {
+ parseRankingExpressions();
+ parseMacros();
+ } catch (ParseException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Passes the contents of macros on to parser. Then put all the implied rank properties
+ * from those macros into the profile's props map.
+ */
+ private void parseMacros() throws ParseException {
+ for (Map.Entry<String, Macro> e : getMacros().entrySet()) {
+ String macroName = e.getKey();
+ Macro macro = e.getValue();
+ RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression());
+ macro.setRankingExpression(expr);
+ macro.setTextualExpression(expr.getRoot().toString());
+ }
+ }
+
+ /**
+ * Passes ranking expressions on to parser
+ * @throws ParseException if either of the ranking expressions could not be parsed
+ */
+ private void parseRankingExpressions() throws ParseException {
+ if (getFirstPhaseRankingString() != null)
+ setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString()));
+ if (getSecondPhaseRankingString() != null)
+ setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString()));
+ }
+
+ private RankingExpression parseRankingExpression(String expName, String exp) throws ParseException {
+ if (exp.trim().length() == 0)
+ throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expName + ".");
+
+ try {
+ RankingExpression expression = new RankingExpression(openRankingExpressionReader(expName, exp.trim()));
+ expression.setName(expName);
+ return expression;
+ }
+ catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
+ ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() +
+ "' in " + getName()+ ", " + expName + ".");
+ throw (ParseException)exception.initCause(e);
+ }
+ }
+
+ private Reader openRankingExpressionReader(String expName, String expression) {
+ if ( ! expression.startsWith("file:")) return new StringReader(expression);
+
+ String fileName = expression.substring("file:".length()).trim();
+ if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX))
+ fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX;
+
+ final File file = new File(fileName);
+ if ( ! (file.isAbsolute()) && file.getPath().contains("/")) // See ticket 4102122
+ throw new IllegalArgumentException("In " + getName() +", " + expName + ", ranking references file '" + file +
+ "' in subdirectory, which is not supported.");
+
+ return search.getRankingExpression(fileName);
+ }
+
+ /** Shallow clones this */
+ @Override
+ public RankProfile clone() {
+ try {
+ // Note: This treats RankingExpression in Macros as immutables even though they are not
+ RankProfile clone = (RankProfile)super.clone();
+ clone.rankSettings = new LinkedHashSet<>(this.rankSettings);
+ clone.matchPhaseSettings = this.matchPhaseSettings; // hmm?
+ clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null;
+ clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null;
+ clone.rankProperties = new LinkedHashMap<>(this.rankProperties);
+ clone.macros = new LinkedHashMap<>(this.macros);
+ clone.filterFields = new HashSet<>(this.filterFields);
+ clone.constants = new HashMap<>(this.constants);
+ return clone;
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Won't happen", e);
+ }
+ }
+
+ /**
+ * Returns a copy of this where the content is optimized for execution.
+ * Compiled profiles should never be modified.
+ */
+ public RankProfile compile() {
+ try {
+ RankProfile compiled = this.clone();
+ compiled.compileThis();
+ return compiled;
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e);
+ }
+ }
+
+ private void compileThis() {
+ parseExpressions();
+
+ checkNameCollisions(getMacros(), getConstants());
+
+ Map<String, Macro> compiledMacros = new LinkedHashMap<>();
+ for (Map.Entry<String, Macro> macroEntry : getMacros().entrySet()) {
+ Macro compiledMacro = macroEntry.getValue().clone();
+ compiledMacro.setRankingExpression(compile(macroEntry.getValue().getRankingExpression(),
+ getConstants(), Collections.<String, Macro>emptyMap()));
+ compiledMacros.put(macroEntry.getKey(), compiledMacro);
+ }
+ macros = compiledMacros;
+ Map<String, Macro> inlineMacros = keepInline(compiledMacros);
+ firstPhaseRanking = compile(this.getFirstPhaseRanking(), getConstants(), inlineMacros);
+ secondPhaseRanking = compile(this.getSecondPhaseRanking(), getConstants(), inlineMacros);
+ }
+
+ private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) {
+ for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) {
+ if (constants.get(macroEntry.getKey()) != null)
+ throw new IllegalArgumentException("Cannot have both a constant and macro named '" + macroEntry.getKey() + "'");
+ }
+ }
+
+ private Map<String, Macro> keepInline(Map<String, Macro> macros) {
+ Map<String, Macro> inlineMacros = new HashMap<>();
+ for (Map.Entry<String, Macro> entry : macros.entrySet())
+ if (entry.getValue().getInline())
+ inlineMacros.put(entry.getKey(), entry.getValue());
+ return inlineMacros;
+ }
+
+ private RankingExpression compile(RankingExpression expression,
+ Map<String, Value> constants,
+ Map<String, Macro> inlineMacros) {
+ if (expression == null) return null;
+ Map<String, String> rankPropertiesOutput = new HashMap<>();
+ expression = new ConstantDereferencer(constants).transform(expression);
+ expression = new ConstantTensorTransformer(constants, rankPropertiesOutput).transform(expression);
+ expression = new MacroInliner(inlineMacros).transform(expression);
+ expression = new Simplifier().transform(expression);
+ for (Map.Entry<String, String> rankProperty : rankPropertiesOutput.entrySet()) {
+ addRankProperty(rankProperty.getKey(), rankProperty.getValue());
+ }
+ return expression;
+ }
+
+ /**
+ * A rank setting. The identity of a rank setting is its field name and type (not value).
+ * A rank setting is immutable.
+ */
+ public static class RankSetting implements Serializable {
+
+ private String fieldName;
+
+ private Type type;
+
+ /** The rank value */
+ private Object value;
+
+ public enum Type {
+
+ RANKTYPE("rank-type"),
+ LITERALBOOST("literal-boost"),
+ WEIGHT("weight"),
+ PREFERBITVECTOR("preferbitvector",true);
+
+ private String name;
+
+ /** True if this setting really pertains to an index, not a field within an index */
+ private boolean isIndexLevel;
+
+ private Type(String name) {
+ this(name,false);
+ }
+
+ private Type(String name,boolean isIndexLevel) {
+ this.name = name;
+ this.isIndexLevel=isIndexLevel;
+ }
+
+ /** True if this setting really pertains to an index, not a field within an index */
+ public boolean isIndexLevel() { return isIndexLevel; }
+
+ /** @return The name of this type */
+ public String getName() {
+ return name;
+ }
+
+ public String toString() {
+ return "type: " + name;
+ }
+
+ }
+
+ public RankSetting(String fieldName, RankSetting.Type type, Object value) {
+ this.fieldName = fieldName;
+ this.type = type;
+ this.value = value;
+ }
+
+ public String getFieldName() { return fieldName; }
+
+ public Type getType() { return type; }
+
+ public Object getValue() { return value; }
+
+ /** @return The value as an int, or a negative value if it is not an integer */
+ public int getIntValue() {
+ if (value instanceof Integer) {
+ return ((Integer)value);
+ }
+ else {
+ return -1;
+ }
+ }
+
+ public int hashCode() {
+ return fieldName.hashCode() + 17 * type.hashCode();
+ }
+
+ public boolean equals(Object object) {
+ if (!(object instanceof RankSetting)) {
+ return false;
+ }
+ RankSetting other = (RankSetting)object;
+ return
+ fieldName.equals(other.fieldName) &&
+ type.equals(other.type);
+ }
+
+ public String toString() {
+ return type + " setting " + fieldName + ": " + value;
+ }
+
+ }
+
+ /** A rank property. Rank properties are Value Objects */
+ public static class RankProperty implements Serializable {
+
+ private String name;
+ private String value;
+
+ public RankProperty(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ public String getName() { return name; }
+
+ public String getValue() { return value; }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode() + 17*value.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (! (object instanceof RankProperty)) return false;
+ RankProperty other=(RankProperty)object;
+ return (other.name.equals(this.name) && other.value.equals(this.value));
+ }
+
+ @Override
+ public String toString() {
+ return name + " = " + value;
+ }
+
+ }
+
+ /**
+ * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro
+ *
+ * @author vegardh
+ */
+ public static class Macro implements Serializable, Cloneable {
+
+ private String name=null;
+ private String textualExpression=null;
+ private RankingExpression expression=null;
+ private List<String> formalParams = new ArrayList<>();
+
+ /** True if this should be inlined into calling expressions. Useful for very cheap macros. */
+ private final boolean inline;
+
+ public Macro(String name, boolean inline) {
+ this.name = name;
+ this.inline = inline;
+ }
+
+ public void addParam(String name) {
+ formalParams.add(name);
+ }
+
+ public List<String> getFormalParams() {
+ return formalParams;
+ }
+
+ public String getTextualExpression() {
+ return textualExpression;
+ }
+
+ public void setTextualExpression(String textualExpression) {
+ this.textualExpression = textualExpression;
+ }
+
+ public void setRankingExpression(RankingExpression expr) {
+ this.expression=expr;
+ }
+
+ public RankingExpression getRankingExpression() {
+ return expression;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean getInline() {
+ return inline && formalParams.size() == 0; // only inline no-arg macros;
+ }
+
+ public ExpressionFunction toExpressionMacro() {
+ return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression());
+ }
+
+ @Override
+ public Macro clone() {
+ try {
+ return (Macro)super.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Won't happen", e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "macro " + getName() + ": " + expression;
+ }
+
+ }
+
+ public static final class DiversitySettings {
+ private String attribute = null;
+ private int minGroups = 0;
+ private double cutoffFactor = 10;
+ private Diversity.CutoffStrategy cutoffStrategy = Diversity.CutoffStrategy.loose;
+
+ public void setAttribute(String value) { attribute = value; }
+ public void setMinGroups(int value) { minGroups = value; }
+ public void setCutoffFactor(double value) { cutoffFactor = value; }
+ public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; }
+ public void setCutoffStrategy(String strategy) { cutoffStrategy = Diversity.CutoffStrategy.valueOf(strategy); }
+ public String getAttribute() { return attribute; }
+ public int getMinGroups() { return minGroups; }
+ public double getCutoffFactor() { return cutoffFactor; }
+ public Diversity.CutoffStrategy getCutoffStrategy() { return cutoffStrategy; }
+ public void checkValid() {
+ if (attribute == null || attribute.isEmpty()) {
+ throw new IllegalArgumentException("'diversity' did not set non-empty diversity attribute name.");
+ }
+ if (minGroups <= 0) {
+ throw new IllegalArgumentException("'diversity' did not set min-groups > 0");
+ }
+ if (cutoffFactor < 1.0) {
+ throw new IllegalArgumentException("diversity.cutoff.factor must be larger or equal to 1.0.");
+ }
+ }
+ }
+
+ public static class MatchPhaseSettings {
+ private String attribute = null;
+ private boolean ascending = false;
+ private int maxHits = 0; // try to get this many hits before degrading the match phase
+ private double maxFilterCoverage = 1.0; // Max coverage of original corpus that will trigger the filter.
+ private DiversitySettings diversity = null;
+ private double evaluationPoint = 0.20;
+ private double prePostFilterTippingPoint = 1.0;
+
+ public void setDiversity(DiversitySettings value) {
+ value.checkValid();
+ diversity = value;
+ }
+ public void setAscending(boolean value) { ascending = value; }
+ public void setAttribute(String value) { attribute = value; }
+ public void setMaxHits(int value) { maxHits = value; }
+ public void setMaxFilterCoverage(double value) { maxFilterCoverage = value; }
+ public void setEvaluationPoint(double evaluationPoint) { this.evaluationPoint = evaluationPoint; }
+ public void setPrePostFilterTippingPoint(double prePostFilterTippingPoint) { this.prePostFilterTippingPoint = prePostFilterTippingPoint; }
+
+ public boolean getAscending() { return ascending; }
+ public String getAttribute() { return attribute; }
+ public int getMaxHits() { return maxHits; }
+ public double getMaxFilterCoverage() { return maxFilterCoverage; }
+ public DiversitySettings getDiversity() { return diversity; }
+ public double getEvaluationPoint() { return evaluationPoint; }
+ public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; }
+
+ public void checkValid() {
+ if (attribute == null) {
+ throw new IllegalArgumentException("match-phase did not set any attribute");
+ }
+ if (! (maxHits > 0)) {
+ throw new IllegalArgumentException("match-phase did not set max-hits > 0");
+ }
+ }
+
+ }
+
+ public static class TypeSettings {
+
+ private final Map<String, String> types = new HashMap<>();
+
+ public void addType(String name, String type) {
+ types.put(name, type);
+ }
+
+ public Map<String, String> getTypes() {
+ return Collections.unmodifiableMap(types);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java
new file mode 100644
index 00000000000..a9ee3c3cc5c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import java.util.*;
+
+/**
+ * Mapping from name to {@link RankProfile} as well as a reverse mapping of {@link RankProfile} to {@link Search}.
+ * Having both of these mappings consolidated here will make it easier to remove dependencies on these mappings at
+ * run time, since it is essentially only used when building rank profile config at deployment time.
+ *
+ * TODO: Reconsider the difference between local and global maps. Right now, the local maps might better be
+ * served from a different class owned by SearchBuilder.
+ *
+ * @author lulf
+ * @since 5.20
+ */
+public class RankProfileRegistry {
+
+ private final Map<RankProfile, Search> rankProfileToSearch = new LinkedHashMap<>();
+ private final Map<Search, Map<String, RankProfile>> rankProfiles = new LinkedHashMap<>();
+ /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */
+ static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked"));
+
+ public RankProfileRegistry() {
+ }
+
+ public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Search search) {
+ RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+ rankProfileRegistry.addRankProfile(new DefaultRankProfile(search, rankProfileRegistry));
+ rankProfileRegistry.addRankProfile(new UnrankedRankProfile(search, rankProfileRegistry));
+ return rankProfileRegistry;
+ }
+
+ /**
+ * Adds a rank profile to this registry
+ *
+ * @param rankProfile the rank profile to add
+ */
+ public void addRankProfile(RankProfile rankProfile) {
+ if (!rankProfiles.containsKey(rankProfile.getSearch())) {
+ rankProfiles.put(rankProfile.getSearch(), new LinkedHashMap<>());
+ }
+ checkForDuplicateRankProfile(rankProfile);
+ rankProfiles.get(rankProfile.getSearch()).put(rankProfile.getName(), rankProfile);
+ rankProfileToSearch.put(rankProfile, rankProfile.getSearch());
+ }
+
+ private void checkForDuplicateRankProfile(RankProfile rankProfile) {
+ final String rankProfileName = rankProfile.getName();
+ RankProfile existingRangProfileWithSameName = rankProfiles.get(rankProfile.getSearch()).get(rankProfileName);
+ if (existingRangProfileWithSameName == null) return;
+
+ if (!overridableRankProfileNames.contains(rankProfileName)) {
+ throw new IllegalArgumentException("Cannot add rank profile '" + rankProfileName + "' in search definition '"
+ + rankProfile.getSearch().getName() + "', since it already exists");
+ }
+ }
+
+ /**
+ * Returns a named rank profile, null if the search definition doesn't have one with the given name
+ *
+ * @param search The {@link Search} that owns the rank profile.
+ * @param name The name of the rank profile
+ * @return The RankProfile to return.
+ */
+ public RankProfile getRankProfile(Search search, String name) {
+ return rankProfiles.get(search).get(name);
+ }
+
+ /**
+ * Rank profiles that are collected across clusters.
+ * @return A set of global {@link RankProfile} instances.
+ */
+ public Set<RankProfile> allRankProfiles() {
+ return rankProfileToSearch.keySet();
+ }
+
+ /**
+ * Rank profiles that are collected for a given search definition
+ * @param search {@link Search} to get rank profiles for.
+ * @return A collection of local {@link RankProfile} instances.
+ */
+ public Collection<RankProfile> localRankProfiles(Search search) {
+ Map<String, RankProfile> mapping = rankProfiles.get(search);
+ if (mapping == null) {
+ return Collections.emptyList();
+ }
+ return mapping.values();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
new file mode 100644
index 00000000000..7d8a87cee8c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.TemporarySDDocumentType;
+
+import java.util.*;
+import java.util.logging.Level;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SDDocumentTypeOrderer {
+ private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>();
+ private Set<Integer> seenTypes = new LinkedHashSet<>();
+ List<SDDocumentType> processingOrder = new LinkedList<>();
+ private DeployLogger deployLogger;
+
+ public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) {
+ this.deployLogger = deployLogger;
+ for (SDDocumentType type : sdTypes) {
+ createdSDTypes.put(type.getDocumentName(), type);
+ }
+ DocumentTypeManager dtm = new DocumentTypeManager();
+ for (DataType type : dtm.getDataTypes()) {
+ seenTypes.add(type.getId());
+ }
+
+ }
+
+ List<SDDocumentType> getOrdered() { return processingOrder; }
+
+ public void process() {
+ for (SDDocumentType type : createdSDTypes.values()) {
+ process(type);
+ }
+ }
+ private void process(SDDocumentType type) {
+ List<DataTypeName> toReplace = new ArrayList<>();
+ for (SDDocumentType sdoc : type.getInheritedTypes()) {
+ if (sdoc instanceof TemporarySDDocumentType) {
+ toReplace.add(sdoc.getDocumentName());
+ }
+ }
+ for (DataTypeName name : toReplace) {
+ SDDocumentType inherited = createdSDTypes.get(name);
+ if (inherited == null) {
+ throw new IllegalStateException("Document type '" + name + "' not found.");
+ }
+ process(inherited);
+ type.inherit(inherited);
+ }
+ visit(type);
+ }
+
+ private void visit(SDDocumentType docOrStruct) {
+ int id;
+ if (docOrStruct.isStruct()) {
+ id = new StructDataType(docOrStruct.getName()).getId();
+ } else {
+ id = new DocumentType(docOrStruct.getName()).getId();
+ }
+
+ if (seenTypes.contains(id)) {
+ return;
+ } else {
+ seenTypes.add((new StructDataType(docOrStruct.getName()).getId()));
+ }
+
+
+ for (Field field : docOrStruct.fieldSet()) {
+ if (!seenTypes.contains(field.getDataType().getId())) {
+ //we haven't seen this before, do it
+ visit(field.getDataType());
+ }
+ }
+ processingOrder.add(docOrStruct);
+ }
+
+ private SDDocumentType find(String name) {
+ SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name));
+ if (sdDocType != null) {
+ return sdDocType;
+ }
+ for(SDDocumentType sdoc : createdSDTypes.values()) {
+ for (SDDocumentType stype : sdoc.getTypes()) {
+ if (stype.getName().equals(name)) {
+ return stype;
+ }
+ }
+ }
+ return null;
+ }
+ private void visit(DataType type) {
+ if (type instanceof StructuredDataType) {
+ StructuredDataType structType = (StructuredDataType) type;
+ SDDocumentType sdDocType = find(structType.getName());
+ if (sdDocType == null) {
+ throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'.");
+ }
+ visit(sdDocType);
+ return;
+ }
+
+ if (type instanceof MapDataType) {
+ MapDataType mType = (MapDataType) type;
+ visit(mType.getValueType());
+ visit(mType.getKeyType());
+ } else if (type instanceof WeightedSetDataType) {
+ WeightedSetDataType wType = (WeightedSetDataType) type;
+ visit(wType.getNestedType());
+ } else if (type instanceof CollectionDataType) {
+ CollectionDataType cType = (CollectionDataType) type;
+ visit(cType.getNestedType());
+ } else if (type instanceof AnnotationReferenceDataType) {
+ //do nothing
+ } else if (type instanceof PrimitiveDataType) {
+ //do nothing
+ } else {
+ deployLogger.log(Level.WARNING, "Unknown type : " + type);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
new file mode 100644
index 00000000000..43c6bb4b441
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -0,0 +1,555 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+import java.io.Reader;
+import java.io.Serializable;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * <p>A search definition describes (or uses) some document types, defines how these are turned into a relevancy tuned
+ * index through indexing and how data from documents should be served at search time.</p> <p>The identity of this
+ * class is its name.</p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+// TODO: Make a class owned by this, for each of these responsibilities:
+// Managing indexes, managing attributes, managing summary classes.
+// Ensure that after the processing step, all implicit instances of the above types are explicitly represented
+public class Search implements Serializable {
+
+ private static final Logger log = Logger.getLogger(Search.class.getName());
+ private static final String SD_DOC_FIELD_NAME = "sddocname";
+ private static final List<String> RESERVED_NAMES = Arrays.asList(
+ "index", "index_url", "summary", "attribute", "select_input", "host", "documentid",
+ "position", "split_foreach", "tokenize", "if", "else", "switch", "case", SD_DOC_FIELD_NAME, "relevancy");
+
+ /**
+ * @return True if the given field name is a reserved name.
+ */
+ public static boolean isReservedName(String name) {
+ return RESERVED_NAMES.contains(name);
+ }
+
+ // Field sets
+ private FieldSets fieldSets = new FieldSets();
+
+ // Whether or not this object has been processed.
+ private boolean processed;
+
+ // The unique name of this search definition.
+ private String name;
+
+ // True if this doesn't define a search, just some documents.
+ private boolean documentsOnly = false;
+
+ // The stemming setting of this search definition. Default is SHORTEST.
+ private Stemming stemming = Stemming.SHORTEST;
+
+ // Documents contained in this definition.
+ private SDDocumentType docType;
+
+ // The extra fields of this search definition.
+ private Map<String, SDField> fields = new LinkedHashMap<>();
+
+ // The explicitly defined indices of this search definition.
+ private Map<String, Index> indices = new LinkedHashMap<>();
+
+ // The explicitly defined summaries of this search definition.
+ private Map<String, DocumentSummary> summaries = new LinkedHashMap<>();
+ // _Must_ preserve order
+
+ private ApplicationPackage sourceApplication;
+
+ /**
+ * Creates a search definition which just holds a set of documents which should not (here, directly) be searchable
+ */
+ protected Search() {
+ documentsOnly = true;
+ }
+
+ /**
+ * Creates a proper search definition
+ * @param name of the the searchdefinition
+ * @param sourceApplication the application containing this
+ */
+ public Search(String name, ApplicationPackage sourceApplication) {
+ this.sourceApplication = sourceApplication;
+ this.name = name;
+ }
+
+ protected void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns true if this doesn't define a search, just some documents
+ *
+ * @return if the searchdefinition only has documents
+ */
+ public boolean isDocumentsOnly() {
+ return documentsOnly;
+ }
+
+ /**
+ * Sets the stemming default of fields. Default is ALL
+ *
+ * @param stemming set default stemming for this searchdefinition
+ * @throws NullPointerException if this is attempted set to null
+ */
+ public void setStemming(Stemming stemming) {
+ if (stemming == null) {
+ throw new NullPointerException("The stemming setting of a search definition " +
+ "can not be null");
+ }
+ this.stemming = stemming;
+ }
+
+ /**
+ * Returns whether fields should be stemmed by default or not. Default is ALL. This is never null.
+ *
+ * @return the default stemming for this searchdefinition
+ */
+ public Stemming getStemming() {
+ return stemming;
+ }
+
+ /**
+ * Adds a document type which is defined in this search definition
+ *
+ * @param document the document type to add
+ */
+ public void addDocument(SDDocumentType document) {
+ if (docType != null) {
+ throw new IllegalArgumentException("Searchdefinition cannot have more than one document");
+ }
+ docType = document;
+ }
+
+ /**
+ * Gets a document from this search definition
+ *
+ * @param name the name of the document to return
+ * @return the contained or used document type, or null if there is no such document
+ */
+ public SDDocumentType getDocument(String name) {
+ if (docType != null && name.equals(docType.getName())) {
+ return docType;
+ }
+ return null;
+ }
+
+ /**
+ * @return true if the document has been added.
+ */
+ public boolean hasDocument() {
+ return docType != null;
+ }
+
+ /**
+ * @return The document in this search.
+ */
+ public SDDocumentType getDocument() {
+ return docType;
+ }
+
+ /**
+ * Returns a list of all the fields of this search definition, that is all fields in all documents, in the documents
+ * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not
+ * impact this Search
+ *
+ * @return the list of fields in this searchdefinition
+ */
+ public List<SDField> allFieldsList() {
+ List<SDField> allFields = new ArrayList<>();
+ allFields.addAll(extraFieldList());
+ for (Field field : docType.fieldSet()) {
+ allFields.add((SDField)field);
+ }
+ return allFields;
+ }
+
+ /**
+ * Returns the content of a ranking expression file
+ */
+ public Reader getRankingExpression(String fileName) {
+ return sourceApplication.getRankingExpression(fileName);
+ }
+
+ /**
+ * Returns a field defined in this search definition or one if its documents. Fields in this search definition takes
+ * precedence over document fields having the same name
+ *
+ * @param name of the field
+ * @return the SDField representing the field
+ */
+ public SDField getField(String name) {
+ SDField field = getExtraField(name);
+ if (field != null) {
+ return field;
+ }
+ return (SDField)docType.getField(name);
+ }
+
+ /**
+ * Returns a field defined in one of the documents of this search definition. This does <b>not</b> include the extra
+ * fields defined outside of a document (those accessible through the getExtraField() method).
+ *
+ * @param name The name of the field to return.
+ * @return The named field, or null if not found.
+ */
+ public SDField getDocumentField(String name) {
+ return (SDField)docType.getField(name);
+ }
+
+ /**
+ * Adds an extra field of this search definition not contained in a document
+ *
+ * @param field to add to the searchdefinitions list of external fields.
+ */
+ public void addExtraField(SDField field) {
+ if (fields.containsKey(field.getName())) {
+ log.warning("Duplicate field " + field.getName() + " in search definition " + getName());
+ } else {
+ field.setIsExtraField(true);
+ fields.put(field.getName(), field);
+ }
+ }
+
+ public Collection<SDField> extraFieldList() {
+ return fields.values();
+ }
+ public Collection<SDField> allExtraFields() {
+ Map<String, SDField> extraFields = new TreeMap<>();
+ for (Field field : docType.fieldSet()) {
+ SDField sdField = (SDField) field;
+ if (sdField.isExtraField()) {
+ extraFields.put(sdField.getName(), sdField);
+ }
+ }
+ for (SDField field : extraFieldList()) {
+ extraFields.put(field.getName(), field);
+ }
+ return extraFields.values();
+ }
+
+ /**
+ * Returns a field by name, or null if it is not present
+ *
+ * @param fieldName the name of the external field to get
+ * @return the SDField of this name
+ */
+ public SDField getExtraField(String fieldName) {
+ return fields.get(fieldName);
+ }
+
+ /**
+ * Adds an explicitly defined index to this search definition
+ *
+ * @param index the index to add
+ */
+ public void addIndex(Index index) {
+ indices.put(index.getName(), index);
+ }
+
+ /**
+ * <p>Returns an index, or null if no index with this name has had some <b>explicit settings</b> applied. Even if
+ * this returns null, the index may be implicitly defined by an indexing statement.</p>
+ * <p>This will return the
+ * index whether it is defined on this search or on one of its fields</p>
+ *
+ * @param name the name of the index to get
+ * @return the index requested
+ */
+ public Index getIndex(String name) {
+ List<Index> sameIndices = new ArrayList<>(1);
+ Index searchIndex = indices.get(name);
+ if (searchIndex != null) {
+ sameIndices.add(searchIndex);
+ }
+
+ for (SDField field : allFieldsList()) {
+ Index index = field.getIndex(name);
+ if (index != null) {
+ sameIndices.add(index);
+ }
+ }
+ if (sameIndices.size() == 0) {
+ return null;
+ }
+ if (sameIndices.size() == 1) {
+ return sameIndices.get(0);
+ }
+ return consolidateIndices(sameIndices);
+ }
+
+ public boolean existsIndex(String name) {
+ if (indices.get(name) != null) {
+ return true;
+ }
+ for (SDField field : allFieldsList()) {
+ if (field.existsIndex(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Consolidates a set of index settings for the same index into one
+ *
+ * @param indices The list of indexes to consolidate.
+ * @return The consolidated index
+ */
+ private Index consolidateIndices(List<Index> indices) {
+ Index first = indices.get(0);
+ Index consolidated = new Index(first.getName());
+ consolidated.setRankType(first.getRankType());
+ consolidated.setType(first.getType());
+ for (Index current : indices) {
+ if (current.isPrefix()) {
+ consolidated.setPrefix(true);
+ }
+
+ if (consolidated.getRankType() == null) {
+ consolidated.setRankType(current.getRankType());
+ } else {
+ if (current.getRankType() != null &&
+ !consolidated.getRankType().equals(current.getRankType()))
+ {
+ log.warning("Conflicting rank type settings for " +
+ first.getName() + " in " + this + ", using " +
+ consolidated.getRankType());
+ }
+ }
+
+ for (Iterator<String> j = current.aliasIterator(); j.hasNext();) {
+ consolidated.addAlias(j.next());
+ }
+ }
+ return consolidated;
+ }
+
+ /**
+ * All explicitly defined indices, both on this search definition itself (returned first) and all its fields
+ *
+ * @return The list of explicit defined indexes.
+ */
+ public List<Index> getExplicitIndices() {
+ List<Index> allIndices = new ArrayList<>(indices.values());
+ for (SDField field : allFieldsList()) {
+ for (Index index : field.getIndices().values()) {
+ allIndices.add(index);
+ }
+ }
+ return Collections.unmodifiableList(allIndices);
+ }
+
+ /**
+ * Adds an explicitly defined summary to this search definition
+ *
+ * @param summary The summary to add.
+ */
+ public void addSummary(DocumentSummary summary) {
+ summaries.put(summary.getName(), summary);
+ }
+
+ /**
+ * <p>Returns a summary class defined by this search definition, or null if no summary with this name is defined.
+ * The default summary, named "default" is always present.</p>
+ *
+ * @param name the name of the summary to get.
+ * @return Summary found.
+ */
+ public DocumentSummary getSummary(String name) {
+ return summaries.get(name);
+ }
+
+ /**
+ * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly
+ * or explicitly) in any summary class.
+ *
+ * @param name The name of the summaryfield to get.
+ * @return SummaryField to return.
+ */
+ public SummaryField getSummaryField(String name) {
+ for (DocumentSummary summary : summaries.values()) {
+ SummaryField summaryField = summary.getSummaryField(name);
+ if (summaryField != null) {
+ return summaryField;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the first explicit instance found of a summary field with this name, or null if not present explicitly in
+ * any summary class
+ *
+ * @param name Thge name of the explicit summary field to get.
+ * @return The SummaryField found.
+ */
+ public SummaryField getExplicitSummaryField(String name) {
+ for (DocumentSummary summary : summaries.values()) {
+ SummaryField summaryField = summary.getSummaryField(name);
+ if (summaryField != null && !summaryField.isImplicit()) {
+ return summaryField;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Summaries defined by fields of this search definition. The default summary, named "default", is always the first
+ * one in the returned iterator.
+ *
+ * @return The map of document summaries.
+ */
+ public Map<String, DocumentSummary> getSummaries() {
+ return summaries;
+ }
+
+ /**
+ * <p>Returns all summary fields, of all document summaries, which has the given field as source. If there are
+ * multiple summary fields with the same name, the last one will be used (they should all have the same content, if
+ * this is a valid search definition).</p> <p>The map gets owned by the receiver.</p>
+ *
+ * @param field The source field.
+ * @return The map of summary fields found.
+ */
+ public Map<String, SummaryField> getSummaryFields(SDField field) {
+ Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>();
+ for (DocumentSummary documentSummary : summaries.values()) {
+ for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+ if (summaryField.hasSource(field.getName())) {
+ summaryFields.put(summaryField.getName(), summaryField);
+ }
+ }
+ }
+ return summaryFields;
+ }
+
+ /**
+ * <p>Returns one summary field for each summary field name. If there are multiple summary fields with the same
+ * name, the last one will be used. Multiple fields of the same name should all have the same content in a valid
+ * search definition, except from the destination set. So this method can be used for all summary handling except
+ * processing the destination set.</p> <p>The map gets owned by the receiver.</p>
+ *
+ * @return Map of unique summary fields
+ */
+ public Map<String, SummaryField> getUniqueNamedSummaryFields() {
+ Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>();
+ for (DocumentSummary documentSummary : summaries.values()) {
+ for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+ summaryFields.put(summaryField.getName(), summaryField);
+ }
+ }
+ return summaryFields;
+ }
+
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ /**
+ * Returns the first occurence of an attribute having this name, or null if none
+ *
+ * @param name Name of attribute
+ * @return The Attribute with given name.
+ */
+ public Attribute getAttribute(String name) {
+ for (SDField field : allFieldsList()) {
+ Attribute attribute = field.getAttributes().get(name);
+ if (attribute != null) {
+ return attribute;
+ }
+ }
+ return null;
+ }
+
+ public boolean equals(Object o) {
+ if (!(o instanceof Search)) {
+ return false;
+ }
+
+ Search other = (Search)o;
+ return getName().equals(other.getName());
+ }
+
+ public String toString() {
+ return "search definition '" + getName() + "'";
+ }
+
+ public boolean isAccessingDiskSummary(SummaryField field) {
+ if (!field.getTransform().isInMemory()) {
+ return true;
+ }
+ if (field.getSources().size() == 0) {
+ return isAccessingDiskSummary(getName());
+ }
+ for (SummaryField.Source source : field.getSources()) {
+ if (isAccessingDiskSummary(source.getName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean isAccessingDiskSummary(String source) {
+ SDField field = getField(source);
+ if (field == null) {
+ return false;
+ }
+ if (field.doesSummarying() && !field.doesAttributing()) {
+ return true;
+ }
+ return false;
+ }
+
+ public void process() {
+ if (processed) {
+ throw new IllegalStateException("Search '" + getName() + "' already processed.");
+ }
+ processed = true;
+ }
+
+ public boolean isProcessed() {
+ return processed;
+ }
+
+ /**
+ * The field set settings for this search
+ *
+ * @return field set settings for this
+ */
+ public FieldSets fieldSets() {
+ return fieldSets;
+ }
+
+ /**
+ * For adding structs defined in document scope
+ *
+ * @param dt The struct to add.
+ * @return self, for chaining
+ */
+ public Search addType(SDDocumentType dt) {
+ docType.addType(dt); // TODO This is a very very dirty thing. It must go
+ return this;
+ }
+
+ public Search addAnnotation(SDAnnotationType dt) {
+ docType.addAnnotation(dt);
+ return this;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
new file mode 100644
index 00000000000..9c9bc559dc3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -0,0 +1,429 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.io.IOUtils;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.searchdefinition.derived.SearchOrderer;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.searchdefinition.parser.SDParser;
+import com.yahoo.searchdefinition.parser.SimpleCharStream;
+import com.yahoo.searchdefinition.parser.TokenMgrError;
+import com.yahoo.searchdefinition.processing.Processing;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+
+/**
+ * Helper class for importing {@link Search} objects in an unambiguous way. The pattern for using this is to 1) Import
+ * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank
+ * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built
+ * search objects using the {@link #getSearch(String)} method.
+ *
+ * @author TODO: Who created this?
+ */
+// TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage
+public class SearchBuilder {
+
+ private final DocumentTypeManager docTypeMgr = new DocumentTypeManager();
+ private List<Search> searchList = new LinkedList<>();
+ private ApplicationPackage app = null;
+ private boolean isBuilt = false;
+ private DocumentModel model = new DocumentModel();
+ private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+
+ public SearchBuilder() {
+ this.app = MockApplicationPackage.createEmpty();
+ }
+
+ public SearchBuilder(ApplicationPackage app) {
+ this.app = app;
+ }
+
+ public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) {
+ this.app = app;
+ this.rankProfileRegistry = rankProfileRegistry;
+ }
+
+ public SearchBuilder(RankProfileRegistry rankProfileRegistry) {
+ this(MockApplicationPackage.createEmpty(), rankProfileRegistry);
+ }
+
+ /**
+ * Import search definition.
+ *
+ * @param fileName The name of the file to import.
+ * @param deployLogger Logger for deploy messages.
+ * @return The name of the imported object.
+ * @throws IOException Thrown if the file can not be read for some reason.
+ * @throws ParseException Thrown if the file does not contain a valid search definition. ```
+ */
+ public String importFile(String fileName, DeployLogger deployLogger) throws IOException, ParseException {
+ File file = new File(fileName);
+ return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent(), deployLogger);
+ }
+
+ /**
+ * Import search definition.
+ *
+ * @param fileName The name of the file to import.
+ * @return The name of the imported object.
+ * @throws IOException Thrown if the file can not be read for some reason.
+ * @throws ParseException Thrown if the file does not contain a valid search definition.
+ */
+ public String importFile(String fileName) throws IOException, ParseException {
+ return importFile(fileName, new BaseDeployLogger());
+ }
+ public String importFile(Path file) throws IOException, ParseException {
+ return importFile(file.toString(), new BaseDeployLogger());
+ }
+
+ /**
+ * Reads and parses the search definition string provided by the given reader. Once all search definitions have been
+ * imported, call {@link #build()}.
+ *
+ * @param reader The reader whose content to import.
+ * @param searchDefDir The path to use when resolving file references.
+ * @return The name of the imported object.
+ * @throws ParseException Thrown if the file does not contain a valid search definition.
+ */
+ public String importReader(NamedReader reader, String searchDefDir, DeployLogger deployLogger) throws IOException, ParseException {
+ return importString(IOUtils.readAll(reader), searchDefDir, deployLogger);
+ }
+
+ /**
+ * See #{@link #importReader}
+ *
+ * Convenience, should only be used for testing as logs will be swallowed.
+ */
+ public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException {
+ return importString(IOUtils.readAll(reader), searchDefDir, new BaseDeployLogger());
+ }
+
+ /**
+ * Import search definition.
+ *
+ * @param str the string to parse.
+ * @return the name of the imported object.
+ * @throws ParseException thrown if the file does not contain a valid search definition.
+ */
+ public String importString(String str) throws ParseException {
+ return importString(str, null, new BaseDeployLogger());
+ }
+
+ private String importString(String str, String searchDefDir, DeployLogger deployLogger) throws ParseException {
+ Search search;
+ SimpleCharStream stream = new SimpleCharStream(str);
+ try {
+ search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir);
+ } catch (TokenMgrError e) {
+ throw new ParseException("Unknown symbol: " + e.getMessage());
+ } catch (ParseException pe) {
+ throw new ParseException(stream.formatException(pe.getMessage()));
+ }
+ return importRawSearch(search);
+ }
+
+ /**
+ * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A
+ * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most
+ * programmatically constructed search objects used in unit tests.
+ *
+ * @param rawSearch The object to import.
+ * @return The name of the imported object.
+ * @throws IllegalArgumentException Thrown if the given search object has already been processed.
+ */
+ public String importRawSearch(Search rawSearch) {
+ if (rawSearch.getName() == null) {
+ throw new IllegalArgumentException("Search has no name.");
+ }
+ String rawName = rawSearch.getName();
+ if (rawSearch.isProcessed()) {
+ throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+ "' has already been processed.");
+ }
+ for (Search search : searchList) {
+ if (rawName.equals(search.getName())) {
+ throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+ "' has already been added.");
+ }
+ }
+ searchList.add(rawSearch);
+ return rawName;
+ }
+
+ /**
+ * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A
+ * {@link Search} object is considered to be "processed" if it has not already been processed. This is the case for most
+ * programmatically constructed search objects used in unit tests.
+ *
+ * @param processed The object to import.
+ * @return The name of the imported object.
+ * @throws IllegalArgumentException Thrown if the given search object has already been processed.
+ */
+ public String importProcessedSearch(Search processed) {
+ if (processed.getName() == null) {
+ throw new IllegalArgumentException("Search has no name.");
+ }
+ String rawName = processed.getName();
+ if (!processed.isProcessed()) {
+ throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+ "' has not been processed.");
+ }
+ for (Search search : searchList) {
+ if (rawName.equals(search.getName())) {
+ throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+ "' has already been added.");
+ }
+ }
+ searchList.add(processed);
+ return rawName;
+ }
+
+ /**
+ * Only for testing.
+ *
+ * Processes and finalizes the imported search definitions so that they become available through the {@link
+ * #getSearch(String)} method.
+ *
+ * @throws IllegalStateException Thrown if this method has already been called.
+ */
+ public void build() {
+ build(new BaseDeployLogger(), new QueryProfiles());
+ }
+
+ /**
+ * Processes and finalizes the imported search definitions so that they become available through the {@link
+ * #getSearch(String)} method.
+ *
+ * @throws IllegalStateException Thrown if this method has already been called.
+ * @param deployLogger The logger to use during build
+ * @param queryProfiles The query profiles contained in the application this search is part of.
+ */
+ public void build(DeployLogger deployLogger, QueryProfiles queryProfiles) {
+ if (isBuilt) {
+ throw new IllegalStateException("Searches already built.");
+ }
+ List<Search> built = new ArrayList<>();
+ List<SDDocumentType> sdocs = new ArrayList<>();
+ sdocs.add(SDDocumentType.VESPA_DOCUMENT);
+ for (Search search : searchList) {
+ if (search.hasDocument()) {
+ sdocs.add(search.getDocument());
+ }
+ }
+ SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+ orderer.process();
+ for (SDDocumentType sdoc : orderer.getOrdered()) {
+ new FieldOperationApplierForStructs().process(sdoc);
+ new FieldOperationApplier().process(sdoc);
+ }
+
+ DocumentModelBuilder builder = new DocumentModelBuilder(model);
+ for (Search search : new SearchOrderer().order(searchList)) {
+ new FieldOperationApplierForSearch().process(search);
+ // These two needed for a couple of old unit tests, ideally these are just read from app
+ process(search, deployLogger, queryProfiles);
+ built.add(search);
+ }
+ builder.addToModel(searchList);
+ if ( ! builder.valid() ) {
+ throw new IllegalArgumentException("Impossible to build a correct model.");
+ }
+ searchList = built;
+ isBuilt = true;
+ }
+
+ /**
+ * Processes and returns the given {@link Search} object. This method has been factored out of the {@link
+ * #build()} method so that subclasses can choose not to build anything.
+ *
+ * @param search The object to build.
+ */
+ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) {
+ Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ /**
+ * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object
+ * built. This method will never return null.
+ *
+ * @return The build object.
+ * @throws IllegalStateException Thrown if there is not exactly one search.
+ */
+ public Search getSearch() {
+ if ( ! isBuilt) throw new IllegalStateException("Searches not built.");
+ if (searchList.size() != 1)
+ throw new IllegalStateException("This call only works if we have 1 search definition. Search definitions: " + searchList);
+
+ return searchList.get(0);
+ }
+
+ public DocumentModel getModel() {
+ return model;
+ }
+
+ /**
+ * Returns the built {@link Search} object that has the given name. If the name is unknown, this method will simply
+ * return null.
+ *
+ * @param name the name of the search definition to return,
+ * or null to return the only one or throw an exception if there are multiple to choose from
+ * @return the built object, or null if none with this name
+ * @throws IllegalStateException if {@link #build()} has not been called.
+ */
+ public Search getSearch(String name) {
+ if ( ! isBuilt) throw new IllegalStateException("Searches not built.");
+ if (name == null) return getSearch();
+
+ for (Search search : searchList)
+ if (search.getName().equals(name)) return search;
+ return null;
+ }
+
+ /**
+ * Convenience method to return a list of all built {@link Search} objects.
+ *
+ * @return The list of built searches.
+ */
+ public List<Search> getSearchList() {
+ return new ArrayList<>(searchList);
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a string.
+ *
+ * @param sd The string to build from.
+ * @return The built {@link SearchBuilder} object.
+ * @throws ParseException Thrown if there was a problem parsing the string.
+ */
+ public static SearchBuilder createFromString(String sd) throws ParseException {
+ SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty());
+ builder.importString(sd);
+ builder.build();
+ return builder;
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a file. Only for testing.
+ *
+ * @param fileName the file to build from
+ * @return the built {@link SearchBuilder} object
+ * @throws IOException if there was a problem reading the file.
+ * @throws ParseException if there was a problem parsing the file content.
+ */
+ public static SearchBuilder createFromFile(String fileName) throws IOException, ParseException {
+ return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry());
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a file.
+ *
+ * @param fileName the file to build from.
+ * @param deployLogger logger for deploy messages.
+ * @param rankProfileRegistry registry for rank profiles.
+ * @return the built {@link SearchBuilder} object.
+ * @throws IOException if there was a problem reading the file.
+ * @throws ParseException if there was a problem parsing the file content.
+ */
+ public static SearchBuilder createFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry)
+ throws IOException, ParseException {
+ SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), rankProfileRegistry);
+ builder.importFile(fileName);
+ builder.build(deployLogger, new QueryProfiles());
+ return builder;
+ }
+
+ public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException {
+ return createFromDirectory(dir, new RankProfileRegistry());
+ }
+ public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException {
+ SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry);
+ for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) {
+ builder.importFile(i.next());
+ }
+ builder.build(new BaseDeployLogger(), new QueryProfiles());
+ return builder;
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a file. Only for testing.
+ *
+ * @param fileName The file to build from.
+ * @return The built {@link Search} object.
+ * @throws IOException Thrown if there was a problem reading the file.
+ * @throws ParseException Thrown if there was a problem parsing the file content.
+ */
+ public static Search buildFromFile(String fileName) throws IOException, ParseException {
+ return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry());
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a file.
+ *
+ * @param fileName The file to build from.
+ * @param rankProfileRegistry Registry for rank profiles.
+ * @return The built {@link Search} object.
+ * @throws IOException Thrown if there was a problem reading the file.
+ * @throws ParseException Thrown if there was a problem parsing the file content.
+ */
+ public static Search buildFromFile(String fileName, RankProfileRegistry rankProfileRegistry)
+ throws IOException, ParseException {
+ return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry);
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a file.
+ *
+ * @param fileName The file to build from.
+ * @param deployLogger Logger for deploy messages.
+ * @param rankProfileRegistry Registry for rank profiles.
+ * @return The built {@link Search} object.
+ * @throws IOException Thrown if there was a problem reading the file.
+ * @throws ParseException Thrown if there was a problem parsing the file content.
+ */
+ public static Search buildFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry)
+ throws IOException, ParseException {
+ return createFromFile(fileName, deployLogger, rankProfileRegistry).getSearch();
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a raw object.
+ *
+ * @param rawSearch The raw object to build from.
+ * @return The built {@link SearchBuilder} object.
+ * @see #importRawSearch(Search)
+ */
+ public static SearchBuilder createFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) {
+ SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
+ builder.importRawSearch(rawSearch);
+ builder.build();
+ return builder;
+ }
+
+ /**
+ * Convenience factory method to import and build a {@link Search} object from a raw object.
+ *
+ * @param rawSearch The raw object to build from.
+ * @return The built {@link Search} object.
+ * @see #importRawSearch(Search)
+ */
+ public static Search buildFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) {
+ return createFromRawSearch(rawSearch, rankProfileRegistry).getSearch();
+ }
+
+ public RankProfileRegistry getRankProfileRegistry() {
+ return rankProfileRegistry;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java
new file mode 100644
index 00000000000..3df0c1f9953
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.io.IOException;
+
+/**
+ * A SearchBuilder that does not run the processing chain for searches
+ *
+ */
+public class UnprocessingSearchBuilder extends SearchBuilder {
+
+ public UnprocessingSearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) {
+ super(app, rankProfileRegistry);
+ }
+
+ public UnprocessingSearchBuilder() {
+ super();
+ }
+
+ public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry) {
+ super(rankProfileRegistry);
+ }
+
+ @Override
+ public void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) {
+ // empty
+ }
+
+ public static Search buildUnprocessedFromFile(String fileName) throws IOException, ParseException {
+ SearchBuilder builder = new UnprocessingSearchBuilder();
+ builder.importFile(fileName);
+ builder.build();
+ return builder.getSearch();
+ }
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java
new file mode 100644
index 00000000000..beab254bd89
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+
+/**
+ * A search that was derived from an sd file containing no search element(s), only
+ * document specifications.
+ *
+ * @author vegardh
+ *
+ */
+ // Award for best class name goes to ...
+public class UnproperSearch extends Search {
+ // This class exists because the parser accepts SD files without search { ... , and
+ // there are unit tests using it too, BUT there are many nullpointer bugs if you try to
+ // deploy such a file. Using this class to try to catch those.
+ // TODO: Throw away this when we properly support doc-only SD files.
+
+ public UnproperSearch() {
+ // empty
+ }
+
+ @Override
+ public void addDocument(SDDocumentType docType) {
+ if (getName() == null) {
+ setName(docType.getName());
+ }
+ super.addDocument(docType);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java
new file mode 100644
index 00000000000..f465112fbaf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+
+/**
+ * A low-cost ranking profile to use for watcher queries etc.
+ *
+ * @author <a href="mailto:vegardh@yahoo-inc.com">Vegard Havdal</a>
+ */
+public class UnrankedRankProfile extends RankProfile {
+
+ public UnrankedRankProfile(Search search, RankProfileRegistry rankProfileRegistry) {
+ super("unranked", search, rankProfileRegistry);
+ try {
+ RankingExpression exp = new RankingExpression("value(0)");
+ this.setFirstPhaseRanking(exp);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse the ranking expression 'value(0)' when setting up " +
+ "the 'unranked' rank profile");
+ }
+ this.setIgnoreDefaultRankFeatures(true);
+ this.setKeepRankCount(0);
+ this.setRerankCount(0);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
new file mode 100644
index 00000000000..1130e5630a3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.Ranking;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression;
+
+import java.util.*;
+
+/**
+ * The set of all attribute fields defined by a search definition
+ *
+ * @author <a href="mailto:bratseth@overture.com">Jon S Bratseth</a>
+ */
+public class AttributeFields extends Derived implements AttributesConfig.Producer {
+ private Map<String, Attribute> attributes = new java.util.LinkedHashMap<>();
+
+ /**
+ * Flag indicating if a position-attribute has been found
+ */
+ private boolean hasPosition = false;
+
+ public AttributeFields(Search search) {
+ derive(search);
+ }
+
+ /**
+ * Derives everything from a field
+ */
+ protected void derive(SDField field, Search search) {
+ if (field.usesStructOrMap() &&
+ !field.getDataType().equals(PositionDataType.INSTANCE) &&
+ !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
+ {
+ return; // Ignore struct fields for indexed search (only implemented for streaming search)
+ }
+ deriveAttributes(field);
+ }
+
+ /**
+ * Return an attribute by name, or null if it doesn't exist
+ */
+ public Attribute getAttribute(String attributeName) {
+ return attributes.get(attributeName);
+ }
+
+ public boolean containsAttribute(String attributeName) {
+ return getAttribute(attributeName) != null;
+ }
+
+ /**
+ * Derives one attribute. TODO: Support non-default named attributes
+ */
+ private void deriveAttributes(SDField field) {
+ for (Attribute fieldAttribute : field.getAttributes().values()) {
+ Attribute attribute = getAttribute(fieldAttribute.getName());
+ if (attribute == null) {
+ attributes.put(fieldAttribute.getName(), fieldAttribute);
+ attribute = getAttribute(fieldAttribute.getName());
+ }
+ Ranking ranking = field.getRanking();
+ if (ranking != null && ranking.isFilter()) {
+ attribute.setEnableBitVectors(true);
+ attribute.setEnableOnlyBitVector(true);
+ }
+ }
+
+ if (field.containsExpression(ToPositionExpression.class)) {
+ // TODO: Move this check to processing and remove this
+ if (hasPosition) {
+ throw new IllegalArgumentException("Can not specify more than one " +
+ "set of position attributes per " + "field: " + field.getName());
+ }
+ hasPosition = true;
+ }
+ }
+
+ /**
+ * Returns a read only attribute iterator
+ */
+ public Iterator attributeIterator() {
+ return attributes().iterator();
+ }
+
+ public Collection<Attribute> attributes() {
+ return Collections.unmodifiableCollection(attributes.values());
+ }
+
+ public String toString() {
+ return "attributes " + getName();
+ }
+
+ protected String getDerivedName() {
+ return "attributes";
+ }
+
+ private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) {
+ Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>();
+ for (AttributesConfig.Attribute.Builder builder : ls) {
+ ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder);
+ }
+ return ret;
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ for (Attribute attribute : attributes.values()) {
+ AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder()
+ .name(attribute.getName())
+ .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName()))
+ .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName()));
+ if (attribute.isRemoveIfZero()) {
+ aaB.removeifzero(true);
+ }
+ if (attribute.isCreateIfNonExistent()) {
+ aaB.createifnonexistent(true);
+ }
+ aaB.enablebitvectors(attribute.isEnabledBitVectors());
+ aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector());
+ if (attribute.isFastSearch()) {
+ aaB.fastsearch(true);
+ }
+ if (attribute.isFastAccess()) {
+ aaB.fastaccess(true);
+ }
+ if (attribute.isHuge()) {
+ aaB.huge(true);
+ }
+ if (attribute.getSorting().isDescending()) {
+ aaB.sortascending(false);
+ }
+ if (attribute.getSorting().getFunction() != Sorting.Function.UCA) {
+ aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString()));
+ }
+ if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) {
+ aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString()));
+ }
+ if (!attribute.getSorting().getLocale().isEmpty()) {
+ aaB.sortlocale(attribute.getSorting().getLocale());
+ }
+ aaB.arity(attribute.arity());
+ aaB.lowerbound(attribute.lowerBound());
+ aaB.upperbound(attribute.upperBound());
+ aaB.densepostinglistthreshold(attribute.densePostingListThreshold());
+ if (attribute.tensorType().isPresent()) {
+ aaB.tensortype(attribute.tensorType().get().toString());
+ }
+ builder.attribute(aaB);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
new file mode 100644
index 00000000000..643eeb23c87
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.ConfigInstance.Builder;
+import com.yahoo.io.IOUtils;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.text.StringUtilities;
+import java.io.IOException;
+import java.io.Writer;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+/**
+ * Superclass of all derived configurations
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public abstract class Derived implements Exportable {
+
+ private String name;
+
+ public String getName() { return name; }
+
+ protected final void setName(String name) { this.name=name; }
+
+ /**
+ * Derives the content of this configuration. This
+ * default calls derive(Document) for each document
+ * and derive(SDField) for each search definition level field
+ * AND sets the name of this to the name of the input search definition
+ */
+ protected void derive(Search search) {
+ setName(search.getName());
+ derive(search.getDocument(), search);
+ for (Index index : search.getExplicitIndices()) {
+ derive(index, search);
+ }
+ for (SDField field : search.allExtraFields() ) {
+ derive(field,search);
+ }
+ }
+
+
+ /**
+ * Derives the content of this configuration. This
+ * default calls derive(SDField) for each document field
+ */
+ protected void derive(SDDocumentType document,Search search) {
+ for (Field field : document.fieldSet()) {
+ SDField sdField = (SDField) field;
+ if (!sdField.isExtraField()) {
+ derive(sdField, search);
+ }
+ }
+ }
+
+ /**
+ * Derives the content of this configuration. This
+ * default does nothing.
+ */
+ protected void derive(SDField field,Search search) {}
+
+ /**
+ * Derives the content of this configuration. This
+ * default does nothing.
+ */
+ protected void derive(Index index, Search search) {
+ }
+
+ protected abstract String getDerivedName();
+
+ /** Returns the value of getName if true, the given number as a string otherwise */
+ protected String getIndex(int number,boolean labels) {
+ return labels ? getName() : String.valueOf(number);
+ }
+
+ /**
+ * Exports this derived configuration to its .cfg file
+ * in toDirectory
+ *
+ * @param toDirectory the directory to export to, or null
+ *
+ */
+ public final void export(String toDirectory)
+ throws IOException {
+ Writer writer=null;
+ try {
+ String fileName=getDerivedName() + ".cfg";
+ if (toDirectory!=null) writer=IOUtils.createWriter(toDirectory + "/" + fileName,false);
+ try {
+ exportBuilderConfig(writer);
+ } catch (ClassNotFoundException | InstantiationException
+ | IllegalAccessException | NoSuchMethodException
+ | SecurityException | IllegalArgumentException
+ | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ finally {
+ if (writer!=null) IOUtils.closeWriter(writer);
+ }
+ }
+
+ /**
+ * Checks what this is a producer of, instantiate that and export to writer
+ */
+ // TODO move to ReflectionUtil, and move that to unexported pkg
+ private void exportBuilderConfig(Writer writer) throws ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, SecurityException, IllegalArgumentException, InvocationTargetException, IOException {
+ for (Class<?> intf : getClass().getInterfaces()) {
+ if (ConfigInstance.Producer.class.isAssignableFrom(intf)) {
+ Class<?> configClass = intf.getEnclosingClass();
+ String builderClassName = configClass.getCanonicalName()+"$Builder";
+ Class<?> builderClass = Class.forName(builderClassName);
+ ConfigInstance.Builder builder = (Builder) builderClass.newInstance();
+ Method getConfig = getClass().getMethod("getConfig", builderClass);
+ getConfig.invoke(this, builder);
+ ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder);
+ List<String> payloadL = ConfigInstance.serialize(inst);
+ String payload = StringUtilities.implodeMultiline(payloadL);
+ writer.write(payload);
+ }
+ }
+ }
+
+ @Override
+ public String getFileName() {
+ return getDerivedName() + ".cfg";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
new file mode 100644
index 00000000000..973e4e6100c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -0,0 +1,184 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.io.IOUtils;
+import com.yahoo.protect.Validator;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.validation.Validation;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.List;
+
+/**
+ * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when
+ * necessary.
+ *
+ * @author bratseth
+ */
+public class DerivedConfiguration {
+
+ private Search search;
+ private Summaries summaries;
+ private SummaryMap summaryMap;
+ private Juniperrc juniperrc;
+ private AttributeFields attributeFields;
+ private RankProfileList rankProfileList;
+ private IndexingScript indexingScript;
+ private IndexInfo indexInfo;
+ private VsmFields streamingFields;
+ private VsmSummary streamingSummary;
+ private IndexSchema indexSchema;
+
+ /**
+ * Creates a complete derived configuration from a search definition.
+ *
+ * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is
+ * live. Which means that this object will be inconsistent when the given search definition is later
+ * modified.
+ * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
+ */
+ public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) {
+ this(search, null, new BaseDeployLogger(), rankProfileRegistry);
+ }
+
+ /**
+ * Creates a complete derived configuration snapshot from a search definition.
+ *
+ * @param search The search to derive a configuration from. Derived objects will be snapshots, but this
+ * argument is live. Which means that this object will be inconsistent when the given
+ * search definition is later modified.
+ * @param abstractSearchList Search definition this one inherits from, only superclass configuration should be
+ * generated. Null or empty list if there is none.
+ * @param deployLogger a {@link DeployLogger} for logging when
+ * doing operations on this
+ * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
+ */
+ public DerivedConfiguration(Search search, List<Search> abstractSearchList, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) {
+ Validator.ensureNotNull("Search definition", search);
+ if (!search.isProcessed()) {
+ throw new IllegalArgumentException("Search '" + search.getName() + "' not processed.");
+ }
+ this.search = search;
+ if (!search.isDocumentsOnly()) {
+ streamingFields = new VsmFields(search);
+ streamingSummary = new VsmSummary(search);
+ }
+ if (abstractSearchList != null) {
+ for (Search abstractSearch : abstractSearchList) {
+ if (!abstractSearch.isProcessed()) {
+ throw new IllegalArgumentException("Search '" + search.getName() + "' not processed.");
+ }
+ }
+ }
+ if (!search.isDocumentsOnly()) {
+ summaries = new Summaries(search, deployLogger);
+ summaryMap = new SummaryMap(search, summaries);
+ juniperrc = new Juniperrc(search);
+ attributeFields = new AttributeFields(search);
+ rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry);
+ indexingScript = new IndexingScript(search);
+ indexInfo = new IndexInfo(search);
+ indexSchema = new IndexSchema(search);
+ }
+ Validation.validate(this, search);
+ }
+
+ /**
+ * Exports a complete set of configuration-server format config files.
+ *
+ * @param toDirectory the directory to export to, current dir if null
+ * @throws IOException if exporting fails, some files may still be created
+ */
+ public void export(String toDirectory) throws IOException {
+ if (!search.isDocumentsOnly()) {
+ summaries.export(toDirectory);
+ summaryMap.export(toDirectory);
+ juniperrc.export(toDirectory);
+ attributeFields.export(toDirectory);
+ streamingFields.export(toDirectory);
+ streamingSummary.export(toDirectory);
+ indexSchema.export(toDirectory);
+ rankProfileList.export(toDirectory);
+ indexingScript.export(toDirectory);
+ indexInfo.export(toDirectory);
+ }
+ }
+
+ public static void exportDocuments(DocumentmanagerConfig.Builder documentManagerCfg, String toDirectory) throws IOException {
+ exportCfg(new DocumentmanagerConfig(documentManagerCfg), toDirectory + "/" + "documentmanager.cfg");
+ }
+
+ public static void exportDocuments(DocumenttypesConfig.Builder documentTypesCfg, String toDirectory) throws IOException {
+ exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg");
+ }
+
+ private static void exportCfg(ConfigInstance instance, String fileName) throws IOException {
+ Writer writer = null;
+ try {
+ writer = IOUtils.createWriter(fileName, false);
+ if (writer != null) {
+ writer.write(instance.toString());
+ writer.write("\n");
+ }
+ } finally {
+ if (writer != null) {
+ IOUtils.closeWriter(writer);
+ }
+ }
+ }
+
+ public Summaries getSummaries() {
+ return summaries;
+ }
+
+ public AttributeFields getAttributeFields() {
+ return attributeFields;
+ }
+
+ public IndexingScript getIndexingScript() {
+ return indexingScript;
+ }
+
+ public IndexInfo getIndexInfo() {
+ return indexInfo;
+ }
+
+ public void setIndexingScript(IndexingScript script) {
+ this.indexingScript = script;
+ }
+
+ public Search getSearch() {
+ return search;
+ }
+
+ public RankProfileList getRankProfileList() {
+ return rankProfileList;
+ }
+
+ public VsmSummary getVsmSummary() {
+ return streamingSummary;
+ }
+
+ public VsmFields getVsmFields() {
+ return streamingFields;
+ }
+
+ public IndexSchema getIndexSchema() {
+ return indexSchema;
+ }
+
+ public Juniperrc getJuniperrc() {
+ return juniperrc;
+ }
+
+ public SummaryMap getSummaryMap() {
+ return summaryMap;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java
new file mode 100644
index 00000000000..76bf4492788
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.UnprocessingSearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Auxiliary facade for deriving configs from search definitions
+ *
+ * @author bratseth
+ */
+public class Deriver {
+
+ /**
+ * Derives only document manager.
+ *
+ *
+ * @param sdFileNames The name of the search definition files to derive from.
+ * @param toDir The directory to write configuration to.
+ * @return The list of Search objects, possibly "unproper ones", from sd files containing only document
+ */
+ public static SearchBuilder deriveDocuments(List<String> sdFileNames, String toDir) {
+ SearchBuilder builder = getUnprocessingSearchBuilder(sdFileNames);
+ DocumentmanagerConfig.Builder documentManagerCfg = new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder());
+ try {
+ DerivedConfiguration.exportDocuments(documentManagerCfg, toDir);
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ return builder;
+ }
+
+ public static SearchBuilder getSearchBuilder(List<String> sds) {
+ SearchBuilder builder = new SearchBuilder();
+ try {
+ for (String s : sds) {
+ builder.importFile(s);
+ }
+ } catch (ParseException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ builder.build();
+ return builder;
+ }
+
+ public static SearchBuilder getUnprocessingSearchBuilder(List<String> sds) {
+ SearchBuilder builder = new UnprocessingSearchBuilder();
+ try {
+ for (String s : sds) {
+ builder.importFile(s);
+ }
+ } catch (ParseException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ builder.build();
+ return builder;
+ }
+
+ public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) {
+ return getDocumentManagerConfig(Collections.singletonList(sd));
+ }
+
+ public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds) {
+ return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder());
+ }
+
+ public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) {
+ return getDocumentTypesConfig(Collections.singletonList(sd));
+ }
+
+ public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> sds) {
+ return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java
new file mode 100644
index 00000000000..bcc295b91e0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+/**
+ * Classes exportable to configurations
+ *
+ * @author <a href="mailto:bratseth@overture.com">bratseth</a>
+ */
+public interface Exportable {
+
+ /**
+ * Exports the configuration of this object
+ *
+ *
+ * @param toDirectory the directory to export to, does not write to disk if null
+ * @throws java.io.IOException if exporting fails, some files may still be created
+ */
+ public void export(String toDirectory) throws java.io.IOException;
+
+ /**
+ * The (short) name of the exported file
+ * @return a String with the (short) name of the exported file
+ */
+ public String getFileName();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
new file mode 100644
index 00000000000..9dfc1bc2af7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * The rank settings of a field used for native rank features.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class FieldRankSettings {
+
+ private static final Logger logger = Logger.getLogger(FieldRankSettings.class.getName());
+
+ private String fieldName;
+
+ private final Map<String, NativeTable> tables = new LinkedHashMap<>();
+
+ public FieldRankSettings(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public void addTable(NativeTable table) {
+ NativeTable existing = tables.get(table.getType().getName());
+ if (existing != null) {
+ logger.info("Using already specified rank table " + existing + " for field " + fieldName + ", not " + table);
+ return;
+ }
+ tables.put(table.getType().getName(), table);
+ }
+
+ public static boolean isIndexFieldTable(NativeTable table) {
+ return isFieldMatchTable(table) || isProximityTable(table);
+ }
+
+ public static boolean isAttributeFieldTable(NativeTable table) {
+ return isAttributeMatchTable(table);
+ }
+
+ private static boolean isFieldMatchTable(NativeTable table) {
+ return (table.getType().equals(NativeTable.Type.FIRST_OCCURRENCE) ||
+ table.getType().equals(NativeTable.Type.OCCURRENCE_COUNT));
+ }
+
+ private static boolean isAttributeMatchTable(NativeTable table) {
+ return (table.getType().equals(NativeTable.Type.WEIGHT));
+ }
+
+ private static boolean isProximityTable(NativeTable table) {
+ return (table.getType().equals(NativeTable.Type.PROXIMITY) ||
+ table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY));
+ }
+
+ public Map<String,String> deriveRankProperties(int part) {
+ Map<String,String> ret = new LinkedHashMap<>();
+ int i = part;
+ for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) {
+ NativeTable table = itr.next();
+ if (isFieldMatchTable(table)) {
+ ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
+ }
+ if (isAttributeMatchTable(table)) {
+ ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
+ }
+ if (isProximityTable(table)) {
+ ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
+ }
+ }
+ return ret;
+ }
+
+ public String toString() {
+ return "rank settings of field " + fieldName;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java
new file mode 100644
index 00000000000..d61e57a621e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+/**
+ * The result transformation of a named field
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class FieldResultTransform {
+
+ private String fieldName;
+
+ private SummaryTransform transform;
+
+ private String argument;
+
+ public FieldResultTransform(String fieldName,SummaryTransform transform,String argument) {
+ this.fieldName=fieldName;
+ this.transform=transform;
+ this.argument = argument;
+ }
+
+ public String getFieldName() { return fieldName; }
+
+ public SummaryTransform getTransform() { return transform; }
+
+ public void setTransform(SummaryTransform transform) { this.transform=transform; }
+
+ /** Returns the argument of this (used as input to the backend docsum rewriter) */
+ public String getArgument() { return argument; }
+
+ public int hashCode() {
+ return fieldName.hashCode() + 11*transform.hashCode() + 17* argument.hashCode();
+ }
+
+ public boolean equals(Object o) {
+ if (! (o instanceof FieldResultTransform)) return false;
+ FieldResultTransform other=(FieldResultTransform)o;
+
+ return
+ this.fieldName.equals(other.fieldName) &&
+ this.transform.equals(other.transform) &&
+ this.argument.equals(other.argument);
+ }
+
+ public String toString() {
+ String sourceString="";
+ if ( ! argument.equals(fieldName))
+ sourceString=" (argument: " + argument + ")";
+ return "field " + fieldName + ": " + transform + sourceString;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java
new file mode 100644
index 00000000000..cab4176d696
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.document.datatypes.*;
+
+/**
+ * A type of an index structure
+ *
+ * @author bratseth
+ */
+public class Index {
+
+ /** The index type enumeration */
+ public static class Type {
+
+ public static final Type TEXT=new Type("text");
+ public static final Type INT64=new Type("long");
+ public static final Type BOOLEANTREE=new Type("booleantree");
+
+ private String name;
+
+ private Type(String name) {
+ this.name=name;
+ }
+
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ public String getName() { return name; }
+
+ public boolean equals(Object other) {
+ if ( ! (other instanceof Type)) return false;
+ return this.name.equals(((Type)other).name);
+ }
+
+ public String toString() {
+ return "type: " + name;
+ }
+
+ }
+
+ /** Sets the right index type from a field type */
+ public static Type convertType(DataType fieldType) {
+ FieldValue fval = fieldType.createFieldValue();
+ if (fieldType instanceof NumericDataType) {
+ return Type.INT64;
+ } else if (fval instanceof StringFieldValue) {
+ return Type.TEXT;
+ } else if (fval instanceof Raw) {
+ return Type.BOOLEANTREE;
+ } else if (fval instanceof PredicateFieldValue) {
+ return Type.BOOLEANTREE;
+ } else if (fieldType instanceof CollectionDataType) {
+ return convertType(((CollectionDataType) fieldType).getNestedType());
+ } else {
+ throw new IllegalArgumentException("Don't know which index type to " +
+ "convert " + fieldType + " to");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
new file mode 100644
index 00000000000..e98ee662b3a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -0,0 +1,559 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.processing.ExactMatch;
+import com.yahoo.searchdefinition.processing.NGramMatch;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.search.config.IndexInfoConfig;
+
+import java.util.*;
+
+/**
+ * Per-index commands which should be applied to queries prior to searching
+ *
+ * @author bratseth
+ */
+public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
+
+ private static final String CMD_ATTRIBUTE = "attribute";
+ private static final String CMD_DEFAULT_POSITION = "default-position";
+ private static final String CMD_DYNTEASER = "dynteaser";
+ private static final String CMD_FULLURL = "fullurl";
+ private static final String CMD_HIGHLIGHT = "highlight";
+ private static final String CMD_INDEX = "index";
+ private static final String CMD_LOWERCASE = "lowercase";
+ private static final String CMD_MATCH_GROUP = "match-group ";
+ private static final String CMD_NORMALIZE = "normalize";
+ private static final String CMD_STEM = "stem";
+ private static final String CMD_URLHOST = "urlhost";
+ private static final String CMD_WORD = "word";
+ private static final String CMD_PLAIN_TOKENS = "plain-tokens";
+ private static final String CMD_MULTIVALUE = "multivalue";
+ private static final String CMD_FAST_SEARCH = "fast-search";
+ private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds";
+ private static final String CMD_NUMERICAL = "numerical";
+ private Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
+ private Map<String, String> aliases = new java.util.LinkedHashMap<>();
+ private Map<String, FieldSet> fieldSets;
+ private Search search;
+
+ public IndexInfo(Search search) {
+ this.fieldSets = search.fieldSets().userFieldSets();
+ addIndexCommand("sddocname", CMD_INDEX);
+ addIndexCommand("sddocname", CMD_WORD);
+ derive(search);
+ }
+
+ protected void derive(Search search) {
+ super.derive(search); // Derive per field
+ this.search = search;
+ // Populate fieldsets with actual field objects, bit late to do that here but
+ for (FieldSet fs : fieldSets.values()) {
+ for (String fieldName : fs.getFieldNames()) {
+ fs.fields().add(search.getField(fieldName));
+ }
+ }
+ // Must follow, because index settings overrides field settings
+ for (Index index : search.getExplicitIndices()) {
+ derive(index, search);
+ }
+
+ // Commands for summary fields
+ // TODO: Move to fieldinfo and implement differently. This is not right
+ for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
+ if (summaryField.getTransform().isTeaser()) {
+ addIndexCommand(summaryField.getName(), CMD_DYNTEASER);
+ }
+ if (summaryField.getTransform().isBolded()) {
+ addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT);
+ }
+ }
+ }
+
+ protected void derive(Index index, Search search) {
+ if (index.getMatchGroup().size() > 0) {
+ addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup()));
+ }
+ }
+
+ private String toSpaceSeparated(Collection c) {
+ StringBuffer b = new StringBuffer();
+ for (Iterator i = c.iterator(); i.hasNext();) {
+ b.append(i.next());
+ if (i.hasNext()) {
+ b.append(" ");
+ }
+ }
+ return b.toString();
+ }
+
+ protected void derive(SDField field, Search search) {
+ if (field.getDataType().equals(DataType.PREDICATE)) {
+ Index index = field.getIndex(field.getName());
+ if (index != null) {
+ BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
+ if (options.hasLowerBound() || options.hasUpperBound()) {
+ addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" +
+ (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." +
+ (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]");
+ }
+ }
+ }
+
+ // Field level aliases
+ for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
+ String alias = e.getKey();
+ String name = e.getValue();
+ addIndexAlias(alias, name);
+ }
+ if (field.usesStructOrMap()) {
+ for (SDField structField : field.getStructFields()) {
+ derive(structField, search); // Recursion
+ }
+ }
+
+ if (field.getDataType().equals(PositionDataType.INSTANCE) ||
+ field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
+ {
+ addIndexCommand(field.getName(), CMD_DEFAULT_POSITION);
+ }
+
+ addIndexCommand(field, CMD_INDEX); // List the indices
+
+ if (field.doesIndexing() || field.doesLowerCasing()) {
+ addIndexCommand(field, CMD_LOWERCASE);
+ }
+
+ if (field.getDataType().isMultivalue()) {
+ addIndexCommand(field, CMD_MULTIVALUE);
+ }
+
+ if (field.doesAttributing() && !field.doesIndexing()) {
+ addIndexCommand(field.getName(), CMD_ATTRIBUTE);
+ Attribute attribute = field.getAttributes().get(field.getName());
+ if (attribute != null && attribute.isFastSearch())
+ addIndexCommand(field.getName(), CMD_FAST_SEARCH);
+ } else if (field.doesIndexing()) {
+ if (stemSomehow(field, search)) {
+ addIndexCommand(field, stemCmd(field, search), new StemmingOverrider(this, search));
+ }
+ if (normalizeAccents(field)) {
+ addIndexCommand(field, CMD_NORMALIZE);
+ }
+ }
+
+ if (isUriField(field)) {
+ addUriIndexCommands(field);
+ }
+
+ if (field.getDataType() instanceof NumericDataType) {
+ addIndexCommand(field, CMD_NUMERICAL);
+ }
+
+ // Explicit commands
+ for (String command : field.getQueryCommands()) {
+ addIndexCommand(field, command);
+ }
+
+ }
+
+ static String stemCmd(SDField field, Search search) {
+ return CMD_STEM + ":" + field.getStemming(search).toStemMode();
+ }
+
+ private boolean stemSomehow(SDField field, Search search) {
+ if (field.getStemming(search).equals(Stemming.NONE)) return false;
+ return isTypeOrNested(field, DataType.STRING);
+ }
+
+ private boolean normalizeAccents(SDField field) {
+ return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING);
+ }
+
+ private boolean isTypeOrNested(SDField field, DataType type) {
+ return field.getDataType().equals(type) || field.getDataType().equals(DataType.getArray(type)) ||
+ field.getDataType().equals(DataType.getWeightedSet(type));
+ }
+
+ private boolean isUriField(Field field) {
+ DataType fieldType = field.getDataType();
+ if (DataType.URI.equals(fieldType)) {
+ return true;
+ }
+ if (fieldType instanceof CollectionDataType &&
+ DataType.URI.equals(((CollectionDataType)fieldType).getNestedType()))
+ {
+ return true;
+ }
+ return false;
+ }
+
+ private void addUriIndexCommands(SDField field) {
+ String fieldName = field.getName();
+ addIndexCommand(fieldName, CMD_FULLURL);
+ addIndexCommand(fieldName, CMD_LOWERCASE);
+ addIndexCommand(fieldName + "." + fieldName, CMD_FULLURL);
+ addIndexCommand(fieldName + "." + fieldName, CMD_LOWERCASE);
+ addIndexCommand(fieldName + ".path", CMD_FULLURL);
+ addIndexCommand(fieldName + ".path", CMD_LOWERCASE);
+ addIndexCommand(fieldName + ".query", CMD_FULLURL);
+ addIndexCommand(fieldName + ".query", CMD_LOWERCASE);
+ addIndexCommand(fieldName + ".hostname", CMD_URLHOST);
+ addIndexCommand(fieldName + ".hostname", CMD_LOWERCASE);
+
+ // XXX hack
+ Index index = field.getIndex("hostname");
+ if (index != null) {
+ addIndexCommand(index, CMD_URLHOST);
+ }
+ }
+
+ /**
+ * Sets a command for all indices of a field
+ */
+ private void addIndexCommand(Index index, String command) {
+ addIndexCommand(index.getName(), command);
+ }
+
+ /**
+ * Sets a command for all indices of a field
+ */
+ private void addIndexCommand(SDField field, String command) {
+ addIndexCommand(field, command, null);
+ }
+
+ /**
+ * Sets a command for all indices of a field
+ */
+ private void addIndexCommand(SDField field, String command, IndexOverrider overrider) {
+ if (overrider == null || !overrider.override(field.getName(), command, field)) {
+ addIndexCommand(field.getName(), command);
+ }
+ }
+
+ private void addIndexCommand(String indexName, String command) {
+ commands.add(new IndexCommand(indexName, command));
+ }
+
+ private void addIndexAlias(String alias, String indexName) {
+ aliases.put(alias, indexName);
+ }
+
+ /**
+ * Returns whether a particular command is prsent in this index info
+ */
+ public boolean hasCommand(String indexName, String command) {
+ return commands.contains(new IndexCommand(indexName, command));
+ }
+
+ private boolean notInCommands(String index) {
+ for (IndexCommand command : commands) {
+ if (command.getIndex().equals(index)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder();
+ iiB.name(getName());
+ for (IndexCommand command : commands) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(command.getIndex())
+ .command(command.getCommand()));
+ }
+ // Make user defined field sets searchable
+ for (FieldSet fieldSet : fieldSets.values()) {
+ if (notInCommands(fieldSet.getName())) {
+ addFieldSetCommands(iiB, fieldSet);
+ }
+ }
+
+ for (Map.Entry<String, String> e : aliases.entrySet()) {
+ iiB.alias(
+ new IndexInfoConfig.Indexinfo.Alias.Builder()
+ .alias(e.getKey())
+ .indexname(e.getValue()));
+ }
+ builder.indexinfo(iiB);
+ }
+
+ // TODO: This implementation is completely brain dead.
+ // Move it to the FieldSetValidity processor (and rename it) as that already has to look at this.
+ // Also add more explicit testing of this, e.g to indexinfo_fieldsets in ExportingTestCase. - Jon
+ private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) {
+ // Explicit query commands on the field set, overrides everything.
+ if (!fieldSet.queryCommands().isEmpty()) {
+ for (String qc : fieldSet.queryCommands()) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(qc));
+ }
+ return;
+ }
+ boolean anyIndexing = false;
+ boolean anyAttributing = false;
+ boolean anyLowerCasing = false;
+ boolean anyStemming = false;
+ boolean anyNormalizing = false;
+ String stemmingCommand = null;
+ Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching
+ // First a pass over the fields to read some params to decide field settings implicitly:
+ for (SDField field : fieldSet.fields()) {
+ if (field.doesIndexing()) {
+ anyIndexing = true;
+ }
+ if (field.doesAttributing()) {
+ anyAttributing = true;
+ }
+ if (field.doesIndexing() || field.doesLowerCasing()) {
+ anyLowerCasing = true;
+ }
+ if (stemming(field)) {
+ anyStemming = true;
+ stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode();
+ }
+ if (field.getNormalizing().doRemoveAccents()) {
+ anyNormalizing = true;
+ }
+ if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType)
+ fieldSetMatching = field.getMatching();
+ }
+ if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) {
+ // We have both attributes and indexes and no explicit match setting ->
+ // use default matching as that at least works if the data in the attribute consists
+ // of single tokens only.
+ fieldSetMatching = new Matching();
+ }
+ if (anyLowerCasing) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_LOWERCASE));
+ }
+ if (hasMultiValueField(fieldSet)) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_MULTIVALUE));
+ }
+ if (anyIndexing) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_INDEX));
+ if ( ! isExactMatch(fieldSetMatching)) {
+ if (anyStemming) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(stemmingCommand));
+ }
+ if (anyNormalizing)
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_NORMALIZE));
+ }
+ } else {
+ // Assume only attribute fields
+ iiB
+ .command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_ATTRIBUTE))
+ .command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_INDEX));
+ }
+ if (fieldSetMatching != null) {
+ // Explicit matching set on fieldset
+ if (fieldSetMatching.getType().equals(Matching.Type.EXACT)) {
+ String term = fieldSetMatching.getExactMatchTerminator();
+ if (term==null) term=ExactMatch.DEFAULT_EXACT_TERMINATOR;
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command("exact "+term));
+ } else if (fieldSetMatching.getType().equals(Matching.Type.WORD)) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command(CMD_WORD));
+ } else if (fieldSetMatching.getType().equals(Matching.Type.GRAM)) {
+ iiB.command(
+ new IndexInfoConfig.Indexinfo.Command.Builder()
+ .indexname(fieldSet.getName())
+ .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE)));
+ } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) {
+
+ }
+
+ }
+
+ }
+
+ private boolean hasMultiValueField(FieldSet fieldSet) {
+ for (SDField field : fieldSet.fields()) {
+ if (field.getDataType().isMultivalue())
+ return true;
+ }
+ return false;
+ }
+
+ private Stemming getEffectiveStemming(SDField field) {
+ Stemming active = field.getStemming(search);
+ if (field.getIndex(field.getName()) != null) {
+ if (field.getIndex(field.getName()).getStemming()!=null) {
+ active = field.getIndex(field.getName()).getStemming();
+ }
+ }
+ if (active != null) {
+ return active;
+ }
+ // assume default
+ return Stemming.SHORTEST;
+ }
+
+ private boolean stemming(SDField field) {
+ if (field.getStemming() != null) {
+ return !field.getStemming().equals(Stemming.NONE);
+ }
+ if (search.getStemming()==Stemming.NONE) return false;
+ if (field.getIndex(field.getName())==null) return true;
+ if (field.getIndex(field.getName()).getStemming()==null) return true;
+ return !(field.getIndex(field.getName()).getStemming().equals(Stemming.NONE));
+ }
+
+ private boolean isExactMatch(Matching m) {
+ if (m==null) return false;
+ if (m.getType().equals(Matching.Type.EXACT)) return true;
+ if (m.getType().equals(Matching.Type.WORD)) return true;
+ return false;
+ }
+
+ /**
+ * Returns a read only iterator over the index commands of this
+ */
+ public Iterator<IndexCommand> indexCommandIterator() {
+ return Collections.unmodifiableSet(commands).iterator();
+ }
+
+ protected String getDerivedName() {
+ return "index-info";
+ }
+
+ /**
+ * An index command. Null commands are also represented, to detect consistency issues. This is an (immutable) value
+ * object.
+ */
+ public static class IndexCommand {
+
+ private String index;
+
+ private String command;
+
+ public IndexCommand(String index, String command) {
+ this.index = index;
+ this.command = command;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public String getCommand() {
+ return command;
+ }
+
+ /**
+ * Returns true if this is the null command (do nothing)
+ */
+ public boolean isNull() {
+ return command.equals("");
+ }
+
+ public int hashCode() {
+ return index.hashCode() + 17 * command.hashCode();
+ }
+
+ public boolean equals(Object object) {
+ if (!(object instanceof IndexCommand)) {
+ return false;
+ }
+
+ IndexCommand other = (IndexCommand)object;
+ return
+ other.index.equals(this.index) &&
+ other.command.equals(this.command);
+ }
+
+ public String toString() {
+ return "index command " + command + " on index " + index;
+ }
+
+ }
+
+ /**
+ * A command which may override the command setting of a field for a particular index
+ */
+ private static abstract class IndexOverrider {
+
+ protected IndexInfo owner;
+
+ public IndexOverrider(IndexInfo owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Override the setting of this index for this field, returns true if overriden, false if this index should be
+ * set according to the field
+ */
+ public abstract boolean override(String indexName, String command, SDField field);
+
+ }
+
+ private static class StemmingOverrider extends IndexOverrider {
+
+ private Search search;
+
+ public StemmingOverrider(IndexInfo owner, Search search) {
+ super(owner);
+ this.search = search;
+ }
+
+ public boolean override(String indexName, String command, SDField field) {
+ if (search == null) {
+ return false;
+ }
+
+ Index index = search.getIndex(indexName);
+ if (index == null) {
+ return false;
+ }
+
+ Stemming indexStemming = index.getStemming();
+ if (indexStemming == null) {
+ return false;
+ }
+
+ if (Stemming.NONE.equals(indexStemming)) {
+ // Add nothing
+ } else {
+ owner.addIndexCommand(indexName, CMD_STEM + ":" + indexStemming.toStemMode());
+ }
+ return true;
+ }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
new file mode 100644
index 00000000000..05b8a1bf5e7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
@@ -0,0 +1,231 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.config.search.IndexschemaConfig;
+
+import java.util.*;
+
+/**
+ * Deriver of indexschema config containing information of all index fields with name and data type.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
+
+ private final List<IndexField> fields = new ArrayList<>();
+ private final Map<String, FieldCollection> collections = new LinkedHashMap<>();
+ private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>();
+
+ public IndexSchema(Search search) {
+ fieldSets.putAll(search.fieldSets().userFieldSets());
+ derive(search);
+ }
+
+ public boolean containsField(String fieldName) {
+ return fields.stream().anyMatch(field -> field.getName().equals(fieldName));
+ }
+
+ @Override
+ protected void derive(Search search) {
+ super.derive(search);
+ }
+
+ private void deriveIndexFields(SDField field, Search search) {
+ if (!field.doesIndexing() &&
+ !field.isIndexStructureField())
+ {
+ return;
+ }
+ List<Field> lst = flattenField(field);
+ if (lst.isEmpty()) {
+ return;
+ }
+ String fieldName = field.getName();
+ for (Field flatField : lst) {
+ deriveIndexFields(flatField, search);
+ }
+ if (lst.size() > 1) {
+ FieldSet fieldSet = new FieldSet(fieldName);
+ for (Field flatField : lst) {
+ fieldSet.addFieldName(flatField.getName());
+ }
+ fieldSets.put(fieldName, fieldSet);
+ }
+ }
+
+ private void deriveIndexFields(Field field, Search search) {
+ IndexField toAdd = new IndexField(field.getName(), Index.convertType(field.getDataType()), field.getDataType());
+ com.yahoo.searchdefinition.Index definedIndex = search.getIndex(field.getName());
+ if (definedIndex != null) {
+ toAdd.setIndexSettings(definedIndex);
+ }
+ fields.add(toAdd);
+ addFieldToCollection(field.getName(), field.getName()); // implicit
+ }
+
+ private FieldCollection getCollection(String collectionName) {
+ FieldCollection retval = collections.get(collectionName);
+ if (retval == null) {
+ collections.put(collectionName, new FieldCollection(collectionName));
+ return collections.get(collectionName);
+ }
+ return retval;
+ }
+
+ private void addFieldToCollection(String fieldName, String collectionName) {
+ FieldCollection collection = getCollection(collectionName);
+ collection.fields.add(fieldName);
+ }
+
+ @Override
+ protected void derive(SDField field, Search search) {
+ if (field.usesStructOrMap()) {
+ return; // unsupported
+ }
+ deriveIndexFields(field, search);
+ }
+
+ @Override
+ protected String getDerivedName() {
+ return "indexschema";
+ }
+
+ @Override
+ public void getConfig(IndexschemaConfig.Builder icB) {
+ for (int i = 0; i < fields.size(); ++i) {
+ IndexField f = fields.get(i);
+ IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder()
+ .name(f.getName())
+ .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType()))
+ .prefix(f.hasPrefix())
+ .phrases(f.hasPhrases())
+ .positions(f.hasPositions());
+ if (f.getSdType() !=null && !f.getSdType().equals(com.yahoo.searchdefinition.Index.Type.VESPA)) {
+ ifB.indextype(IndexschemaConfig.Indexfield.Indextype.Enum.valueOf(f.getSdType().toString()));
+ }
+ if (!f.getCollectionType().equals("SINGLE")) {
+ ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType()));
+ }
+ icB.indexfield(ifB);
+ }
+ for (FieldSet fieldSet : fieldSets.values()) {
+ IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder()
+ .name(fieldSet.getName());
+ for (String f : fieldSet.getFieldNames()) {
+ fsB.field(new IndexschemaConfig.Fieldset.Field.Builder()
+ .name(f));
+ }
+ icB.fieldset(fsB);
+ }
+ }
+
+ static List<Field> flattenField(Field field) {
+ DataType fieldType = field.getDataType();
+ if (fieldType.getPrimitiveType() != null){
+ return Collections.singletonList(field);
+ }
+ if (fieldType instanceof ArrayDataType) {
+ boolean header = field.isHeader();
+ List<Field> ret = new LinkedList<>();
+ Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header);
+ for (Field flatField : flattenField(innerField)) {
+ ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header));
+ }
+ return ret;
+ }
+ if (fieldType instanceof StructuredDataType) {
+ List<Field> ret = new LinkedList<>();
+ String fieldName = field.getName();
+ for (Field childField : ((StructuredDataType)fieldType).getFields()) {
+ for (Field flatField : flattenField(childField)) {
+ ret.add(new Field(fieldName + "." + flatField.getName(), flatField));
+ }
+ }
+ return ret;
+ }
+ throw new UnsupportedOperationException(fieldType.getName());
+ }
+
+ public List<IndexField> getFields() {
+ return fields;
+ }
+
+ /**
+ * Representation of an index field with name and data type.
+ */
+ public static class IndexField {
+ private String name;
+ private Index.Type type;
+ private com.yahoo.searchdefinition.Index.Type sdType; // The index type in "user intent land"
+ private DataType sdFieldType;
+ private boolean prefix = false;
+ private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility?
+ private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility?
+ private BooleanIndexDefinition boolIndex = null;
+
+ public IndexField(String name, Index.Type type, DataType sdFieldType) {
+ this.name = name;
+ this.type = type;
+ this.sdFieldType = sdFieldType;
+ }
+ public void setIndexSettings(com.yahoo.searchdefinition.Index index) {
+ if (type.equals(Index.Type.TEXT)) {
+ prefix = index.isPrefix();
+ }
+ sdType = index.getType();
+ boolIndex = index.getBooleanIndexDefiniton();
+ }
+ public String getName() { return name; }
+ public Index.Type getRawType() { return type; }
+ public String getType() {
+ return type.equals(Index.Type.INT64)
+ ? "INT64"
+ : type.equals(Index.Type.BOOLEANTREE)
+ ? "BOOLEANTREE"
+ : "STRING";
+ }
+ public String getCollectionType() {
+ return (sdFieldType == null)
+ ? "SINGLE"
+ : (sdFieldType instanceof WeightedSetDataType)
+ ? "WEIGHTEDSET"
+ : (sdFieldType instanceof ArrayDataType)
+ ? "ARRAY"
+ : "SINGLE";
+ }
+ public boolean hasPrefix() { return prefix; }
+ public boolean hasPhrases() { return phrases; }
+ public boolean hasPositions() { return positions; }
+
+ public BooleanIndexDefinition getBooleanIndexDefinition() {
+ return boolIndex;
+ }
+
+ /**
+ * The user set index type
+ * @return the type
+ */
+ public com.yahoo.searchdefinition.Index.Type getSdType() {
+ return sdType;
+ }
+ }
+
+ /**
+ * Representation of a collection of fields (aka index, physical view).
+ */
+ @SuppressWarnings({ "UnusedDeclaration" })
+ private static class FieldCollection {
+
+ private final String name;
+ private final List<String> fields = new ArrayList<>();
+
+ FieldCollection(String name) {
+ this.name = name;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
new file mode 100644
index 00000000000..94e4dec567f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.*;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder;
+
+import java.util.*;
+
+/**
+ * An indexing language script derived from a search definition. An indexing script contains a set of indexing
+ * statements, organized in a composite structure of indexing code snippets.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public final class IndexingScript extends Derived implements IlscriptsConfig.Producer {
+
+ private final List<String> docFields = new LinkedList<>();
+ private final List<Expression> expressions = new LinkedList<>();
+
+ public IndexingScript(Search search) {
+ derive(search);
+ }
+
+ protected void derive(SDField field, Search search) {
+ if (!field.isExtraField()) {
+ docFields.add(field.getName());
+ }
+ if (field.usesStructOrMap() &&
+ !field.getDataType().equals(PositionDataType.INSTANCE) &&
+ !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
+ {
+ return; // unsupported
+ }
+ ScriptExpression script = field.getIndexingScript();
+ if (!script.isEmpty()) {
+ expressions.add(new StatementExpression(new ClearStateExpression(),
+ new GuardExpression(script)));
+ }
+ }
+
+ public Iterable<Expression> expressions() {
+ return Collections.unmodifiableCollection(expressions);
+ }
+
+ @Override
+ public String getDerivedName() {
+ return "ilscripts";
+ }
+
+ @Override
+ public void getConfig(IlscriptsConfig.Builder configBuilder) {
+ IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder();
+ ilscriptBuilder.doctype(getName());
+ for (String fieldName : docFields) {
+ ilscriptBuilder.docfield(fieldName);
+ }
+ addContentInOrder(ilscriptBuilder);
+ configBuilder.ilscript(ilscriptBuilder);
+ }
+
+ private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) {
+ ArrayList<Expression> later = new ArrayList<>();
+ Set<String> touchedFields = new HashSet<String>();
+ for (Expression exp : expressions) {
+ FieldScanVisitor fieldFetcher = new FieldScanVisitor();
+ if (modifiesSelf(exp)) {
+ later.add(exp);
+ } else {
+ ilscriptBuilder.content(exp.toString());
+ }
+ fieldFetcher.visit(exp);
+ touchedFields.addAll(fieldFetcher.touchedFields());
+ }
+ for (Expression exp : later) {
+ ilscriptBuilder.content(exp.toString());
+ }
+ generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields);
+ }
+
+ private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set<String> touchedFields) {
+ Set<String> fieldsWithSyntheticStatements = new HashSet<String>(docFields);
+ fieldsWithSyntheticStatements.removeAll(touchedFields);
+ List<String> orderedFields = new ArrayList<String>(fieldsWithSyntheticStatements);
+ Collections.sort(orderedFields);
+ for (String fieldName : orderedFields) {
+ StatementExpression copyField = new StatementExpression(new InputExpression(fieldName),
+ new PassthroughExpression(fieldName));
+ ilscriptBuilder.content(copyField.toString());
+ }
+ }
+
+ private boolean modifiesSelf(Expression exp) {
+ MyExpVisitor visitor = new MyExpVisitor();
+ visitor.visit(exp);
+ return visitor.modifiesSelf();
+ }
+
+ private class MyExpVisitor extends ExpressionVisitor {
+ private String inputField = null;
+ private String outputField = null;
+
+ public boolean modifiesSelf() { return outputField != null && outputField.equals(inputField); }
+
+ @Override
+ protected void doVisit(Expression expression) {
+ if (modifiesSelf()) {
+ return;
+ }
+ if (expression instanceof InputExpression) {
+ inputField = ((InputExpression) expression).getFieldName();
+ }
+ if (expression instanceof OutputExpression) {
+ outputField = ((OutputExpression) expression).getFieldName();
+ }
+ }
+ }
+
+ private static class FieldScanVisitor extends ExpressionVisitor {
+ List<String> touchedFields = new ArrayList<String>();
+ List<String> candidates = new ArrayList<String>();
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (exp instanceof OutputExpression) {
+ touchedFields.add(((OutputExpression) exp).getFieldName());
+ }
+ if (exp instanceof InputExpression) {
+ candidates.add(((InputExpression) exp).getFieldName());
+ }
+ if (exp instanceof ZCurveExpression) {
+ touchedFields.addAll(candidates);
+ }
+ }
+
+ Collection<String> touchedFields() {
+ Collection<String> output = touchedFields;
+ touchedFields = null; // deny re-use to try and avoid obvious bugs
+ return output;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
new file mode 100644
index 00000000000..9ef0ddbc723
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
+
+import java.util.Set;
+
+/**
+ * Generated juniperrc-config for controlling juniper.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Juniperrc extends Derived implements JuniperrcConfig.Producer {
+
+ // List of all fields that should be bolded.
+ private Set<String> boldingFields = new java.util.LinkedHashSet<>();
+
+ /**
+ * Constructs a new juniper rc instance for a given search object. This will derive the configuration automatically,
+ * so there is no need to call {@link #derive(com.yahoo.searchdefinition.Search)}.
+ *
+ * @param search The search model to use for deriving.
+ */
+ public Juniperrc(Search search) {
+ derive(search);
+ }
+
+ // Inherit doc from Derived.
+ @Override
+ protected void derive(Search search) {
+ super.derive(search);
+ for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
+ if (summaryField.getTransform() == SummaryTransform.BOLDED) {
+ boldingFields.add(summaryField.getName());
+ }
+ }
+ }
+
+ // Inherit doc from Derived.
+ protected String getDerivedName() {
+ return "juniperrc";
+ }
+
+ @Override
+ public void getConfig(JuniperrcConfig.Builder builder) {
+ if (boldingFields.size() != 0) {
+ builder.prefix(true);
+ for (String name : boldingFields) {
+ builder.override(new JuniperrcConfig.Override.Builder()
+ .fieldname(name)
+ .length(65536)
+ .max_matches(1)
+ .min_length(8192)
+ .surround_max(65536));
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java
new file mode 100644
index 00000000000..c1176807519
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.RankType;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * The definition of a rank type used for native rank features.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NativeRankTypeDefinition {
+
+ /** The type this defines */
+ private RankType type;
+
+ /** The rank tables of this rank type */
+ private List<NativeTable> rankTables = new java.util.ArrayList<>();
+
+ public NativeRankTypeDefinition(RankType type) {
+ this.type = type;
+ }
+
+ public RankType getType() {
+ return type;
+ }
+
+ public void addTable(NativeTable table) {
+ rankTables.add(table);
+ }
+
+ /** Returns an unmodifiable list of the tables in this type definition */
+ public Iterator<NativeTable> rankSettingIterator() {
+ return Collections.unmodifiableList(rankTables).iterator();
+ }
+
+ public String toString() {
+ return "native definition of rank type '" + type + "'";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java
new file mode 100644
index 00000000000..18856627b70
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.RankType;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * A set of rank type definitions used for native rank features.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NativeRankTypeDefinitionSet {
+
+ /** The name of this rank definition set */
+ private String name;
+
+ /** The unmodifiable rank type implementations in this set */
+ private final Map<RankType, NativeRankTypeDefinition> typeDefinitions;
+
+ /** Returns the default rank type (about) */
+ public static RankType getDefaultRankType() { return RankType.ABOUT; }
+
+ public NativeRankTypeDefinitionSet(String name) {
+ this.name = name;
+
+ Map<RankType, NativeRankTypeDefinition> typeDefinitions = new java.util.LinkedHashMap<>();
+ typeDefinitions.put(RankType.IDENTITY, createIdentityRankType(RankType.IDENTITY));
+ typeDefinitions.put(RankType.ABOUT, createAboutRankType(RankType.ABOUT));
+ typeDefinitions.put(RankType.TAGS, createTagsRankType(RankType.TAGS));
+ typeDefinitions.put(RankType.EMPTY, createEmptyRankType(RankType.EMPTY));
+ this.typeDefinitions = Collections.unmodifiableMap(typeDefinitions);
+ }
+
+ private NativeRankTypeDefinition createEmptyRankType(RankType type) {
+ NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+ rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)"));
+ rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)"));
+ rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)"));
+ rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)"));
+ rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)"));
+ return rank;
+ }
+
+ private NativeRankTypeDefinition createAboutRankType(RankType type) {
+ NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+ rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"));
+ rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"));
+ rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"));
+ rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"));
+ rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"));
+ return rank;
+ }
+
+ private NativeRankTypeDefinition createIdentityRankType(RankType type) {
+ NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+ rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)"));
+ rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"));
+ rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)"));
+ rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)"));
+ rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"));
+ return rank;
+ }
+
+ private NativeRankTypeDefinition createTagsRankType(RankType type) {
+ NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+ rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"));
+ rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"));
+ rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"));
+ rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"));
+ rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)"));
+ return rank;
+ }
+
+ /**
+ * Returns a rank type definition if given an existing rank type name,
+ * or null if given a rank type which has no native implementation (meaning somebody forgot to add it),
+ */
+ public NativeRankTypeDefinition getRankTypeDefinition(RankType type) {
+ if (type == RankType.DEFAULT)
+ type = getDefaultRankType();
+ return typeDefinitions.get(type);
+ }
+
+ /** Returns an unmodifiable map of the type definitions in this */
+ public Map<RankType, NativeRankTypeDefinition> types() { return typeDefinitions; }
+
+ public String toString() {
+ return "native rank type definitions " + name;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java
new file mode 100644
index 00000000000..512d4a37647
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+/**
+ * A named rank table of a certain type.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NativeTable {
+
+ private String name;
+
+ private Type type;
+
+ /** A table type enumeration */
+ public static class Type {
+
+ public static Type FIRST_OCCURRENCE = new Type("firstOccurrenceTable");
+ public static Type OCCURRENCE_COUNT = new Type("occurrenceCountTable");
+ public static Type WEIGHT = new Type("weightTable");
+ public static Type PROXIMITY = new Type("proximityTable");
+ public static Type REVERSE_PROXIMITY = new Type("reverseProximityTable");
+
+ private String name;
+
+ private Type(String name) {
+ this.name = name;
+ }
+
+ public String getName() { return name; }
+
+ public boolean equals(Object object) {
+ if (!(object instanceof Type)) {
+ return false;
+ }
+ Type other = (Type)object;
+ return this.name.equals(other.name);
+ }
+
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ public String toString() {
+ return getName();
+ }
+ }
+
+ public NativeTable(Type type, String name) {
+ this.type = type;
+ this.name = name;
+ }
+
+ public Type getType() { return type; }
+
+ public String getName() { return name; }
+
+ public int hashCode() {
+ return type.hashCode() + 17*name.hashCode();
+ }
+
+ public boolean equals(Object object) {
+ if (! (object instanceof NativeTable)) return false;
+ NativeTable other = (NativeTable)object;
+ return other.getName().equals(this.getName()) && other.getType().equals(this.getType());
+ }
+
+ public String toString() {
+ return getType() + ": " + getName();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
new file mode 100644
index 00000000000..183bfd6ddd4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.Search;
+import java.util.Map;
+
+/**
+ * The derived rank profiles of a search definition
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class RankProfileList extends Derived implements RankProfilesConfig.Producer {
+
+ private RawRankProfile defaultProfile;
+
+ private Map<String, RawRankProfile> rankProfiles=new java.util.LinkedHashMap<>();
+
+ /**
+ * Creates a rank profile
+ *
+ * @param search the search definition this is a rank profile from
+ * @param attributeFields the attribute fields to create a ranking for
+ */
+ public RankProfileList(Search search, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry) {
+ setName(search.getName());
+ deriveRankProfiles(rankProfileRegistry, search, attributeFields);
+ }
+
+ private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, Search search, AttributeFields attributeFields) {
+ defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), attributeFields);
+ rankProfiles.put(defaultProfile.getName(), defaultProfile);
+
+ for (RankProfile rank : rankProfileRegistry.localRankProfiles(search)) {
+ if ("default".equals(rank.getName()))
+ continue;
+ RawRankProfile rawRank=new RawRankProfile(rank, attributeFields);
+ rankProfiles.put(rawRank.getName(), rawRank);
+ }
+ }
+
+ public Map<String, RawRankProfile> getRankProfiles() {
+ return rankProfiles;
+ }
+
+ /** @return A named raw rank profile, or null if it is not present */
+ public RawRankProfile getRankProfile(String name) {
+ return rankProfiles.get(name);
+ }
+
+ @Override
+ public String getDerivedName() { return "rank-profiles"; }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ for (RawRankProfile rank : rankProfiles.values() ) {
+ rank.getConfig(builder);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
new file mode 100644
index 00000000000..cf3b10ccadd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -0,0 +1,374 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import java.util.*;
+
+/**
+ * A rank profile derived from a search definition, containing exactly the features available natively in the server
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class RawRankProfile implements RankProfilesConfig.Producer {
+
+ private String name;
+
+ public String getName() {
+ return name;
+ }
+
+ public String toString() {
+ return " rank profile " + name;
+ }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName());
+ getRankProperties(b);
+ builder.rankprofile(b);
+ }
+
+ private void getRankProperties(RankProfilesConfig.Rankprofile.Builder b) {
+ RankProfilesConfig.Rankprofile.Fef.Builder fefB = new RankProfilesConfig.Rankprofile.Fef.Builder();
+ for (Map.Entry<String, Object> e : configProperties.entrySet()) {
+ String key = e.getKey().replaceFirst(".part\\d+$", "");
+ String val = e.getValue().toString();
+ fefB.property(new RankProfilesConfig.Rankprofile.Fef.Property.Builder().name(key).value(val));
+ }
+ b.fef(fefB);
+ }
+
+ // TODO: These are to expose coupling between the strings used here and elsewhere
+ public final static String summaryFeatureFefPropertyPrefix = "vespa.summary.feature";
+ public final static String rankFeatureFefPropertyPrefix = "vespa.dump.feature";
+
+ /**
+ * Returns an immutable view of the config properties this returns
+ */
+ public Map<String, Object> configProperties() {
+ return Collections.unmodifiableMap(configProperties);
+ }
+
+ private final Map<String, Object> configProperties;
+
+ /**
+ * Creates a raw rank profile from the given rank profile
+ */
+ public RawRankProfile(RankProfile rankProfile, AttributeFields attributeFields) {
+ this.name = rankProfile.getName();
+ configProperties = new Deriver(rankProfile, attributeFields).derive();
+ }
+
+ private static class Deriver {
+
+ /**
+ * The field rank settings of this profile
+ */
+ private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>();
+
+ private RankingExpression firstPhaseRanking = null;
+
+ private RankingExpression secondPhaseRanking = null;
+
+ private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>();
+
+ private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>();
+
+ private List<RankProfile.RankProperty> rankProperties = new ArrayList<>();
+
+ /**
+ * Rank properties for weight settings to make these available to feature executors
+ */
+ private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>();
+
+ private boolean ignoreDefaultRankFeatures = false;
+
+ private RankProfile.MatchPhaseSettings matchPhaseSettings = null;
+
+ private int rerankCount = -1;
+ private int keepRankCount = -1;
+ private int numThreadsPerSearch = -1;
+ private int numSearchPartitions = -1;
+ private double termwiseLimit = 1.0;
+ private double rankScoreDropLimit = -Double.MAX_VALUE;
+
+ /**
+ * The rank type definitions used to derive settings for the native rank features
+ */
+ private NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default");
+
+ private RankProfile rankProfile;
+
+ private Set<String> filterFields = new java.util.LinkedHashSet<>();
+
+ /**
+ * Creates a raw rank profile from the given rank profile
+ */
+ public Deriver(RankProfile rankProfile, AttributeFields attributeFields) {
+ this.rankProfile = rankProfile.compile();
+ deriveRankingFeatures(this.rankProfile);
+ deriveRankTypeSetting(this.rankProfile, attributeFields);
+ deriveFilterFields(this.rankProfile);
+ deriveWeightProperties(this.rankProfile);
+ }
+
+ private void deriveFilterFields(RankProfile rp) {
+ filterFields.addAll(rp.allFilterFields());
+ }
+
+ public void deriveRankingFeatures(RankProfile rankProfile) {
+ firstPhaseRanking = rankProfile.getFirstPhaseRanking();
+ secondPhaseRanking = rankProfile.getSecondPhaseRanking();
+ summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures());
+ rankFeatures = rankProfile.getRankFeatures();
+ rerankCount = rankProfile.getRerankCount();
+ matchPhaseSettings = rankProfile.getMatchPhaseSettings();
+ numThreadsPerSearch = rankProfile.getNumThreadsPerSearch();
+ numSearchPartitions = rankProfile.getNumSearchPartitions();
+ termwiseLimit = rankProfile.getTermwiseLimit();
+ keepRankCount = rankProfile.getKeepRankCount();
+ rankScoreDropLimit = rankProfile.getRankScoreDropLimit();
+ ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures();
+ rankProperties = new ArrayList<>(rankProfile.getRankProperties());
+ derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros());
+ }
+
+ private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) {
+ if (macros.isEmpty()) return;
+ Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>();
+ for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) {
+ expressionMacros.put(macro.getKey(), macro.getValue().toExpressionMacro());
+ }
+
+ Map<String, String> macroProperties = new LinkedHashMap<>();
+ macroProperties.putAll(deriveMacroProperties(expressionMacros));
+ if (firstPhaseRanking != null) {
+ macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values())));
+ }
+ if (secondPhaseRanking != null) {
+ macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values())));
+ }
+ for (Map.Entry<String, String> e : macroProperties.entrySet()) {
+ rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue()));
+ }
+ SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties);
+ replaceMacroSummaryFeatures(context);
+ }
+
+ private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) {
+ SerializationContext context = new SerializationContext(eMacros);
+ for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) {
+ String script = e.getValue().getBody().getRoot().toString(context, null, null);
+ context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script);
+ }
+ return context.serializedFunctions();
+ }
+
+ private void replaceMacroSummaryFeatures(SerializationContext context) {
+ if (summaryFeatures == null) return;
+ Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>();
+ for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) {
+ ReferenceNode referenceNode = i.next();
+ // Is the feature a macro?
+ if (context.getFunction(referenceNode.getName()) != null) {
+ context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()),
+ referenceNode.toString(context, null, null));
+ ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput());
+ macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode);
+ i.remove(); // Will add the expanded one in next block
+ }
+ }
+ // Then, replace the summary features that were macros
+ for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) {
+ summaryFeatures.add(e.getValue());
+ }
+ }
+
+ private void deriveWeightProperties(RankProfile rankProfile) {
+
+ for (RankProfile.RankSetting setting : rankProfile.rankSettings()) {
+ if (!setting.getType().equals(RankProfile.RankSetting.Type.WEIGHT)) {
+ continue;
+ }
+ boostAndWeightRankProperties.add(new RankProfile.RankProperty("vespa.fieldweight." + setting.getFieldName(),
+ String.valueOf(setting.getIntValue())));
+ }
+ }
+
+ /**
+ * Adds the type boosts from a rank profile
+ */
+ private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) {
+ for (Iterator<RankProfile.RankSetting> i = rankProfile.rankSettingIterator(); i.hasNext(); ) {
+ RankProfile.RankSetting setting = i.next();
+ if (!setting.getType().equals(RankProfile.RankSetting.Type.RANKTYPE)) continue;
+
+ deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields,
+ hasDefaultRankTypeSetting(rankProfile, setting.getFieldName()));
+ }
+ }
+
+ public void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) {
+ if (isDefaultSetting) return;
+
+ NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType);
+ if (definition == null) throw new IllegalArgumentException("In field '" + fieldName + "': " +
+ rankType + " is known but has no implementation. " +
+ "Supported rank types: " +
+ nativeRankTypeDefinitions.types().keySet());
+
+ FieldRankSettings settings = deriveFieldRankSettings(fieldName);
+ for (Iterator<NativeTable> i = definition.rankSettingIterator(); i.hasNext(); ) {
+ NativeTable table = i.next();
+ // only add index field tables if we are processing an index field and
+ // only add attribute field tables if we are processing an attribute field
+ if ((FieldRankSettings.isIndexFieldTable(table) && attributeFields.getAttribute(fieldName) == null) ||
+ (FieldRankSettings.isAttributeFieldTable(table) && attributeFields.getAttribute(fieldName) != null)) {
+ settings.addTable(table);
+ }
+ }
+ }
+
+ private boolean hasDefaultRankTypeSetting(RankProfile rankProfile, String fieldName) {
+ RankProfile.RankSetting setting =
+ rankProfile.getRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE);
+ return setting != null && setting.getValue().equals(RankType.DEFAULT);
+ }
+
+ public FieldRankSettings deriveFieldRankSettings(String fieldName) {
+ FieldRankSettings settings = fieldRankSettings.get(fieldName);
+ if (settings == null) {
+ settings = new FieldRankSettings(fieldName);
+ fieldRankSettings.put(fieldName, settings);
+ }
+ return settings;
+ }
+
+ /**
+ * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file
+ *
+ * @return map of the derived properties
+ */
+ public Map<String, Object> derive() {
+ Map<String, Object> props = new LinkedHashMap<>();
+ int i = 0;
+ for (RankProfile.RankProperty property : rankProperties) {
+ if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) {
+ // Could have been set by macro expansion. Set expressions, then skip this property.
+ try {
+ firstPhaseRanking = new RankingExpression(property.getValue());
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse second phase expression", e);
+ }
+ continue;
+ }
+ if ("rankingExpression(secondphase).rankingScript".equals(property.getName())) {
+ try {
+ secondPhaseRanking = new RankingExpression(property.getValue());
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse second phase expression", e);
+ }
+ continue;
+ }
+ props.put(property.getName() + ".part" + i, property.getValue());
+ i++;
+ }
+ props.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase"));
+ props.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase"));
+ for (FieldRankSettings settings : fieldRankSettings.values()) {
+ props.putAll(settings.deriveRankProperties(i));
+ }
+ i = 0;
+ for (RankProfile.RankProperty property : boostAndWeightRankProperties) {
+ props.put(property.getName() + ".part" + i, property.getValue());
+ i++;
+ }
+ i = 0;
+ for (ReferenceNode feature : summaryFeatures) {
+ props.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString());
+ i++;
+ }
+ i = 0;
+ for (ReferenceNode feature : rankFeatures) {
+ props.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString());
+ i++;
+ }
+ if (numThreadsPerSearch > 0) {
+ props.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + "");
+ }
+ if (numSearchPartitions >= 0) {
+ props.put("vespa.matching.numsearchpartitions", numSearchPartitions + "");
+ }
+ if (termwiseLimit < 1.0) {
+ props.put("vespa.matching.termwise_limit", termwiseLimit + "");
+ }
+ if (matchPhaseSettings != null) {
+ props.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute());
+ props.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "");
+ props.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "");
+ props.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "");
+ props.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "");
+ props.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "");
+ RankProfile.DiversitySettings diversitySettings = rankProfile.getMatchPhaseSettings().getDiversity();
+ if (diversitySettings != null) {
+ props.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute());
+ props.put("vespa.matchphase.diversity.mingroups", diversitySettings.getMinGroups());
+ props.put("vespa.matchphase.diversity.cutoff.factor", diversitySettings.getCutoffFactor());
+ props.put("vespa.matchphase.diversity.cutoff.strategy", diversitySettings.getCutoffStrategy());
+ }
+ }
+ if (rerankCount > -1) {
+ props.put("vespa.hitcollector.heapsize", rerankCount + "");
+ }
+ if (keepRankCount > -1) {
+ props.put("vespa.hitcollector.arraysize", keepRankCount + "");
+ }
+ if (rankScoreDropLimit > -Double.MAX_VALUE) {
+ props.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "");
+ }
+ if (ignoreDefaultRankFeatures) {
+ props.put("vespa.dump.ignoredefaultfeatures", true);
+ }
+ Iterator filterFieldsIterator = filterFields.iterator();
+ while (filterFieldsIterator.hasNext()) {
+ String fieldName = (String) filterFieldsIterator.next();
+ props.put("vespa.isfilterfield." + fieldName + ".part42", true);
+ }
+ for (Map.Entry<String, String> attributeType : rankProfile.getAttributeTypes().entrySet()) {
+ props.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue());
+ }
+ for (Map.Entry<String, String> queryFeatureType : rankProfile.getQueryFeatureTypes().entrySet()) {
+ props.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue());
+ }
+ if (props.size() >= 1000000) throw new RuntimeException("Too many rank properties");
+ return props;
+ }
+
+ private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) {
+ Map<String, String> ret = new LinkedHashMap<>();
+ if (expression == null) {
+ return ret;
+ }
+ String name = expression.getName();
+ if ("".equals(name)) {
+ name = phase;
+ }
+ if (expression.getRoot() instanceof ReferenceNode) {
+ ret.put("vespa.rank." + phase, expression.getRoot().toString());
+ } else {
+ ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")");
+ ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString());
+ }
+ return ret;
+ }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java
new file mode 100644
index 00000000000..69133e31fb3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java
@@ -0,0 +1,105 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.DataTypeName;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.Search;
+
+import java.util.*;
+
+/**
+ * <p>A class which can reorder a list of search definitions such that any supertype
+ * always preceed any subtype. Subject to this condition the given order
+ * is preserved (the minimal reordering is done).</p>
+ *
+ * <p>This class is <b>not</b> multithread safe. Only one ordering must be done
+ * at the time in any instance.</p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SearchOrderer {
+
+ /** A map from DataTypeName to the Search defining them */
+ private Map<DataTypeName, Search> documentNameToSearch=new java.util.HashMap<>();
+
+ /**
+ * Reorders the given list of search definitions such that any supertype
+ * always preceed any subtype. Subject to this condition the given order
+ * is preserved (the minimal reordering is done).
+ *
+ * @return a new list containing the same search instances in the right order
+ */
+ public List<Search> order(List<Search> unordered) {
+ Collections.sort(unordered, new Comparator<Search>() {
+ @Override
+ public int compare(Search lhs, Search rhs) {
+ return lhs.getName().compareTo(rhs.getName());
+ }
+ });
+
+ // No, this is not a fast algorithm...
+ indexOnDocumentName(unordered);
+ List<Search> ordered=new java.util.ArrayList<>(unordered.size());
+ List<Search> moveOutwards=new java.util.ArrayList<>();
+ for (Search search: unordered) {
+ if (containsInherited(ordered,search)) {
+ addOrdered(ordered,search,moveOutwards);
+ }
+ else {
+ moveOutwards.add(search);
+ }
+ }
+
+ // Any leftovers means we have search definitions with undefined inheritants.
+ // This is warned about elsewhere.
+ ordered.addAll(moveOutwards);
+
+ documentNameToSearch.clear();
+ return ordered;
+ }
+
+ private void addOrdered(List<Search> ordered,Search search,List<Search> moveOutwards) {
+ ordered.add(search);
+ Search eligibleMove;
+ do {
+ eligibleMove=removeFirstEligibleMoveOutwards(moveOutwards,ordered);
+ if (eligibleMove!=null)
+ ordered.add(eligibleMove);
+ } while (eligibleMove!=null);
+ }
+
+ /** Removes and returns the first search from the move list which can now be added, or null if none */
+ private Search removeFirstEligibleMoveOutwards(List<Search> moveOutwards,List<Search> ordered) {
+ for (Search move : moveOutwards) {
+ if (containsInherited(ordered,move)) {
+ moveOutwards.remove(move);
+ return move;
+ }
+ }
+ return null;
+ }
+
+ private boolean containsInherited(List<Search> list,Search search) {
+ if (search.getDocument() == null) {
+ return true;
+ }
+ for (SDDocumentType sdoc : search.getDocument().getInheritedTypes() ) {
+ DataTypeName inheritedName=sdoc.getDocumentName();
+ if ("document".equals(inheritedName.getName())) continue;
+ Search inheritedSearch=documentNameToSearch.get(inheritedName);
+ if (!list.contains(inheritedSearch))
+ return false;
+ }
+ return true;
+ }
+
+ private void indexOnDocumentName(List<Search> searches) {
+ documentNameToSearch.clear();
+ for (Search search : searches) {
+ if (search.getDocument() != null) {
+ documentNameToSearch.put(search.getDocument().getDocumentName(),search);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
new file mode 100644
index 00000000000..357e0d40f49
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import java.util.List;
+
+/**
+ * A list of derived summaries
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class Summaries extends Derived implements SummaryConfig.Producer {
+
+ private List<SummaryClass> summaries=new java.util.ArrayList<>(1);
+
+ public Summaries(Search search, DeployLogger deployLogger) {
+ // Make sure the default is first
+ summaries.add(new SummaryClass(search,search.getSummary("default"), deployLogger));
+ for (DocumentSummary summary : search.getSummaries().values()) {
+ if (!summary.getName().equals("default"))
+ summaries.add(new SummaryClass(search,summary, deployLogger));
+ }
+ }
+
+ protected String getDerivedName() { return "summary"; }
+
+ @Override
+ public void getConfig(SummaryConfig.Builder builder) {
+ builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode());
+ for (SummaryClass summaryClass : summaries) {
+ builder.classes(summaryClass.getSummaryClassConfig());
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
new file mode 100644
index 00000000000..d21523caea2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.prelude.fastsearch.DocsumDefinitionSet;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.logging.Level;
+
+/**
+ * A summary derived from a search definition.
+ * Each summary definition have at least one summary, the default
+ * which has the same name as the search definition.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SummaryClass extends Derived {
+
+ /** True if this summary class needs to access summary information on disk */
+ private boolean accessingDiskSummary=false;
+
+ /** The summary fields of this indexed by name */
+ private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>();
+
+ private DeployLogger deployLogger = new BaseDeployLogger();
+
+ private final Random random = new Random(7);
+
+ /**
+ * Creates a summary class from a search definition summary
+ *
+ * @param deployLogger a {@link DeployLogger}
+ */
+ public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) {
+ this.deployLogger = deployLogger;
+ deriveName(summary);
+ deriveFields(search,summary);
+ deriveImplicitFields(summary);
+ }
+
+ private void deriveName(DocumentSummary summary) {
+ setName(summary.getName());
+ }
+
+ /** MUST be called after all other fields are added */
+ private void deriveImplicitFields(DocumentSummary summary) {
+ if (summary.getName().equals("default")) {
+ addField("documentid", DataType.STRING);
+ }
+ }
+
+ private void deriveFields(Search search, DocumentSummary summary) {
+ for (SummaryField summaryField : summary.getSummaryFields()) {
+ if (!accessingDiskSummary && search.isAccessingDiskSummary(summaryField)) {
+ accessingDiskSummary = true;
+ }
+ addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform());
+ }
+ }
+
+ private void addField(String name, DataType type) {
+ addField(name, type, null);
+ }
+
+ private void addField(String name, DataType type, SummaryTransform transform) {
+ if (fields.containsKey(name)) {
+ SummaryClassField sf = fields.get(name);
+ if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) {
+ deployLogger.log(Level.WARNING, "Conflicting definition of field " + name + ". " +
+ "Declared as type " + sf.getType() + " and " +
+ type);
+ }
+ } else {
+ fields.put(name, new SummaryClassField(name, type, transform));
+ }
+ }
+
+
+ /** Returns an iterator of the fields of this summary. Removes on this iterator removes the field from this summary */
+ public Iterator<SummaryClassField> fieldIterator() {
+ return fields.values().iterator();
+ }
+
+ public void addField(SummaryClassField field) {
+ fields.put(field.getName(),field);
+ }
+
+ /** Returns the writable map of fields of this summary */ // TODO: Make read only, move writers to iterator/addField
+ public Map<String,SummaryClassField> getFields() { return fields; }
+
+ public SummaryClassField getField(String name) {
+ return fields.get(name);
+ }
+
+ public int getFieldCount() { return fields.size(); }
+
+ public int hashCode() {
+ int number = 1;
+ int hash = getName().hashCode();
+ for (Iterator i = fieldIterator(); i.hasNext(); ) {
+ SummaryClassField field = (SummaryClassField)i.next();
+ hash += number * (field.getName().hashCode() +
+ 17*field.getType().getName().hashCode());
+ number++;
+ }
+ if (hash < 0)
+ hash *= -1;
+ return hash;
+ }
+
+ public SummaryConfig.Classes.Builder getSummaryClassConfig() {
+ SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder();
+ int id = hashCode();
+ if (id == DocsumDefinitionSet.SLIME_MAGIC_ID) {
+ deployLogger.log(Level.WARNING, "Summary class '" + getName() + "' hashes to the SLIME_MAGIC_ID '" + id +
+ "'. This is unlikely but I autofix it for you by adding a random number.");
+ id += random.nextInt();
+ }
+ classBuilder.
+ id(id).
+ name(getName());
+ for (SummaryClassField field : fields.values() ) {
+ classBuilder.fields(new SummaryConfig.Classes.Fields.Builder().
+ name(field.getName()).
+ type(field.getType().getName()));
+ }
+ return classBuilder;
+ }
+
+ protected String getDerivedName() { return "summary"; }
+
+ public String toString() {
+ return "summary class " + getName();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
new file mode 100644
index 00000000000..bb1dd87f314
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+/**
+ * A summary field derived from a search definition
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SummaryClassField {
+
+ private final String name;
+
+ private final Type type;
+
+ /** The summary field type enumeration */
+ public enum Type {
+
+ BYTE("byte"),
+ SHORT("short"),
+ INTEGER("integer"),
+ INT64("int64"),
+ FLOAT("float"),
+ DOUBLE("double"),
+ STRING("string"),
+ DATA("data"),
+ LONGSTRING("longstring"),
+ LONGDATA("longdata"),
+ XMLSTRING("xmlstring"),
+ FEATUREDATA("featuredata"),
+ JSONSTRING("jsonstring");
+
+ private String name;
+
+ private Type(String name) {
+ this.name = name;
+ }
+
+ /** Returns the name of this type */
+ public String getName() {
+ return name;
+ }
+
+ public String toString() {
+ return "type: " + name;
+ }
+ }
+
+ public SummaryClassField(String name, DataType type, SummaryTransform transform) {
+ this.name = name;
+ this.type = convertDataType(type, transform);
+ }
+
+ public String getName() { return name; }
+
+ public Type getType() { return type; }
+
+ /** Converts to the right summary field type from a field datatype and a transform*/
+ public static Type convertDataType(DataType fieldType, SummaryTransform transform) {
+ FieldValue fval = fieldType.createFieldValue();
+ if (fval instanceof StringFieldValue) {
+ if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) {
+ return Type.FEATUREDATA;
+ } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) {
+ return Type.FEATUREDATA;
+ } else {
+ return Type.LONGSTRING;
+ }
+ } else if (fval instanceof IntegerFieldValue) {
+ return Type.INTEGER;
+ } else if (fval instanceof LongFieldValue) {
+ return Type.INT64;
+ } else if (fval instanceof FloatFieldValue) {
+ return Type.FLOAT;
+ } else if (fval instanceof DoubleFieldValue) {
+ return Type.DOUBLE;
+ } else if (fval instanceof ByteFieldValue) {
+ return Type.BYTE;
+ } else if (fval instanceof Raw) {
+ return Type.DATA;
+ } else if (fval instanceof Struct) {
+ return Type.JSONSTRING;
+ } else if (fval instanceof PredicateFieldValue) {
+ return Type.STRING;
+ } else if (fval instanceof TensorFieldValue) {
+ return Type.JSONSTRING;
+ } else if (fieldType instanceof CollectionDataType) {
+ if (transform != null && transform.equals(SummaryTransform.POSITIONS)) {
+ return Type.XMLSTRING;
+ } else {
+ return Type.JSONSTRING;
+ }
+ } else if (fieldType instanceof MapDataType) {
+ return Type.JSONSTRING;
+ } else {
+ throw new IllegalArgumentException("Don't know which summary type to " +
+ "convert " + fieldType + " to");
+ }
+ }
+
+ public String toString() {
+ return "summary class field " + name;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
new file mode 100644
index 00000000000..d59c671e6a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.config.search.SummarymapConfig;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * A summary map (describing search-time summary field transformations)
+ * derived from a search definition
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SummaryMap extends Derived implements SummarymapConfig.Producer {
+
+ private Map<String,FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>();
+
+ /** Crateate a summary map from a search definition */
+ public SummaryMap(Search search, Summaries summaries) {
+ derive(search, summaries);
+ }
+
+ protected void derive(Search search, Summaries summaries) {
+ // TODO: This should really derive from the 'summaries' argument. Bug?
+ for (DocumentSummary documentSummary : search.getSummaries().values()) {
+ derive(documentSummary);
+ }
+ super.derive(search);
+ }
+
+ protected void derive(SDField field, Search search) {
+ }
+
+ private void derive(DocumentSummary documentSummary) {
+ for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+ if (summaryField.getTransform()== SummaryTransform.NONE) continue;
+
+ if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE ||
+ summaryField.getTransform()==SummaryTransform.DISTANCE ||
+ summaryField.getTransform()==SummaryTransform.GEOPOS ||
+ summaryField.getTransform()==SummaryTransform.POSITIONS) {
+ resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(),
+ summaryField.getTransform(),
+ summaryField.getSingleSource()));
+ } else {
+ // Note: Currently source mapping is handled in the indexing statement,
+ // by creating a summary field for each of the values
+ // This works, but is suboptimal. We could consolidate to a minimal set and
+ // use the right value from the minimal set as the third parameter here,
+ // and add "override" commands to multiple static values
+ resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(),
+ summaryField.getTransform(),
+ summaryField.getName()));
+ }
+ }
+ }
+
+ /** Returns a read-only iterator of the FieldResultTransforms of this summary map */
+ public Iterator resultTransformIterator() {
+ return Collections.unmodifiableCollection(resultTransforms.values()).iterator();
+ }
+
+ protected String getDerivedName() { return "summarymap"; }
+
+ /** Returns the command name of a transform */
+ private String getCommand(SummaryTransform transform) {
+ if (transform.equals(SummaryTransform.DISTANCE))
+ return "absdist";
+ else if (transform.isDynamic())
+ return "dynamicteaser";
+ else
+ return transform.getName();
+ }
+
+ /**
+ * Does this summary command name stand for a dynamic transform?
+ * We need this because some model information is shared through configs instead of model - see usage
+ */
+ public static boolean isDynamicCommand(String commandName) {
+ return (commandName.equals("dynamicteaser") || commandName.equals("smartsummary"));
+ }
+
+ @Override
+ public void getConfig(SummarymapConfig.Builder builder) {
+ builder.defaultoutputclass(-1);
+ for (FieldResultTransform frt : resultTransforms.values()) {
+ SummarymapConfig.Override.Builder oB = new SummarymapConfig.Override.Builder()
+ .field(frt.getFieldName())
+ .command(getCommand(frt.getTransform()));
+ if (frt.getTransform().isDynamic() ||
+ frt.getTransform().equals(SummaryTransform.ATTRIBUTE) ||
+ frt.getTransform().equals(SummaryTransform.DISTANCE) ||
+ frt.getTransform().equals(SummaryTransform.GEOPOS) ||
+ frt.getTransform().equals(SummaryTransform.POSITIONS) ||
+ frt.getTransform().equals(SummaryTransform.TEXTEXTRACTOR))
+ {
+ oB.arguments(frt.getArgument());
+ } else {
+ oB.arguments("");
+ }
+ builder.override(oB);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
new file mode 100644
index 00000000000..65698675884
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
@@ -0,0 +1,275 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.searchdefinition.FieldSets;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Vertical streaming matcher field specification
+ */
+public class VsmFields extends Derived implements VsmfieldsConfig.Producer {
+
+ private final Map<String, StreamingField> fields=new LinkedHashMap<>();
+ private final Map<String, StreamingDocumentType> doctypes=new LinkedHashMap<>();
+
+ public VsmFields(Search search) {
+ addSearchdefinition(search);
+ }
+
+ private void addSearchdefinition(Search search) {
+ derive(search);
+ }
+
+ protected void derive(SDDocumentType document,Search search) {
+ super.derive(document, search);
+ StreamingDocumentType docType=getDocumentType(document.getName());
+ if (docType == null) {
+ docType = new StreamingDocumentType(document.getName(), search.fieldSets());
+ doctypes.put(document.getName(), docType);
+ }
+ for (Object o : document.fieldSet()) {
+ derive(docType, (SDField) o);
+ }
+ }
+
+ protected void derive(StreamingDocumentType document, SDField field) {
+ if (field.usesStructOrMap()) {
+ for (SDField structField : field.getStructFields()) {
+ derive(document, structField); // Recursion
+ }
+ } else {
+
+ if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) )
+ return;
+
+ StreamingField streamingField=new StreamingField(field);
+ addField(streamingField.getName(),streamingField);
+ deriveIndices(document, field, streamingField);
+ }
+ }
+
+ private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField) {
+ if (field.doesIndexing()) {
+ addFieldToIndices(document, field.getName(), streamingField);
+ } else if (field.doesAttributing()) {
+ for (String indexName : field.getAttributes().keySet()) {
+ addFieldToIndices(document, indexName, streamingField);
+ }
+ }
+ }
+
+ private void addFieldToIndices(StreamingDocumentType document, String indexName, StreamingField streamingField) {
+ if (indexName.contains(".")) {
+ addFieldToIndices(document, indexName.substring(0,indexName.lastIndexOf(".")), streamingField); // Recursion
+ }
+ document.addIndexField(indexName, streamingField.getName());
+ }
+
+ private void addField(String name, StreamingField field) {
+ fields.put(name, field);
+ }
+
+ /** Returns a streaming index, or null if there is none with this name */
+ public StreamingDocumentType getDocumentType(String name) {
+ return doctypes.get(name);
+ }
+
+ public String getDerivedName() {
+ return "vsmfields";
+ }
+
+ @Override
+ public void getConfig(VsmfieldsConfig.Builder vsB) {
+ for (StreamingField streamingField : fields.values()) {
+ vsB.fieldspec(streamingField.getFieldSpecConfig());
+ }
+ for (StreamingDocumentType streamingDocType : doctypes.values()) {
+ vsB.documenttype(streamingDocType.getDocTypeConfig());
+ }
+ }
+
+ private static class StreamingField {
+
+ private final String name;
+
+ /** Whether this field does prefix matching by default */
+ private final Matching matching;
+
+ /** The type of this field */
+ private final Type type;
+
+ private final boolean isAttribute;
+
+ /** The streaming field type enumeration */
+ public static class Type {
+
+ public static Type INT8=new Type("int8","INT8");
+ public static Type INT16=new Type("int16","INT16");
+ public static Type INT32=new Type("int32","INT32");
+ public static Type INT64=new Type("int64","INT64");
+ public static Type FLOAT=new Type("float","FLOAT");
+ public static Type DOUBLE=new Type("double","DOUBLE");
+ public static Type STRING=new Type("string","AUTOUTF8");
+ public static Type UNSEARCHABLESTRING=new Type("string","NONE");
+
+ private String name;
+
+ private String searchMethod;
+
+ private Type(String name,String searchMethod) {
+ this.name=name;
+ this.searchMethod=searchMethod;
+ }
+
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ /** Returns the name of this type */
+ public String getName() { return name; }
+
+ public String getSearchMethod() { return searchMethod; }
+
+ public boolean equals(Object other) {
+ if ( ! (other instanceof Type)) return false;
+ return this.name.equals(((Type)other).name);
+ }
+
+ public String toString() {
+ return "type: " + name;
+ }
+
+ }
+
+ public StreamingField(SDField field) {
+ this(field.getName(),field.getDataType(),field.getMatching(), field.doesAttributing());
+ }
+
+ private StreamingField(String name,DataType sourceType, Matching matching, boolean isAttribute) {
+ this.name = name;
+ this.type = convertType(sourceType);
+ this.matching = matching;
+ this.isAttribute = isAttribute;
+ }
+
+ /** Converts to the right index type from a field datatype */
+ private static Type convertType(DataType fieldType) {
+ FieldValue fval = fieldType.createFieldValue();
+ if (fieldType.equals(DataType.FLOAT)) {
+ return Type.FLOAT;
+ } else if (fieldType.equals(DataType.LONG)) {
+ return Type.INT64;
+ } else if (fieldType.equals(DataType.DOUBLE)) {
+ return Type.DOUBLE;
+ } else if (fieldType.equals(DataType.BYTE)) {
+ return Type.INT8;
+ } else if (fieldType instanceof NumericDataType) {
+ return Type.INT32;
+ } else if (fval instanceof StringFieldValue) {
+ return Type.STRING;
+ } else if (fval instanceof Raw) {
+ return Type.STRING;
+ } else if (fval instanceof PredicateFieldValue) {
+ return Type.UNSEARCHABLESTRING;
+ } else if (fval instanceof TensorFieldValue) {
+ return Type.UNSEARCHABLESTRING;
+ } else if (fieldType instanceof CollectionDataType) {
+ return convertType(((CollectionDataType) fieldType).getNestedType());
+ } else {
+ throw new IllegalArgumentException("Don't know which streaming" +
+ " field type to " + "convert " + fieldType + " to");
+ }
+ }
+
+ public String getName() { return name; }
+
+ public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() {
+ VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder();
+ String matchingName = matching.getType().getName();
+ if (matching.getType().equals(Matching.Type.TEXT))
+ matchingName = "";
+ if (matching.getType() != Matching.Type.EXACT) {
+ if (matching.isPrefix()) {
+ matchingName = "prefix";
+ } else if (matching.isSubstring()) {
+ matchingName = "substring";
+ } else if (matching.isSuffix()) {
+ matchingName = "suffix";
+ }
+ }
+ if (type != Type.STRING) {
+ matchingName = "";
+ }
+ fB.name(getName())
+ .searchmethod(VsmfieldsConfig.Fieldspec.Searchmethod.Enum.valueOf(type.getSearchMethod()))
+ .arg1(matchingName)
+ .fieldtype(isAttribute
+ ? VsmfieldsConfig.Fieldspec.Fieldtype.ATTRIBUTE
+ : VsmfieldsConfig.Fieldspec.Fieldtype.INDEX);
+ if (matching.maxLength() != null) {
+ fB.maxlength(matching.maxLength());
+ }
+ return fB;
+ }
+
+ public boolean equals(Object o) {
+ if (o.getClass().equals(getClass())) {
+ StreamingField sf = (StreamingField)o;
+ return name.equals(sf.name) &&
+ matching.equals(sf.matching) &&
+ type.equals(sf.type);
+ }
+ return false;
+ }
+
+ }
+
+ private static class StreamingDocumentType {
+ private final String name;
+ private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>();
+ private final Map<String, FieldSet> userFieldSets;
+
+ public StreamingDocumentType(String name, FieldSets fieldSets) {
+ this.name=name;
+ userFieldSets = fieldSets.userFieldSets();
+ }
+
+ public VsmfieldsConfig.Documenttype.Builder getDocTypeConfig() {
+ VsmfieldsConfig.Documenttype.Builder dtB = new VsmfieldsConfig.Documenttype.Builder();
+ dtB.name(name);
+ Map<String, FieldSet> all = new LinkedHashMap<>();
+ all.putAll(fieldSets);
+ all.putAll(userFieldSets);
+ for (Map.Entry<String, FieldSet> e : all.entrySet()) {
+ VsmfieldsConfig.Documenttype.Index.Builder indB = new VsmfieldsConfig.Documenttype.Index.Builder();
+ indB.name(e.getValue().getName());
+ for (String field : e.getValue().getFieldNames()) {
+ indB.field(new VsmfieldsConfig.Documenttype.Index.Field.Builder().name(field));
+ }
+ dtB.index(indB);
+ }
+ return dtB;
+ }
+
+ public String getName() { return name; }
+
+ public void addIndexField(String indexName, String fieldName) {
+ FieldSet fs = fieldSets.get(indexName);
+ if (fs == null) {
+ fs = new FieldSet(indexName);
+ fieldSets.put(indexName, fs);
+ }
+ fs.addFieldName(fieldName);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
new file mode 100644
index 00000000000..aaf376f5cd9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
+
+import java.util.*;
+
+/**
+ * Vertical streaming matcher summary specification
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class VsmSummary extends Derived implements VsmsummaryConfig.Producer {
+ private Map<SummaryField, List<String>> summaryMap = new java.util.LinkedHashMap<>(1);
+
+ public VsmSummary(Search search) {
+ derive(search);
+ }
+
+ @Override
+ protected void derive(Search search) {
+ // Use the default class, as it is the superset
+ derive(search, search.getSummary("default"));
+ }
+
+ private void derive(Search search, DocumentSummary documentSummary) {
+ if (documentSummary==null) return;
+ for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+ List<String> from = toStringList(summaryField.sourceIterator());
+
+ if (doMapField(search, summaryField)) {
+ SDField sdField = search.getField(summaryField.getName());
+ if (sdField != null && PositionDataType.INSTANCE.equals(sdField.getDataType())) {
+ summaryMap.put(summaryField, Collections.singletonList(summaryField.getName()));
+ } else {
+ summaryMap.put(summaryField, from);
+ }
+ }
+ }
+ }
+
+ /**
+ * Don't include field in map if sources are the same as the struct sub fields for the SDField.
+ * But do map if not all do summarying.
+ * Don't map if not struct either.
+ * @param summaryField a {@link SummaryField}
+ */
+ private boolean doMapField(Search search, SummaryField summaryField) {
+ SDField sdField = search.getField(summaryField.getName());
+ SDDocumentType document = search.getDocument();
+ if (sdField==null || ((document != null) && (document.getField(summaryField.getName()) == sdField))) {
+ return true;
+ }
+ if (summaryField.getVsmCommand().equals(SummaryField.VsmCommand.FLATTENJUNIPER)) {
+ return true;
+ }
+ if (!sdField.usesStructOrMap()) {
+ return !(sdField.getName().equals(summaryField.getName()));
+ }
+ if (summaryField.getSourceCount()==sdField.getStructFields().size()) {
+ for (SummaryField.Source source : summaryField.getSources()) {
+ if (!sdField.getStructFields().contains(new SDField(search.getDocument(), source.getName(), sdField.getDataType()))) { // equals() uses just name
+ return true;
+ }
+ if (sdField.getStructField(source.getName())!=null && !sdField.getStructField(source.getName()).doesSummarying()) {
+ return true;
+ }
+ }
+ // The sources in the summary field are the same as the sub-fields in the SD field.
+ // All sub fields do summarying.
+ // Don't map.
+ return false;
+ }
+ return true;
+ }
+
+ private List<String> toStringList(Iterator i) {
+ List<String> ret = new ArrayList<>();
+ while (i.hasNext()) {
+ ret.add(i.next().toString());
+ }
+ return ret;
+ }
+
+ public String getDerivedName() {
+ return "vsmsummary";
+ }
+
+ @Override
+ public void getConfig(VsmsummaryConfig.Builder vB) {
+ for (Map.Entry<SummaryField, List<String>> entry : summaryMap.entrySet()) {
+ VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName());
+ for (String field : entry.getValue()) {
+ fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field));
+ }
+ fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString()));
+ vB.fieldmap(fmB);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java
new file mode 100644
index 00000000000..4e63729ea34
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java
new file mode 100644
index 00000000000..360b4d0d637
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived.validation;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.searchdefinition.derived.IndexingScript;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+
+/**
+ * @author <a href="mailto:mlidal@yahoo-inc.com">Mathias M Lidal</a>
+ */
+public class IndexStructureValidator extends Validator {
+
+ public IndexStructureValidator(DerivedConfiguration config, Search search) {
+ super(config, search);
+ }
+
+ public void validate() {
+ IndexingScript script = config.getIndexingScript();
+ for (Expression exp : script.expressions()) {
+ new OutputVisitor(search.getDocument(), exp).visit(exp);
+ }
+ }
+
+ private static class OutputVisitor extends ExpressionVisitor {
+
+ final SDDocumentType docType;
+ final Expression exp;
+
+ public OutputVisitor(SDDocumentType docType, Expression exp) {
+ this.docType = docType;
+ this.exp = exp;
+ }
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (!(exp instanceof OutputExpression)) {
+ return;
+ }
+ String fieldName = ((OutputExpression)exp).getFieldName();
+ if (docType.getField(fieldName) != null) {
+ return;
+ }
+ throw new IllegalArgumentException("Indexing expression '" + this.exp + "' refers to field '" +
+ fieldName + "' which does not exist in the index structure.");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java
new file mode 100644
index 00000000000..9b01881ddd8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived.validation;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+
+public class Validation {
+
+ public static void validate(DerivedConfiguration config, Search search) {
+ new IndexStructureValidator(config, search).validate();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java
new file mode 100644
index 00000000000..8c7c5afcb15
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived.validation;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+
+import java.util.logging.Logger;
+
+/**
+ * @author mathiasm
+ */
+public abstract class Validator {
+ protected DerivedConfiguration config;
+ protected Search search;
+
+ protected Validator(DerivedConfiguration config, Search search) {
+ this.config = config;
+ this.search = search;
+ }
+
+ public abstract void validate();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
new file mode 100644
index 00000000000..35f6a41f0d8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -0,0 +1,320 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.tensor.TensorType;
+
+import java.io.Serializable;
+import java.util.LinkedHashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * A search-time document attribute (per-document in-memory value).
+ * This belongs to the field defining the attribute.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public final class Attribute implements Cloneable, Serializable {
+
+ // Remember to change hashCode and equals when you add new fields
+
+ private String name;
+
+ private Type type;
+ private CollectionType collectionType;
+
+ /** True if only the enum information should be read from this attribute
+ * (i.e. the actual values are not relevant, only which documents have the
+ * same values) Used for collapsing and unique.
+ */
+ private boolean removeIfZero = false;
+ private boolean createIfNonExistent = false;
+ private boolean enableBitVectors = false;
+ private boolean enableOnlyBitVector = false;
+
+ private boolean fastSearch = false;
+ private boolean fastAccess = false;
+ private boolean huge = false;
+ private int arity = BooleanIndexDefinition.DEFAULT_ARITY;
+ private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND;
+ private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND;
+ private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD;
+ private Optional<TensorType> tensorType = Optional.empty();
+
+ private boolean isPosition = false;
+ private final Sorting sorting = new Sorting();
+
+ /** The aliases for this attribute */
+ private final Set<String> aliases = new LinkedHashSet<>();
+
+ /**
+ * True if this attribute should be returned during first pass of search.
+ * Null means make the default decision for this kind of attribute
+ */
+ private Boolean prefetch = null;
+
+ /** The attribute type enumeration */
+ public enum Type {
+ BYTE("byte", "INT8"),
+ SHORT("short", "INT16"),
+ INTEGER("integer", "INT32"),
+ LONG("long", "INT64"),
+ FLOAT("float", "FLOAT"),
+ DOUBLE("double", "DOUBLE"),
+ STRING("string", "STRING"),
+ PREDICATE("predicate", "PREDICATE"),
+ TENSOR("tensor", "TENSOR");
+
+ private final String myName; // different from what name() returns.
+ private final String exportAttributeTypeName;
+
+ private Type(String name, String exportAttributeTypeName) {
+ this.myName=name;
+ this.exportAttributeTypeName = exportAttributeTypeName;
+ }
+
+ public String getName() { return myName; }
+ public String getExportAttributeTypeName() { return exportAttributeTypeName; }
+
+ public String toString() {
+ return "type: " + myName;
+ }
+ }
+
+ /** The attribute collection type enumeration */
+ public enum CollectionType {
+
+ SINGLE("SINGLE"),
+ ARRAY("ARRAY"),
+ WEIGHTEDSET ("WEIGHTEDSET");
+
+ private final String name;
+
+ private CollectionType(String name) {
+ this.name=name;
+ }
+
+ public String getName() { return name; }
+
+ public String toString() {
+ return "collectiontype: " + name;
+ }
+ }
+
+ /** Creates an attribute with default settings */
+ public Attribute(String name,DataType fieldType) {
+ this(name,convertDataType(fieldType), convertCollectionType(fieldType));
+ setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false);
+ setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false);
+ }
+
+ public Attribute(String name,Type type, CollectionType collectionType) {
+ this.name=name;
+ setType(type);
+ setCollectionType(collectionType);
+ }
+
+ /**
+ * <p>Returns whether this attribute should be included in the "attributeprefetch" summary
+ * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc.
+ *
+ * <p>Single value attributes are prefetched by default if summary is true.
+ * Multi value attributes are not.</p>
+ */
+ public boolean isPrefetch() {
+ if (prefetch!=null) return prefetch.booleanValue();
+
+ if (CollectionType.SINGLE.equals(collectionType)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ /** Returns the prefetch value of this, null if the default is used. */
+ public Boolean getPrefetchValue() { return prefetch; }
+
+ public boolean isRemoveIfZero() { return removeIfZero; }
+ public boolean isCreateIfNonExistent(){ return createIfNonExistent; }
+ public boolean isEnabledBitVectors() { return enableBitVectors; }
+ public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; }
+ public boolean isFastSearch() { return fastSearch; }
+ public boolean isFastAccess() { return fastAccess; }
+ public boolean isHuge() { return huge; }
+ public boolean isPosition() { return isPosition; }
+
+ public int arity() { return arity; }
+ public long lowerBound() { return lowerBound; }
+ public long upperBound() { return upperBound; }
+ public double densePostingListThreshold() { return densePostingListThreshold; }
+ public Optional<TensorType> tensorType() { return tensorType; }
+
+ public Sorting getSorting() { return sorting; }
+
+ public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; }
+ public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; }
+
+ /**
+ * Sets whether this should be included in the "attributeprefetch" document summary.
+ * True or false to override default, null to use default
+ */
+ public void setPrefetch(Boolean prefetch) { this.prefetch = prefetch; }
+ public void setEnableBitVectors(boolean enableBitVectors) { this.enableBitVectors = enableBitVectors; }
+ public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; }
+ public void setFastSearch(boolean fastSearch) { this.fastSearch = fastSearch; }
+ public void setHuge(boolean huge) { this.huge = huge; }
+ public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; }
+ public void setPosition(boolean position) { this.isPosition = position; }
+ public void setArity(int arity) { this.arity = arity; }
+ public void setLowerBound(long lowerBound) { this.lowerBound = lowerBound; }
+ public void setUpperBound(long upperBound) { this.upperBound = upperBound; }
+ public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; }
+ public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); }
+
+ public String getName() { return name; }
+ public Type getType() { return type; }
+ public CollectionType getCollectionType() { return collectionType; }
+
+ public void setName(String name) { this.name=name; }
+ private void setType(Type type) { this.type=type; }
+ public void setCollectionType(CollectionType type) { this.collectionType=type; }
+
+ /** Converts to the right attribute type from a field datatype */
+ public static Type convertDataType(DataType fieldType) {
+ FieldValue fval = fieldType.createFieldValue();
+ if (fval instanceof StringFieldValue) {
+ return Type.STRING;
+ } else if (fval instanceof IntegerFieldValue) {
+ return Type.INTEGER;
+ } else if (fval instanceof LongFieldValue) {
+ return Type.LONG;
+ } else if (fval instanceof FloatFieldValue) {
+ return Type.FLOAT;
+ } else if (fval instanceof DoubleFieldValue) {
+ return Type.DOUBLE;
+ } else if (fval instanceof ByteFieldValue) {
+ return Type.BYTE;
+ } else if (fval instanceof Raw) {
+ return Type.BYTE;
+ } else if (fval instanceof PredicateFieldValue) {
+ return Type.PREDICATE;
+ } else if (fval instanceof TensorFieldValue) {
+ return Type.TENSOR;
+ } else if (fieldType instanceof CollectionDataType) {
+ return convertDataType(((CollectionDataType) fieldType).getNestedType());
+ } else {
+ throw new IllegalArgumentException("Don't know which attribute type to " +
+ "convert " + fieldType + " to");
+ }
+ }
+
+ /** Converts to the right attribute type from a field datatype */
+ public static CollectionType convertCollectionType(DataType fieldType) {
+ if (fieldType instanceof ArrayDataType) {
+ return CollectionType.ARRAY;
+ } else if (fieldType instanceof WeightedSetDataType) {
+ return CollectionType.WEIGHTEDSET;
+ } else if (fieldType instanceof PrimitiveDataType) {
+ return CollectionType.SINGLE;
+ } else {
+ throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType");
+ }
+ }
+
+ /** Converts to the right field type from an attribute type */
+ public static DataType convertAttrType(Type attrType) {
+ if (attrType== Type.STRING) {
+ return DataType.STRING;
+ } else if (attrType== Type.INTEGER) {
+ return DataType.INT;
+ } else if (attrType== Type.LONG) {
+ return DataType.LONG;
+ } else if (attrType== Type.FLOAT) {
+ return DataType.FLOAT;
+ } else if (attrType== Type.DOUBLE) {
+ return DataType.DOUBLE;
+ } else if (attrType == Type.BYTE) {
+ return DataType.BYTE;
+ } else if (attrType == Type.PREDICATE) {
+ return DataType.PREDICATE;
+ } else if (attrType == Type.TENSOR) {
+ return DataType.TENSOR;
+ } else {
+ throw new IllegalArgumentException("Don't know which attribute type to " +
+ "convert " + attrType + " to");
+ }
+ }
+
+ public DataType getDataType() {
+ DataType dataType = Attribute.convertAttrType(type);
+ if (collectionType.equals(Attribute.CollectionType.ARRAY)) {
+ return DataType.getArray(dataType);
+ } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) {
+ return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero);
+ } else {
+ return dataType;
+ }
+ }
+
+ public int hashCode() {
+ return name.hashCode() +
+ type.hashCode() +
+ collectionType.hashCode() +
+ sorting.hashCode() +
+ (isPrefetch() ? 13 : 0) +
+ (fastSearch ? 17 : 0) +
+ (removeIfZero ? 47 : 0) +
+ (createIfNonExistent ? 53 : 0) +
+ (isPosition ? 61 : 0) +
+ (huge ? 67 : 0) +
+ (enableBitVectors ? 71 : 0) +
+ (enableOnlyBitVector ? 73 : 0) +
+ tensorType.hashCode();
+ }
+
+ public boolean equals(Object object) {
+ if (! (object instanceof Attribute)) return false;
+
+ Attribute other=(Attribute)object;
+ if (!this.name.equals(other.name)) return false;
+ return isCompatible(other);
+ }
+
+ /** Returns whether these attributes describes the same entity, even if they have different names */
+ public boolean isCompatible(Attribute other) {
+ if ( ! this.type.equals(other.type)) return false;
+ if ( ! this.collectionType.equals(other.collectionType)) return false;
+ if (this.isPrefetch() != other.isPrefetch()) return false;
+ if (this.removeIfZero != other.removeIfZero) return false;
+ if (this.createIfNonExistent != other.createIfNonExistent) return false;
+ if (this.enableBitVectors != other.enableBitVectors) return false;
+ if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false;
+ // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now
+ if (this.fastSearch != other.fastSearch) return false;
+ if (this.huge != other.huge) return false;
+ if ( ! this.sorting.equals(other.sorting)) return false;
+ if (!this.tensorType.equals(other.tensorType)) return false;
+
+ return true;
+ }
+
+ public @Override Attribute clone() {
+ try {
+ return (Attribute)super.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Programming error");
+ }
+ }
+
+ public String toString() {
+ return "attribute '" + name + "' (" + type + ")";
+ }
+
+ public Set<String> getAliases() {
+ return aliases;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java
new file mode 100644
index 00000000000..fc5494d0f53
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.util.OptionalDouble;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+/**
+ * Encapsulates values required for native implementation of boolean search.
+ *
+ * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a>
+ * @since 5.2
+ */
+public final class BooleanIndexDefinition
+{
+ public static final int DEFAULT_ARITY = 8;
+ public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE;
+ public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE;
+ public static final double DEFAULT_DENSE_POSTING_LIST_THRESHOLD = 0.4;
+
+ private final OptionalInt arity; // mandatory field value
+ private final OptionalLong lowerBound;
+ private final OptionalLong upperBound;
+ private final OptionalDouble densePostingListThreshold;
+
+ public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound,
+ OptionalLong upperBound, OptionalDouble densePostingListThreshold) {
+ this.arity = arity;
+ this.lowerBound = lowerBound;
+ this.upperBound = upperBound;
+ this.densePostingListThreshold = densePostingListThreshold;
+ }
+
+ public int getArity() {
+ return arity.getAsInt();
+ }
+
+ public boolean hasArity() {
+ return arity.isPresent();
+ }
+
+ public long getLowerBound() {
+ return lowerBound.orElse(DEFAULT_LOWER_BOUND);
+ }
+
+ public boolean hasLowerBound() {
+ return lowerBound.isPresent();
+ }
+
+ public long getUpperBound() {
+ return upperBound.orElse(DEFAULT_UPPER_BOUND);
+ }
+
+ public boolean hasUpperBound() {
+ return upperBound.isPresent();
+ }
+
+ public double getDensePostingListThreshold() {
+ return densePostingListThreshold.orElse(DEFAULT_DENSE_POSTING_LIST_THRESHOLD);
+ }
+
+ public boolean hasDensePostingListThreshold() {
+ return densePostingListThreshold.isPresent();
+ }
+
+ @Override
+ public String toString() {
+ return "BooleanIndexDefinition [arity=" + arity + ", lowerBound="
+ + lowerBound + ", upperBound=" + upperBound + ", densePostingListThreshold="
+ + densePostingListThreshold + "]";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
new file mode 100644
index 00000000000..040a798d3b3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Searchable collection of fields.
+ *
+ * @author balder
+ */
+public class FieldSet {
+
+ private final String name;
+ private final Set<String> queryCommands = new LinkedHashSet<>();
+ private final Set<String> fieldNames = new TreeSet<>();
+ private final Set<SDField> fields = new TreeSet<>();
+ private Matching matching = null;
+
+ public FieldSet(String name) { this.name = name; }
+ public String getName() { return name; }
+ public FieldSet addFieldName(String field) { fieldNames.add(field); return this; }
+ public Set<String> getFieldNames() { return fieldNames; }
+ public Set<SDField> fields() { return fields; }
+
+ public Set<String> queryCommands() {
+ return queryCommands;
+ }
+
+ public void setMatching(Matching matching) {
+ this.matching = matching;
+ }
+
+ public Matching getMatching() {
+ return matching;
+ }
+
+ @Override
+ public String toString() { return "fieldset '" + name + "'"; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java
new file mode 100644
index 00000000000..3c90bcef513
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+
+/**
+ * Defines how a field should be matched.
+ * Matching objects can be compared based on their content, but they are <i>not</i> immutable.
+ *
+ * @author bratseth
+ */
+public class Matching implements Cloneable, Serializable {
+
+ public static final Type defaultType = Type.TEXT;
+
+ public static enum Type {
+ TEXT("text"),
+ WORD("word"),
+ EXACT("exact"),
+ GRAM("gram");
+ private String name;
+ Type(String name) { this.name = name; }
+ public String getName() { return name; }
+ }
+
+ /** Which match algorithm is used by this matching setup */
+ public enum Algorithm {
+ NORMAL("normal"),
+ PREFIX("prefix"),
+ SUBSTRING("substring"),
+ SUFFIX("suffix");
+ private String name;
+ Algorithm(String name) { this.name = name; }
+ public String getName() { return name; }
+ }
+
+ private Type type = Type.TEXT;
+
+ /** The basic match algorithm */
+ private Algorithm algorithm = Algorithm.NORMAL;
+
+ private boolean typeUserSet = false;
+
+ private boolean algorithmUserSet = false;
+
+ /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */
+ private int gramSize=-1;
+
+ /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */
+ private Integer maxLength;
+
+ private String exactMatchTerminator=null;
+
+ /** Creates a matching of type "text" */
+ public Matching() {}
+
+ public Matching(Type type) {
+ this.type = type;
+ }
+
+ public Type getType() { return type; }
+
+ public void setType(Type type) {
+ this.type = type;
+ typeUserSet = true;
+ }
+
+ public Integer maxLength() { return maxLength; }
+ public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; }
+ public boolean isTypeUserSet() { return typeUserSet; }
+
+ public Algorithm getAlgorithm() { return algorithm; }
+
+ public void setAlgorithm(Algorithm algorithm) {
+ this.algorithm = algorithm;
+ algorithmUserSet = true;
+ }
+
+ public boolean isAlgorithmUserSet() { return algorithmUserSet; }
+
+ public boolean isPrefix() { return algorithm == Algorithm.PREFIX; }
+
+ public boolean isSubstring() { return algorithm == Algorithm.SUBSTRING; }
+
+ public boolean isSuffix() { return algorithm == Algorithm.SUFFIX; }
+
+ /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */
+ public int getGramSize() { return gramSize; }
+
+ public void setGramSize(int gramSize) { this.gramSize=gramSize; }
+
+ /**
+ * Merge data from another matching object
+ */
+ public void merge(Matching m) {
+ if (m.isAlgorithmUserSet()) {
+ this.setAlgorithm(m.getAlgorithm());
+ }
+ if (m.isTypeUserSet()) {
+ this.setType(m.getType());
+ if (m.getType()==Type.GRAM)
+ gramSize=m.gramSize;
+ }
+ if (m.getExactMatchTerminator() != null) {
+ this.setExactMatchTerminator(m.getExactMatchTerminator());
+ }
+ }
+
+ /**
+ * If exact matching is used, this returns the terminator string
+ * which terminates an exact matched sequence in queries. If exact
+ * matching is not used, or no terminator is set, this is null
+ */
+ public String getExactMatchTerminator() { return exactMatchTerminator; }
+
+ /**
+ * Sets the terminator string which terminates an exact matched
+ * sequence in queries (used if type is EXACT).
+ */
+ public void setExactMatchTerminator(String exactMatchTerminator) {
+ this.exactMatchTerminator = exactMatchTerminator;
+ }
+
+ public String toString() {
+ return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + "], [exact-terminator "+exactMatchTerminator+"]";
+ }
+
+ public Matching clone() {
+ try {
+ return (Matching)super.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Programming error");
+ }
+ }
+
+ public boolean equals(Object o) {
+ if (! (o instanceof Matching)) return false;
+
+ Matching other=(Matching)o;
+ if ( ! other.type.equals(this.type)) return false;
+ if ( ! other.algorithm.equals(this.algorithm)) return false;
+ if ( this.exactMatchTerminator==null && other.exactMatchTerminator!=null) return false;
+ if ( this.exactMatchTerminator!=null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) )
+ return false;
+ if ( gramSize!=other.gramSize) return false;
+ return true;
+ }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java
new file mode 100644
index 00000000000..945593d550b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+/**
+ * class representing the character normalization
+ * we want to do on query and indexed text.
+ * Levels are strict subsets, so doing accent
+ * removal means doing codepoint normalizing
+ * and case normalizing also.
+ */
+// TODO: Missing author
+public class NormalizeLevel {
+
+ /**
+ * The current levels are as follows:
+ * NONE: no changes to input text
+ * CODEPOINT: convert text into Unicode
+ * Normalization Form Compatibility Composition
+ * LOWERCASE: also convert text into lowercase letters
+ * ACCENT: do both above and remove accents on characters
+ */
+ public enum Level {
+ NONE, CODEPOINT, LOWERCASE, ACCENT
+ }
+
+ private boolean userSpecified = false;
+ private Level level = Level.ACCENT;
+
+ /**
+ * Returns whether accents should be removed from text
+ */
+ public boolean doRemoveAccents() { return level == Level.ACCENT; }
+
+ /**
+ * Construct a default (full) normalizelevel,
+ */
+ public NormalizeLevel() {}
+
+ /**
+ * Construct for a specific level, possibly user specified
+ *
+ * @param level which level to use
+ * @param fromUser whether this was specified by the user
+ */
+ public NormalizeLevel(Level level, boolean fromUser) {
+ this.level = level;
+ this.userSpecified = fromUser;
+ }
+
+ /**
+ * Change the current level to CODEPOINT as inferred
+ * by other features' needs. If the current level
+ * was user specified it will not change; also this
+ * will not increase the level.
+ */
+ public void inferCodepoint() {
+ if (userSpecified) {
+ // ignore inferred changes if user specified something
+ return;
+ }
+ // do not increase level
+ if (level != Level.NONE) level = Level.CODEPOINT;
+ }
+
+ /**
+ * Change the current level to LOWERCASE as inferred
+ * by other features' needs. If the current level
+ * was user specified it will not change; also this
+ * will not increase the level.
+ */
+ public void inferLowercase() {
+ if (userSpecified) {
+ // ignore inferred changes if user specified something
+ return;
+ }
+ // do not increase level
+ if (level == Level.NONE) return;
+ if (level == Level.CODEPOINT) return;
+
+ level = Level.LOWERCASE;
+ }
+
+ public Level getLevel() {
+ return level;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java
new file mode 100644
index 00000000000..f2b0b6a9164
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * The rank type of a field. For now this is just a container of a string name.
+ * This class is immutable.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public enum RankType {
+
+ /** *implicit* default: No type has been set. */
+ DEFAULT,
+
+ // Rank types which can be set explicitly. These are defined for Vespa in NativeRankTypeDefinitionSet
+ IDENTITY, ABOUT, TAGS, EMPTY;
+
+ @Override
+ public String toString() {
+ return "rank type " + name().toLowerCase();
+ }
+
+ /**
+ * Returns the rank type from a string, regardless of its case.
+ *
+ * @param rankTypeName a rank type name in any casing
+ * @return the rank type found
+ * @throws IllegalArgumentException if not found
+ */
+ public static RankType fromString(String rankTypeName) {
+ try {
+ return RankType.valueOf(rankTypeName.toUpperCase());
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Unknown rank type '" + rankTypeName + "'. Supported rank types are " +
+ "'identity', 'about', 'tags' and 'empty'.");
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java
new file mode 100644
index 00000000000..9e0bda7fd5a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+
+/**
+ * The rank settings given in a rank clause in the search definition.
+ *
+ * @author <a href="mailto:vehardh@yahoo-inc.com">Vegard Havdal</a>
+ */
+public class Ranking implements Cloneable, Serializable {
+
+ private boolean literal = false;
+ private boolean filter = false;
+ private boolean normal = false;
+
+ /**
+ * <p>Returns whether literal (non-stemmed, non-normalized) forms of the words should
+ * be indexed in a separate index which is searched by a automatically added rank term
+ * during searches.</p>
+ *
+ * <p>Default is false.</p>
+ */
+ public boolean isLiteral() { return literal; }
+
+ public void setLiteral(boolean literal) { this.literal = literal; }
+
+ /**
+ * <p>Returns whether this is a filter. Filters will only tell if they are matched or not,
+ * no detailed relevance information will be available about the match.</p>
+ *
+ * <p>Matching a filter is much cheaper for the search engine than matching a regular field.</p>
+ *
+ * <p>Default is false.</p>
+ */
+ public boolean isFilter() { return filter && !normal; }
+
+ public void setFilter(boolean filter) { this.filter = filter; }
+
+ /** Whether user has explicitly requested normal (non-filter) behavior */
+ public boolean isNormal() { return normal; }
+ public void setNormal(boolean n) { this.normal = n; }
+
+ /** Returns true if the given rank settings are the same */
+ public @Override boolean equals(Object o) {
+ if ( ! (o instanceof Ranking)) return false;
+
+ Ranking other=(Ranking)o;
+ if (this.filter != other.filter) return false;
+ if (this.literal != other.literal) return false;
+ if (this.normal != other.normal) return false;
+ return true;
+ }
+
+ public @Override String toString() {
+ return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]";
+ }
+
+ public @Override Ranking clone() {
+ try {
+ return (Ranking)super.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Programming error",e);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
new file mode 100644
index 00000000000..6292dc9ef72
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -0,0 +1,316 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.AnnotationTypeRegistry;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.FieldSets;
+import com.yahoo.searchdefinition.Search;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ <p>A document definition is a list of fields. Documents may inherit other documents,
+ implicitly acquiring their fields as it's own. If a document is not set to inherit
+ any document, it will always inherit the document "document.0".</p>
+
+ @author <a href="thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a>
+*/
+public class SDDocumentType implements Cloneable, Serializable {
+ public static final SDDocumentType VESPA_DOCUMENT;
+ private Map<DataTypeName, SDDocumentType> inheritedTypes = new HashMap<>();
+ private Map<NewDocumentType.Name, SDDocumentType> ownedTypes = new HashMap<>();
+ private AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry();
+ private DocumentType docType;
+ private DataType structType;
+ // The field sets here are set from the processing step in SD, to ensure that the full Search and this SDDocumentType is built first.
+ private FieldSets fieldSets;
+
+ static {
+ VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
+ VESPA_DOCUMENT.addType(createSDDocumentType(PositionDataType.INSTANCE));
+ }
+
+ public SDDocumentType clone() throws CloneNotSupportedException {
+ SDDocumentType type = (SDDocumentType) super.clone();
+ type.docType = docType.clone();
+ type.inheritedTypes.putAll(inheritedTypes);
+ type.structType = structType;
+ return type;
+ }
+
+ /**
+ * For adding structs defined in document scope
+ *
+ * @param dt The struct to add.
+ * @return self, for chaining
+ */
+ public SDDocumentType addType(SDDocumentType dt) {
+ NewDocumentType.Name name = new NewDocumentType.Name(dt.getName());
+ if (getType(name) != null) {
+ throw new IllegalArgumentException("Data type '" + name.toString() + "' has already been used.");
+ }
+ if (name.getName() == docType.getName()) {
+ throw new IllegalArgumentException("Data type '" + name.toString() + "' can not have same name as its defining document.");
+ }
+ ownedTypes.put(name, dt);
+ return this;
+ }
+ public final SDDocumentType getOwnedType(String name) {
+ return getOwnedType(new NewDocumentType.Name(name));
+ }
+ public SDDocumentType getOwnedType(DataTypeName name) {
+ return getOwnedType(name.getName());
+ }
+
+ public SDDocumentType getOwnedType(NewDocumentType.Name name) {
+ return ownedTypes.get(name);
+ }
+
+ public final SDDocumentType getType(String name) {
+ return getType(new NewDocumentType.Name(name));
+ }
+
+ public SDDocumentType getType(NewDocumentType.Name name) {
+ SDDocumentType type = ownedTypes.get(name);
+ if (type == null) {
+ for (SDDocumentType inherited : inheritedTypes.values()) {
+ type = inherited.getType(name);
+ if (type != null) {
+ return type;
+ }
+ }
+ }
+ return type;
+ }
+
+ public SDDocumentType addAnnotation(AnnotationType annotation) {
+ annotationTypes.register(annotation);
+ return this;
+ }
+
+ /**
+ * Access to all owned datatypes.
+ * @return all types
+ */
+ public Collection<SDDocumentType> getTypes() { return ownedTypes.values(); }
+ public Collection<AnnotationType> getAnnotations() { return annotationTypes.getTypes().values(); }
+ public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); }
+
+ public Collection<SDDocumentType> getAllTypes() {
+ Collection<SDDocumentType> list = new ArrayList<>();
+ list.addAll(getTypes());
+ for (SDDocumentType inherited : inheritedTypes.values()) {
+ list.addAll(inherited.getAllTypes());
+ }
+ return list;
+ }
+
+ /**
+ * Creates a new document type.
+ * The document type id will be generated as a hash from the document type name.
+ *
+ * @param name The name of the new document type
+ */
+ public SDDocumentType(String name) {
+ this(name,null);
+ }
+
+ public SDDocumentType(DataTypeName name) {
+ this(name.getName());
+ }
+
+ /**
+ * Creates a new document type.
+ * The document type id will be generated as a hash from the document type name.
+ *
+ * @param name The name of the new document type
+ * @param search check for type ID collisions in this search definition
+ */
+ public SDDocumentType(String name, Search search) {
+ docType = new DocumentType(name);
+ docType.getHeaderType().setCompressionConfig(new CompressionConfig());
+ docType.getBodyType().setCompressionConfig(new CompressionConfig());
+ validateId(search);
+ inherit(VESPA_DOCUMENT);
+ }
+
+ public boolean isStruct() { return getStruct() != null; }
+ public DataType getStruct() { return structType; }
+ public SDDocumentType setStruct(DataType structType) {
+ if (structType != null) {
+ this.structType = structType;
+ inheritedTypes.clear();
+ } else {
+ if (docType.getHeaderType() != null) {
+ this.structType = docType.getHeaderType();
+ inheritedTypes.clear();
+ } else {
+ throw new IllegalArgumentException("You can not set a null struct");
+ }
+ }
+ return this;
+ }
+
+ public String getName() { return docType.getName(); }
+ public DataTypeName getDocumentName() { return docType.getDataTypeName(); }
+ public DocumentType getDocumentType() { return docType; }
+
+ public void inherit(DataTypeName name) {
+ if ( ! inheritedTypes.containsKey(name)) {
+ inheritedTypes.put(name, new TemporarySDDocumentType(name));
+ }
+ }
+
+ public void inherit(SDDocumentType type) {
+ if (type != null) {
+ if (!inheritedTypes.containsKey(type.getDocumentName()) || (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) {
+ inheritedTypes.put(type.getDocumentName(), type);
+ }
+ }
+ }
+
+ public Collection<SDDocumentType> getInheritedTypes() { return inheritedTypes.values(); }
+
+ protected void validateId(Search search) {
+ if (search == null) return;
+ if (search.getDocument(getName()) == null) return;
+ SDDocumentType doc = search.getDocument();
+ throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " +
+ "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'");
+ }
+
+ public void setFieldId(SDField field, int id) {
+ field.setId(id, docType);
+ }
+
+ /** Override getField, as it may need to ask inherited types that isn't registered in document type.
+ * @param name Name of field to get.
+ * @return The field found.
+ */
+ public Field getField(String name) {
+ if (name.contains(".")) {
+ String superFieldName = name.substring(0,name.indexOf("."));
+ String subFieldName = name.substring(name.indexOf(".")+1);
+ Field f = docType.getField(superFieldName);
+ if (f != null) {
+ if (f instanceof SDField) {
+ SDField superField = (SDField)f;
+ return superField.getStructField(subFieldName);
+ } else {
+ throw new IllegalArgumentException("Field "+f.getName()+" is not SDField");
+ }
+ }
+ }
+ Field f = docType.getField(name);
+ if (f == null) {
+ for(SDDocumentType parent : inheritedTypes.values()) {
+ f = parent.getField(name);
+ if (f != null) return f;
+ }
+ }
+ return f;
+ }
+
+ public void addField(Field field) {
+ verifyInheritance(field);
+ for (Iterator<Field> i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) {
+ if (field.getName().equalsIgnoreCase((i.next()).getName())) {
+ throw new IllegalArgumentException("Duplicate (case insensitively) " + field + " in " + this);
+ }
+ }
+ docType.addField(field);
+ }
+
+ /**
+ * This is SD parse-time inheritance check.
+ *
+ * @param field The field being verified.
+ */
+ private void verifyInheritance(Field field) {
+ for (SDDocumentType parent : inheritedTypes.values()) {
+ for (Field pField : parent.fieldSet()) {
+ if (pField.getName().equals(field.getName())) {
+ if (!pField.getDataType().equals(field.getDataType())) {
+ throw new IllegalArgumentException("For search '"+getName()+"', field '"+field.getName()+"': " +
+ "datatype can't be different from that of same field in supertype '"+parent.getName()+"'.");
+ }
+ }
+ }
+ }
+ }
+
+ public SDField addField(String string, DataType dataType) {
+ SDField field = new SDField(this, string, dataType);
+ addField(field);
+ return field;
+ }
+
+ public Field addField(String string, DataType dataType, boolean header, int code) {
+ SDField field = new SDField(this, string, code, dataType, header);
+ addField(field);
+ return field;
+ }
+
+ private Map<String, Field> fieldsInherited() {
+ Map<String, Field> map = new LinkedHashMap<>();
+ for (SDDocumentType parent : inheritedTypes.values()) {
+ for (Field field : parent.fieldSet()) {
+ map.put(field.getName(), field);
+ }
+ }
+ return map;
+ }
+
+ public Set<Field> fieldSet() {
+ Map<String, Field> map = fieldsInherited();
+ Iterator<Field> it = docType.fieldIteratorThisTypeOnly();
+ while (it.hasNext()) {
+ Field field = it.next();
+ map.put(field.getName(), field);
+ }
+ return new LinkedHashSet<>(map.values());
+ }
+
+ public Iterator<Field> fieldIterator() {
+ return fieldSet().iterator();
+ }
+
+ public int getFieldCount() {
+ return docType.getFieldCount();
+ }
+
+ public String toString() {
+ return "SD document type '" + docType.getName() + "'";
+ }
+
+ private static SDDocumentType createSDDocumentType(StructDataType structType) {
+ SDDocumentType docType = new SDDocumentType(structType.getName());
+ for (Field field : structType.getFields()) {
+ docType.addField(new SDField(docType, field.getName(), field.getDataType()));
+ }
+ docType.setStruct(structType);
+ return docType;
+ }
+
+ /**
+ * The field sets defined for this type and its {@link Search}
+ * @return fieldsets
+ */
+ public FieldSets getFieldSets() {
+ return fieldSets;
+ }
+
+ /**
+ * Sets the field sets for this
+ * @param fieldSets field sets to set for this object
+ */
+ public void setFieldSets(FieldSets fieldSets) {
+ this.fieldSets = fieldSets;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
new file mode 100644
index 00000000000..b379abd63ec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -0,0 +1,770 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.*;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.fieldoperation.FieldOperation;
+import com.yahoo.searchdefinition.fieldoperation.FieldOperationContainer;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.indexinglanguage.ExpressionSearcher;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+
+import java.io.Serializable;
+import java.util.*;
+
+/**
+ * The field class represents a document field. It is used in
+ * the Document class to get and set fields. Each SDField has
+ * a name, a numeric ID, a data type, and a boolean that says whether it's
+ * a header field. The numeric ID is used when the fields are stored
+ * in serialized form.
+ *
+ * @author bratseth
+ */
+public class SDField extends Field implements TypedKey, FieldOperationContainer, Serializable {
+
+ /** Use this field for modifying index-structure, even if it doesn't
+ have any indexing code */
+ private boolean indexStructureField = false;
+
+ /** The indexing statements to be applied to this value during indexing */
+ private ScriptExpression indexingScript = new ScriptExpression();
+
+ /** The default rank type for indices of this field */
+ private RankType rankType = RankType.DEFAULT;
+
+ /** Rank settings in a "rank" block for the field. */
+ private Ranking ranking = new Ranking();
+
+ /**
+ * The literal boost of this field. This boost is added to a rank score
+ * when a query term matched as query term exactly (unnormalized and unstemmed).
+ * Non-positive boosts causes no boosting, 0 allows boosts
+ * to be specified in other rank profiles, while negative values
+ * turns the capability off.
+ */
+ private int literalBoost=-1;
+
+ /** The weight of this field. This is a percentage,
+ * so 100 is default to provide the identity transform. */
+ private int weight=100;
+
+ /**
+ * Indicates what kind of matching should be done on this field
+ */
+ private Matching matching=new Matching();
+
+ /** Attribute settings, or null if there are none */
+ private Map<String, Attribute> attributes = new TreeMap<>();
+
+ /**
+ * The stemming setting of this field, or null to use the default.
+ * Default is determined by the owning search definition.
+ */
+ private Stemming stemming=null;
+
+ /** How content of this field should be accent normalized etc. */
+ private NormalizeLevel normalizing = new NormalizeLevel();
+
+ /** Extra query commands of this field */
+ private List<String> queryCommands=new java.util.ArrayList<>(0);
+
+ /** Summary fields defined in this field */
+ private Map<String,SummaryField> summaryFields = new java.util.LinkedHashMap<>(0);
+
+ /** The explicitly index settings on this field */
+ private Map<String, Index> indices=new java.util.LinkedHashMap<>();
+
+ /** True if body or header is set explicitly for this field */
+ private boolean headerOrBodyDefined = false;
+
+ private boolean idOverride = false;
+
+ /** Struct fields defined in this field */
+ private Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0);
+
+ /** The document that this field was declared in, or null*/
+ protected SDDocumentType ownerDocType = null;
+
+ /** The aliases declared for this field. May pertain to indexes or attributes */
+ private Map<String, String> aliasToName = new HashMap<>();
+
+ /** Pending operations that must be applied after parsing, due to use of not-yet-defined structs. */
+ private List<FieldOperation> pendingOperations = new LinkedList<>();
+
+ private boolean isExtraField = false;
+
+ /**
+ Creates a new field. This method is only used to create reserved fields
+ @param name The name of the field
+ @param dataType The datatype of the field
+ @param isHeader Whether this is a "header" field or a "content" field
+ (true = "header").
+ */
+ protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) {
+ super(name, id, dataType, isHeader);
+ populate(populate, repo, name, dataType, isHeader);
+ }
+
+ public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) {
+ this(repo, name, id, dataType, isHeader, true);
+ }
+
+ /**
+ Creates a new field.
+
+ @param name The name of the field
+ @param dataType The datatype of the field
+ @param isHeader Whether this is a "header" field or a "content" field
+ (true = "header").
+ */
+ public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) {
+ super(name,dataType,isHeader);
+ populate(populate, repo, name, dataType, isHeader);
+ }
+
+ private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) {
+ populate(populate,repo, name, dataType, isHeader, null, 0);
+ }
+
+ private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching, int recursion) {
+ if (populate || (dataType instanceof MapDataType)) {
+ populateWithStructFields(repo, name, dataType, isHeader, recursion);
+ populateWithStructMatching(repo, name, dataType, fieldMatching);
+ }
+ }
+
+ public SDField(String name, DataType dataType, boolean isHeader) {
+ this(null, name, dataType, isHeader, true);
+ }
+
+ /**
+ Creates a new field.
+
+ @param name The name of the field
+ @param dataType The datatype of the field
+ @param isHeader Whether this is a "header" field or a "content" field
+ (true = "header").
+ @param owner the owning document (used to check for id collisions)
+ */
+ protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) {
+ super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+ this.ownerDocType=owner;
+ populate(populate, repo, name, dataType, isHeader);
+ }
+
+ /**
+ Creates a new field.
+
+ @param name The name of the field
+ @param dataType The datatype of the field
+ @param isHeader Whether this is a "header" field or a "content" field
+ (true = "header").
+ @param owner The owning document (used to check for id collisions)
+ @param fieldMatching The matching object to set for the field
+ */
+ protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) {
+ super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+ this.ownerDocType=owner;
+ if (fieldMatching != null) {
+ this.setMatching(fieldMatching);
+ }
+ populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion);
+ }
+
+ /**
+ Constructor for <b>header</b> fields
+
+ @param name The name of the field
+ @param dataType The datatype of the field
+ */
+ public SDField(SDDocumentType repo, String name, DataType dataType) {
+ this(repo, name,dataType,true, true);
+ }
+ public SDField(String name, DataType dataType) {
+ this(null, name,dataType);
+ }
+
+ public void setIsExtraField(boolean isExtra) {
+ isExtraField = isExtra;
+ }
+
+ public boolean isExtraField() {
+ return isExtraField;
+ }
+
+ public boolean doesAttributing() {
+ return containsExpression(AttributeExpression.class);
+ }
+
+ public boolean doesIndexing() {
+ return containsExpression(IndexExpression.class);
+ }
+
+ public boolean doesSummarying() {
+ if (usesStruct()) {
+ for (SDField structField : getStructFields()) {
+ if (structField.doesSummarying()) {
+ return true;
+ }
+ }
+ }
+ return containsExpression(SummaryExpression.class);
+ }
+
+ public boolean doesLowerCasing() {
+ return containsExpression(LowerCaseExpression.class);
+ }
+
+ public <T extends Expression> boolean containsExpression(Class<T> searchFor) {
+ return findExpression(searchFor) != null;
+ }
+
+ public <T extends Expression> T findExpression(Class<T> searchFor) {
+ return new ExpressionSearcher<>(searchFor).searchIn(indexingScript);
+ }
+
+ public void addSummaryFieldSources(SummaryField summaryField) {
+ if (usesStruct()) {
+ /*
+ * How this works for structs: When at least one sub-field in a struct is to
+ * be used for summary, that whole struct field is included in summary.cfg. Then,
+ * vsmsummary.cfg specifies the sub-fields used for each struct field.
+ * So we recurse into each struct, adding the destination classes set for each sub-field
+ * to the main summary-field for the struct field.
+ */
+ for (SDField structField : getStructFields()) {
+ for (SummaryField sumF : structField.getSummaryFields()) {
+ for (String dest : sumF.getDestinations()) {
+ summaryField.addDestination(dest);
+ }
+ }
+ structField.addSummaryFieldSources(summaryField);
+ }
+ } else {
+ if (doesSummarying()) {
+ summaryField.addSource(getName());
+ }
+ }
+ }
+
+ public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) {
+ DataType dt = getFirstStructOrMapRecursive();
+ if (dt == null) {
+ return;
+ }
+ if (dataType instanceof MapDataType) {
+ MapDataType mdt = (MapDataType) dataType;
+
+ SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(),
+ isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+ structFields.put("key", keyField);
+
+ SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(),
+ isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+ structFields.put("value", valueField);
+ } else {
+ if (recursion >= 10) {
+ return;
+ }
+ if (dataType instanceof CollectionDataType) {
+ dataType = ((CollectionDataType)dataType).getNestedType();
+ }
+ SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
+ if (subType == null) {
+ throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'.");
+ }
+ for (Field field : subType.fieldSet()) {
+ SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(),
+ isHeader, subType, new Matching(), true, recursion + 1);
+ structFields.put(field.getName(), subField);
+ }
+ }
+ }
+
+ public void populateWithStructMatching(SDDocumentType sdoc, String name, DataType dataType,
+ Matching superFieldMatching) {
+ DataType dt = getFirstStructOrMapRecursive();
+ if (dt != null) {
+ if (dataType instanceof MapDataType) {
+ MapDataType mdt = (MapDataType) dataType;
+
+ Matching keyFieldMatching = new Matching();
+ if (superFieldMatching != null) {
+ keyFieldMatching.merge(superFieldMatching);
+ }
+ SDField keyField = structFields.get(name.concat(".key"));
+ if (keyField != null) {
+ keyField.populateWithStructMatching(sdoc, name.concat(".key"), mdt.getKeyType(), keyFieldMatching);
+ keyField.setMatching(keyFieldMatching);
+ }
+
+ Matching valueFieldMatching = new Matching();
+ if (superFieldMatching != null) {
+ valueFieldMatching.merge(superFieldMatching);
+ }
+ SDField valueField = structFields.get(name.concat(".value"));
+ if (valueField != null) {
+ valueField.populateWithStructMatching(sdoc, name.concat(".value"), mdt.getValueType(),
+ valueFieldMatching);
+ valueField.setMatching(valueFieldMatching);
+ }
+
+ } else {
+
+ if (dataType instanceof CollectionDataType) {
+ dataType = ((CollectionDataType)dataType).getNestedType();
+ }
+ SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
+ if (subType != null) {
+ for (Field f : subType.fieldSet()) {
+ if (f instanceof SDField) {
+ SDField field = (SDField)f;
+ Matching subFieldMatching = new Matching();
+ if (superFieldMatching != null) {
+ subFieldMatching.merge(superFieldMatching);
+ }
+ subFieldMatching.merge(field.getMatching());
+ SDField subField = structFields.get(field.getName());
+ if (subField != null) {
+ subField.populateWithStructMatching(sdoc, name.concat(".").concat(field.getName()), field.getDataType(),
+ subFieldMatching);
+ subField.setMatching(subFieldMatching);
+ }
+ } else {
+ throw new IllegalArgumentException("Field in struct is not SDField " + f.getName());
+ }
+ }
+ } else {
+ throw new IllegalArgumentException("Could not find struct " + dataType.getName());
+ }
+ }
+ }
+ }
+
+ public void addOperation(FieldOperation op) {
+ pendingOperations.add(op);
+ }
+
+ public void applyOperations(SDField field) {
+ if (pendingOperations.isEmpty()) {
+ return;
+ }
+ ListIterator<FieldOperation> ops = pendingOperations.listIterator();
+ while (ops.hasNext()) {
+ FieldOperation op = ops.next();
+ ops.remove();
+ op.apply(field);
+ }
+ }
+
+ public void applyOperations() {
+ applyOperations(this);
+ }
+
+ public void setId(int fieldId, DocumentType owner) {
+ super.setId(fieldId, owner);
+ idOverride = true;
+ }
+
+ public StructDataType getFirstStructRecursive() {
+ DataType dataType = getDataType();
+ while (true) { // Currently no nesting of collections
+ if (dataType instanceof CollectionDataType) {
+ dataType = ((CollectionDataType)dataType).getNestedType();
+ } else if (dataType instanceof MapDataType) {
+ dataType = ((MapDataType)dataType).getValueType();
+ } else {
+ break;
+ }
+ }
+ return (dataType instanceof StructDataType) ? (StructDataType)dataType : null;
+ }
+
+ public DataType getFirstStructOrMapRecursive() {
+ DataType dataType = getDataType();
+ while (dataType instanceof CollectionDataType) { // Currently no nesting of collections
+ dataType = ((CollectionDataType)dataType).getNestedType();
+ }
+ return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null;
+ }
+
+ public boolean usesStruct() {
+ DataType dt = getFirstStructRecursive();
+ return (dt != null);
+ }
+
+ public boolean usesStructOrMap() {
+ DataType dt = getFirstStructOrMapRecursive();
+ return (dt != null);
+ }
+
+ /** Parse an indexing expression which will use the simple linguistics implementatino suitable for testing */
+ public void parseIndexingScript(String script) {
+ parseIndexingScript(script, new SimpleLinguistics());
+ }
+
+ public void parseIndexingScript(String script, Linguistics linguistics) {
+ try {
+ ScriptParserContext config = new ScriptParserContext(linguistics);
+ config.setInputStream(new IndexingInput(script));
+ setIndexingScript(ScriptExpression.newInstance(config));
+ } catch (ParseException e) {
+ throw new RuntimeException("Failed to parser script '" + script + "'.", e);
+ }
+ }
+
+ /** Sets the indexing script of this, or null to not use a script */
+ public void setIndexingScript(ScriptExpression exp) {
+ if (exp == null) {
+ exp = new ScriptExpression();
+ }
+ indexingScript = exp;
+ if (indexingScript.isEmpty()) {
+ return; // TODO: This causes empty expressions not to be propagate to struct fields!! BAD BAD BAD!!
+ }
+ if (!usesStructOrMap()) {
+ new ExpressionVisitor() {
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (!(exp instanceof AttributeExpression)) {
+ return;
+ }
+ String fieldName = ((AttributeExpression)exp).getFieldName();
+ if (fieldName == null) {
+ fieldName = getName();
+ }
+ Attribute attribute = attributes.get(fieldName);
+ if (attribute == null) {
+ addAttribute(new Attribute(fieldName, getDataType()));
+ }
+ }
+ }.visit(indexingScript);
+ }
+ for (SDField structField : getStructFields()) {
+ structField.setIndexingScript(exp);
+ }
+ }
+
+ public ScriptExpression getIndexingScript() {
+ return indexingScript;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void setDataType(DataType type) {
+ if (type.equals(DataType.URI)) { // Different defaults, naturally
+ normalizing.inferLowercase();
+ stemming=Stemming.NONE;
+ }
+ this.dataType = type;
+ if (!idOverride) {
+ this.fieldId = calculateIdV7(null);
+ }
+ }
+
+ public boolean isIndexStructureField() {
+ return indexStructureField;
+ }
+
+ public void setIndexStructureField(boolean indexStructureField) {
+ this.indexStructureField = indexStructureField;
+ }
+
+ /**
+ * Returns an iterator of the index names this should index to
+ * (whether set explicitly or not)
+ */
+ public Iterator<String> getFieldNameAsIterator() { // TODO: Replace usage by getName
+ return Collections.singletonList(getName()).iterator();
+ }
+
+ /** Returns 1 if this is indexed, 0 if it is not indexed */ // TODO: Replace by a boolean method, or something, see hasIndex
+ public int getIndexToCount() {
+ if (getIndexingScript() == null) return 0;
+ if (!doesIndexing()) return 0;
+
+ return 1;
+ }
+
+ /** Sets the literal boost of this field */
+ public void setLiteralBoost(int literalBoost) { this.literalBoost=literalBoost; }
+
+ /**
+ * Returns the literal boost of this field. This boost is added to a literal score
+ * when a query term matched as query term exactly (unnormalized and unstemmed).
+ * Default is non-positive.
+ */
+ public int getLiteralBoost() { return literalBoost; }
+
+ /** Sets the weight of this field */
+ public void setWeight(int weight) { this.weight=weight; }
+
+ /** Returns the weight of this field, or 0 if nothing is set */
+ public int getWeight() { return weight; }
+
+ /**
+ * Returns what kind of matching type should be applied.
+ */
+ public Matching getMatching() { return matching; }
+
+ /**
+ * Sets what kind of matching type should be applied.
+ * (Token matching is default, PREFIX, SUBSTRING, SUFFIX are alternatives)
+ */
+ public void setMatching(Matching matching) { this.matching=matching; }
+
+ /**
+ * Set the matching type for this field and all subfields.
+ */
+ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right
+ // Matching object for struct fields as lookup time instead.
+ public void setMatchingType(Matching.Type type) {
+ this.getMatching().setType(type);
+ for (SDField structField : getStructFields()) {
+ structField.setMatchingType(type);
+ }
+ }
+
+ /**
+ * Set matching algorithm for this field and all subfields.
+ */
+ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right
+ // Matching object for struct fields as lookup time instead.
+ public void setMatchingAlgorithm(Matching.Algorithm algorithm) {
+ this.getMatching().setAlgorithm(algorithm);
+ for (SDField structField : getStructFields()) {
+ structField.getMatching().setAlgorithm(algorithm);
+ }
+ }
+
+ /** Adds an explicit index defined in this field */
+ public void addIndex(Index index) {
+ indices.put(index.getName(),index);
+ }
+
+ /**
+ * Returns an index, or null if no index with this name has had
+ * some <b>explicit settings</b> applied in this field (even if this returns null,
+ * the index may be implicitly defined by an indexing statement)
+ */
+ public Index getIndex(String name) {
+ return indices.get(name);
+ }
+
+ /**
+ * Returns an index if this field has one (implicitly or
+ * explicitly) targeting the given name.
+ */
+ public boolean existsIndex(String name) {
+ if (indices.get(name) != null) return true;
+ if (name.equals(getName())) {
+ if (doesIndexing()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Defined indices on this field
+ * @return defined indices on this
+ */
+ public Map<String, Index> getIndices() {
+ return indices;
+ }
+
+ /**
+ * Sets the default rank type of this fields indices, and sets this rank type
+ * to all indices explicitly defined here which has no index set.
+ * (This complex behavior is dues to the fact than we would prefer to have rank types
+ * per field, not per index)
+ */
+ public void setRankType(RankType rankType) {
+ this.rankType=rankType;
+ for (Index index : getIndices().values()) {
+ if (index.getRankType()==null)
+ index.setRankType(rankType);
+ }
+
+ }
+
+ /** Returns the rank settings set in a "rank" block for this field. This is never null. */
+ public Ranking getRanking() { return ranking; }
+
+ /** Returns the default rank type of indices of this field, or null if nothing is set */
+ public RankType getRankType() { return this.rankType; }
+
+ /**
+ * <p>Returns the search-time attribute settings of this field
+ * or null if none is set.</p>
+ *
+ * <p>TODO: Make unmodifiable.</p>
+ */
+ public Map<String, Attribute> getAttributes() { return attributes; }
+
+ public void addAttribute(Attribute attribute) {
+ String name = attribute.getName();
+ if (name == null || "".equals(name)) {
+ name = getName();
+ attribute.setName(name);
+ }
+ attributes.put(attribute.getName(),attribute);
+ }
+
+ /**
+ * Returns the stemming setting of this field.
+ * Default is determined by the owning search definition.
+ *
+ * @return the stemming setting of this, or null, to use the default
+ */
+ public Stemming getStemming() { return stemming; }
+
+ /**
+ * Whether this field should be stemmed in this search definition
+ */
+ public Stemming getStemming(Search search) {
+ if (stemming!=null)
+ return stemming;
+ else
+ return search.getStemming();
+ }
+
+ /**
+ * Sets how this field should be stemmed, or set to null to use the default.
+ */
+ public void setStemming(Stemming stemming) {
+ this.stemming=stemming;
+ }
+
+ /**
+ * List of static summary fields
+ * @return list of static summary fields
+ */
+ public Collection<SummaryField> getSummaryFields() { return summaryFields.values(); }
+
+ /**
+ * Add summary field
+ */
+ public void addSummaryField(SummaryField summaryField) {
+ summaryFields.put(summaryField.getName(),summaryField);
+ }
+
+ /**
+ * Returns a summary field defined (implicitly or explicitly) by this field.
+ * Returns null if there is no such summary field defined.
+ */
+ public SummaryField getSummaryField(String name) {
+ return summaryFields.get(name);
+ }
+
+ /**
+ * Returns a summary field defined (implicitly or explicitly) by this field.
+ *
+ * @param create true to create the summary field and add it to this field before returning if it is missing
+ * @return the summary field, or null if not present and create is false
+ */
+ public SummaryField getSummaryField(String name,boolean create) {
+ SummaryField summaryField=summaryFields.get(name);
+ if (summaryField==null && create) {
+ summaryField=new SummaryField(name, getDataType());
+ addSummaryField(summaryField);
+ }
+ return summaryFields.get(name);
+ }
+
+ /** Returns list of static struct fields */
+ public Collection<SDField> getStructFields() { return structFields.values(); }
+
+ /**
+ * Returns a struct field defined in this field,
+ * potentially traversing into nested structs.
+ * Returns null if there is no such struct field defined.
+ */
+ public SDField getStructField(String name) {
+ if (name.contains(".")) {
+ String superFieldName = name.substring(0,name.indexOf("."));
+ String subFieldName = name.substring(name.indexOf(".")+1);
+ SDField superField = structFields.get(superFieldName);
+ if (superField != null) {
+ return superField.getStructField(subFieldName);
+ }
+ return null;
+ }
+ return structFields.get(name);
+ }
+
+ /**
+ * Returns how the content of this field should be accent normalized etc
+ */
+ public NormalizeLevel getNormalizing() { return normalizing; }
+
+ /**
+ * Change how the content of this field should be accent normalized etc
+ */
+ public void setNormalizing(NormalizeLevel level) { normalizing = level; }
+
+ public void addQueryCommand(String name) {
+ queryCommands.add(name);
+ }
+
+ public boolean hasQueryCommand(String name) {
+ return queryCommands.contains(name);
+ }
+
+ /**
+ * A list of query commands
+ * @return a list of strings with query commands.
+ */
+ public List<String> getQueryCommands() {
+ return queryCommands;
+ }
+
+
+ public boolean isHeaderOrBodyDefined() {
+ return headerOrBodyDefined;
+ }
+
+ public void setHeaderOrBodyDefined(boolean headerOrBodySetExplicitly) {
+ this.headerOrBodyDefined = headerOrBodySetExplicitly;
+ }
+
+ /**
+ * The document that this field was declared in, or null
+ *
+ */
+ public SDDocumentType getOwnerDocType() {
+ return ownerDocType;
+ }
+
+ /**
+ * Two fields are equal if they have the same name
+ * No, they are not.
+ */
+ public boolean equals(Object other) {
+ if ( ! (other instanceof SDField)) return false;
+ return super.equals(other);
+ }
+
+ public int hashCode() {
+ return getName().hashCode();
+ }
+
+ public String toString() {
+ return "field '" + getName() + "'";
+ }
+
+ /** The aliases declared for this field */
+ public Map<String, String> getAliasToName() {
+ return aliasToName;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java
new file mode 100644
index 00000000000..a0511dec4b0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+
+/**
+ * A search-time document attribute sort specification(per-document in-memory value).
+ * This belongs to the attribute or field(implicitt attribute).
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public final class Sorting implements Cloneable, Serializable {
+
+ // Remember to change hashCode and equals when you add new fields
+ public enum Function {UCA, RAW, LOWERCASE}
+ public enum Strength {PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL}
+ private boolean ascending = true;
+ private Function function = Function.UCA;
+ private String locale = "";
+ private Strength strength = Strength.PRIMARY;
+
+ public boolean isAscending() { return ascending; }
+ public boolean isDescending() { return ! ascending; }
+ public String getLocale() { return locale; }
+ public Function getFunction() { return function; }
+ public Strength getStrength() { return strength; }
+
+ public void setAscending() { ascending = true; }
+ public void setDescending() { ascending = false; }
+ public void setFunction(Function function) { this.function = function; }
+ public void setLocale(String locale) { this.locale = locale; }
+ public void setStrength(Strength strength) { this.strength = strength; }
+
+ public int hashCode() {
+ return locale.hashCode() +
+ strength.hashCode() +
+ function.hashCode() +
+ (isDescending() ? 13 : 0);
+ }
+
+ public boolean equals(Object object) {
+ if (! (object instanceof Sorting)) return false;
+
+ Sorting other=(Sorting)object;
+ return this.locale.equals(other.locale) &&
+ (ascending == other.ascending) &&
+ (function == other.function) &&
+ (strength == other.strength);
+ }
+
+ public @Override Sorting clone() {
+ try {
+ return (Sorting)super.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Programming error");
+ }
+ }
+
+ public String toString() {
+ return "sorting '" + (isAscending() ? '+' : '-') + function.toString() + "(" + strength.toString() + ", " + locale + ")";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
new file mode 100644
index 00000000000..f471201f55e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.language.process.StemMode;
+
+import java.util.logging.Logger;
+
+/**
+ * <p>The stemming setting of a field. This describes how the search engine
+ * should transform content of this field into base forms (stems) to increase
+ * recall (find "car" when you search for "cars" etc.).</p>
+ *
+ * @author bratseth
+ */
+public enum Stemming {
+
+ /** No stemming */
+ NONE("none"),
+
+ /** Stem as much as possible */
+ ALL("all"),
+
+ /** select shortest possible stem */
+ SHORTEST("shortest"),
+
+ /** index (and query?) multiple stems */
+ MULTIPLE("multiple");
+
+ private static Logger log=Logger.getLogger(Stemming.class.getName());
+
+ private final String name;
+
+ /**
+ * Returns the stemming object for the given string.
+ * The legal stemming names are the stemming constants in any capitalization.
+ *
+ * @throws IllegalArgumentException if there is no stemming type with the given name
+ */
+ public static Stemming get(String stemmingName) {
+ try {
+ Stemming stemming = Stemming.valueOf(stemmingName.toUpperCase());
+ if (stemming.equals(ALL)) {
+ log.warning("note: stemming ALL is the same as stemming mode SHORTEST");
+ stemming = SHORTEST;
+ }
+ return stemming;
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting");
+ }
+ }
+
+ private Stemming(String name) {
+ this.name = name;
+ }
+
+ public String getName() { return name; }
+
+ public String toString() {
+ return "stemming " + name;
+ }
+
+ public StemMode toStemMode() {
+ if (this == Stemming.SHORTEST) {
+ return StemMode.SHORTEST;
+ }
+ if (this == Stemming.MULTIPLE) {
+ return StemMode.ALL;
+ }
+ return StemMode.NONE;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java
new file mode 100644
index 00000000000..193ec75acd0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.DataTypeName;
+
+/**
+ * @author balder
+ */
+public class TemporarySDDocumentType extends SDDocumentType {
+ public TemporarySDDocumentType(DataTypeName name) {
+ super(name);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
new file mode 100644
index 00000000000..00012d65434
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class TemporarySDField extends SDField {
+
+ public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) {
+ super(owner, name, dataType, isHeader, owner, false);
+ }
+
+ public TemporarySDField(String name, DataType dataType) {
+ super(null, name, dataType, true, false);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java
new file mode 100644
index 00000000000..6a10ebe642d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.DataType;
+
+/**
+ * Common interface for various typed key (or field definitions).
+ * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping
+ *
+ * @author bratseth
+ */
+public interface TypedKey {
+
+ String getName();
+
+ void setDataType(DataType type);
+
+ DataType getDataType();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java
new file mode 100644
index 00000000000..0f8e200c4c6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document.annotation;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.document.annotation.AnnotationType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SDAnnotationType extends AnnotationType {
+ private SDDocumentType sdDocType;
+ private String inherits;
+
+ public SDAnnotationType(String name) {
+ super(name);
+ }
+
+ public SDAnnotationType(String name, SDDocumentType dataType, String inherits) {
+ super(name);
+ this.sdDocType = dataType;
+ this.inherits = inherits;
+ }
+
+ public SDDocumentType getSdDocType() {
+ return sdDocType;
+ }
+
+ public String getInherits() {
+ return inherits;
+ }
+
+ public void inherit(String inherits) {
+ this.inherits = inherits;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java
new file mode 100644
index 00000000000..5b82b068908
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document.annotation;
+
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class TemporaryAnnotationReferenceDataType extends AnnotationReferenceDataType {
+ private final String target;
+
+ public TemporaryAnnotationReferenceDataType(String target) {
+ this.target = target;
+ }
+
+ public String getTarget() {
+ return target;
+ }
+
+ @Override
+ public void setAnnotationType(AnnotationType type) {
+ super.setName("annotationreference<" + type.getName() + ">");
+ super.setAnnotationType(type);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java
new file mode 100644
index 00000000000..60aeb3d50cc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AliasOperation implements FieldOperation {
+ private String aliasedName;
+ private String alias;
+
+ public AliasOperation(String aliasedName, String alias) {
+ this.aliasedName = aliasedName;
+ this.alias = alias;
+ }
+
+ public String getAliasedName() {
+ return aliasedName;
+ }
+
+ public void setAliasedName(String aliasedName) {
+ this.aliasedName = aliasedName;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public void apply(SDField field) {
+ if (aliasedName == null) {
+ aliasedName = field.getName();
+ }
+ field.getAliasToName().put(alias, aliasedName);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java
new file mode 100644
index 00000000000..19ee857c627
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.tensor.TensorType;
+
+import java.util.Optional;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AttributeOperation implements FieldOperation, FieldOperationContainer {
+ private final String name;
+ private Boolean huge;
+ private Boolean fastSearch;
+ private Boolean fastAccess;
+ private Boolean prefetch;
+ private Boolean enableBitVectors;
+ private Boolean enableOnlyBitVector;
+ //TODO: Husk sorting!!
+ private boolean doAlias = false;
+ private String alias;
+ private String aliasedName;
+ private Optional<TensorType> tensorType = Optional.empty();
+
+ public AttributeOperation(String name) {
+ this.name = name;
+ }
+
+ public void addOperation(FieldOperation op) {
+ //TODO: Implement this method:
+
+ }
+
+ public void applyOperations(SDField field) {
+ //TODO: Implement this method:
+
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Boolean getHuge() {
+ return huge;
+ }
+
+ public void setHuge(Boolean huge) {
+ this.huge = huge;
+ }
+
+ public Boolean getFastSearch() {
+ return fastSearch;
+ }
+
+ public void setFastSearch(Boolean fastSearch) {
+ this.fastSearch = fastSearch;
+ }
+
+ public Boolean getFastAccess() {
+ return fastAccess;
+ }
+
+ public void setFastAccess(Boolean fastAccess) {
+ this.fastAccess = fastAccess;
+ }
+
+ public Boolean getPrefetch() {
+ return prefetch;
+ }
+
+ public void setPrefetch(Boolean prefetch) {
+ this.prefetch = prefetch;
+ }
+
+ public Boolean getEnableBitVectors() {
+ return enableBitVectors;
+ }
+
+ public void setEnableBitVectors(Boolean enableBitVectors) {
+ this.enableBitVectors = enableBitVectors;
+ }
+
+ public Boolean getEnableOnlyBitVector() {
+ return enableOnlyBitVector;
+ }
+
+ public void setEnableOnlyBitVector(Boolean enableOnlyBitVector) {
+ this.enableOnlyBitVector = enableOnlyBitVector;
+ }
+
+ public boolean isDoAlias() {
+ return doAlias;
+ }
+
+ public void setDoAlias(boolean doAlias) {
+ this.doAlias = doAlias;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getAliasedName() {
+ return aliasedName;
+ }
+
+ public void setAliasedName(String aliasedName) {
+ this.aliasedName = aliasedName;
+ }
+
+ public void setTensorType(TensorType tensorType) {
+ this.tensorType = Optional.of(tensorType);
+ }
+
+ public void apply(SDField field) {
+ Attribute attribute = field.getAttributes().get(name);
+ if (attribute == null) {
+ attribute = new Attribute(name, field.getDataType());
+ field.addAttribute(attribute);
+ }
+
+ if (huge != null) {
+ attribute.setHuge(huge);
+ }
+ if (fastSearch != null) {
+ attribute.setFastSearch(fastSearch);
+ }
+ if (fastAccess != null) {
+ attribute.setFastAccess(fastAccess);
+ }
+ if (prefetch != null) {
+ attribute.setPrefetch(prefetch);
+ }
+ if (enableBitVectors != null) {
+ attribute.setEnableBitVectors(enableBitVectors);
+ }
+ if (enableOnlyBitVector != null) {
+ attribute.setEnableOnlyBitVector(enableOnlyBitVector);
+ }
+ if (doAlias) {
+ field.getAliasToName().put(alias, aliasedName);
+ }
+ if (tensorType.isPresent()) {
+ attribute.setTensorType(tensorType.get());
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java
new file mode 100644
index 00000000000..b830407ea2e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class BodyOperation implements FieldOperation {
+ public void apply(SDField field) {
+ field.setHeader(false);
+ field.setHeaderOrBodyDefined(true);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java
new file mode 100644
index 00000000000..f6c7277869e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class BoldingOperation implements FieldOperation {
+
+ private final boolean bold;
+
+ public BoldingOperation(boolean bold) {
+ this.bold = bold;
+ }
+
+ public void apply(SDField field) {
+ SummaryField summaryField = field.getSummaryField(field.getName(), true);
+ summaryField.addSource(field.getName());
+ summaryField.addDestination("default");
+ summaryField.setTransform(bold ? summaryField.getTransform().bold() : summaryField.getTransform().unbold());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
new file mode 100644
index 00000000000..20caf0b13c7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface FieldOperation {
+ public void apply(SDField field);
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
new file mode 100644
index 00000000000..30d16e369b1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface FieldOperationContainer {
+ public void addOperation(FieldOperation op);
+ public void applyOperations(SDField field);
+ public String getName();
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java
new file mode 100644
index 00000000000..48d2711b39e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class HeaderOperation implements FieldOperation {
+ public void apply(SDField field) {
+ field.setHeader(true);
+ field.setHeaderOrBodyDefined(true);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java
new file mode 100644
index 00000000000..25318e2db85
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IdOperation implements FieldOperation {
+ private SDDocumentType document;
+ private int fieldId;
+
+ public SDDocumentType getDocument() {
+ return document;
+ }
+
+ public void setDocument(SDDocumentType document) {
+ this.document = document;
+ }
+
+ public int getFieldId() {
+ return fieldId;
+ }
+
+ public void setFieldId(int fieldId) {
+ this.fieldId = fieldId;
+ }
+
+ public void apply(SDField field) {
+ document.setFieldId(field, fieldId);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
new file mode 100644
index 00000000000..8ab065ce419
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Index.Type;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexOperation implements FieldOperation {
+ private String indexName;
+ private Optional<Boolean> prefix = Optional.empty();
+ private List<String> aliases = new LinkedList<>();
+ private Optional<String> stemming = Optional.empty();
+ private Optional<Type> type = Optional.empty();
+
+ private OptionalInt arity = OptionalInt.empty(); // For predicate data type in boolean search
+ private OptionalLong lowerBound = OptionalLong.empty();
+ private OptionalLong upperBound = OptionalLong.empty();
+ private OptionalDouble densePostingListThreshold = OptionalDouble.empty();
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public void setIndexName(String indexName) {
+ this.indexName = indexName;
+ }
+
+ public boolean getPrefix() {
+ return prefix.get();
+ }
+
+ public void setPrefix(Boolean prefix) {
+ this.prefix = Optional.of(prefix);
+ }
+
+ public void addAlias(String alias) {
+ aliases.add(alias);
+ }
+
+ public String getStemming() {
+ return stemming.get();
+ }
+
+ public void setStemming(String stemming) {
+ this.stemming = Optional.of(stemming);
+ }
+
+ public void apply(SDField field) {
+ Index index = field.getIndex(indexName);
+
+ if (index == null) {
+ index = new Index(indexName);
+ field.addIndex(index);
+ }
+
+ applyToIndex(index);
+ }
+
+ public void applyToIndex(Index index) {
+ if (prefix.isPresent()) {
+ index.setPrefix(prefix.get());
+ }
+ for (String alias : aliases) {
+ index.addAlias(alias);
+ }
+ if (stemming.isPresent()) {
+ index.setStemming(Stemming.get(stemming.get()));
+ }
+ if (type.isPresent()) {
+ index.setType(type.get());
+ }
+ if (arity.isPresent() || lowerBound.isPresent() ||
+ upperBound.isPresent() || densePostingListThreshold.isPresent()) {
+ index.setBooleanIndexDefiniton(
+ new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold));
+ }
+ }
+
+ public Type getType() {
+ return type.get();
+ }
+
+ public void setType(Type type) {
+ this.type = Optional.of(type);
+ }
+
+ public void setArity(int arity) {
+ this.arity = OptionalInt.of(arity);
+ }
+
+ public void setLowerBound(long value) {
+ this.lowerBound = OptionalLong.of(value);
+ }
+
+ public void setUpperBound(long value) {
+ this.upperBound = OptionalLong.of(value);
+ }
+
+ public void setDensePostingListThreshold(double densePostingListThreshold) {
+ this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
new file mode 100644
index 00000000000..af3852bffb8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.searchdefinition.parser.SimpleCharStream;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexingOperation implements FieldOperation {
+
+ private final ScriptExpression script;
+
+ public IndexingOperation(ScriptExpression script) {
+ this.script = script;
+ }
+
+ public void apply(SDField field) {
+ field.setIndexingScript(script);
+ }
+
+ /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */
+ public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException {
+ return fromStream(input, multiLine, new SimpleLinguistics());
+ }
+
+ public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, Linguistics linguistics)
+ throws ParseException {
+ ScriptParserContext config = new ScriptParserContext(linguistics);
+ config.setAnnotatorConfig(new AnnotatorConfig());
+ config.setInputStream(input);
+ ScriptExpression exp;
+ try {
+ if (multiLine) {
+ exp = ScriptExpression.newInstance(config);
+ } else {
+ exp = new ScriptExpression(StatementExpression.newInstance(config));
+ }
+ } catch (com.yahoo.vespa.indexinglanguage.parser.ParseException e) {
+ ParseException t = new ParseException("Error reported by IL parser: " + e.getMessage());
+ t.initCause(e);
+ throw t;
+ }
+ return new IndexingOperation(exp);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
new file mode 100644
index 00000000000..a416895a6a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexingRewriteOperation implements FieldOperation {
+ public void apply(SDField field) {
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java
new file mode 100644
index 00000000000..9128778b7ea
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class MatchOperation implements FieldOperation {
+ private Matching.Type matchingType;
+ private Integer gramSize;
+ private Integer maxLength;
+ private Matching.Algorithm matchingAlgorithm;
+ private String exactMatchTerminator;
+
+ public void setMatchingType(Matching.Type matchingType) {
+ this.matchingType = matchingType;
+ }
+
+ public void setGramSize(Integer gramSize) {
+ this.gramSize = gramSize;
+ }
+ public void setMaxLength(Integer maxLength) {
+ this.maxLength = maxLength;
+ }
+
+ public void setMatchingAlgorithm(Matching.Algorithm matchingAlgorithm) {
+ this.matchingAlgorithm = matchingAlgorithm;
+ }
+
+ public void setExactMatchTerminator(String exactMatchTerminator) {
+ this.exactMatchTerminator = exactMatchTerminator;
+ }
+
+ public void apply(SDField field) {
+ if (matchingType != null) {
+ field.setMatchingType(matchingType);
+ }
+ if (gramSize != null) {
+ field.getMatching().setGramSize(gramSize);
+ }
+ if (maxLength != null) {
+ field.getMatching().maxLength(maxLength);
+ }
+ if (matchingAlgorithm != null) {
+ field.setMatchingAlgorithm(matchingAlgorithm);
+ }
+ if (exactMatchTerminator != null) {
+ field.getMatching().setExactMatchTerminator(exactMatchTerminator);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java
new file mode 100644
index 00000000000..e0d15cabbe0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.NormalizeLevel;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class NormalizingOperation implements FieldOperation {
+ private NormalizeLevel.Level level;
+
+ public NormalizingOperation(String setting) {
+ if ("none".equals(setting)) {
+ this.level = NormalizeLevel.Level.NONE;
+ } else if ("codepoint".equals(setting)) {
+ this.level = NormalizeLevel.Level.CODEPOINT;
+ } else if ("lowercase".equals(setting)) {
+ this.level = NormalizeLevel.Level.LOWERCASE;
+ } else if ("accent".equals(setting)) {
+ this.level = NormalizeLevel.Level.ACCENT;
+ } else if ("all".equals(setting)) {
+ this.level = NormalizeLevel.Level.ACCENT;
+ } else {
+ throw new IllegalArgumentException("invalid normalizing setting: "+setting);
+ }
+ }
+
+ public void apply(SDField field) {
+ field.setNormalizing(new NormalizeLevel(level, true));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java
new file mode 100644
index 00000000000..e4af5660e7b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class QueryCommandOperation implements FieldOperation {
+ private List<String> queryCommands = new java.util.ArrayList<>(0);
+
+ public void addQueryCommand(String name) {
+ queryCommands.add(name);
+ }
+
+ public void apply(SDField field) {
+ for (String command : queryCommands) {
+ field.addQueryCommand(command);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java
new file mode 100644
index 00000000000..42b1e5514a1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class RankOperation implements FieldOperation {
+
+ private Boolean literal = null;
+ private Boolean filter = null;
+ private Boolean normal = null;
+
+ public Boolean getLiteral() { return literal; }
+ public void setLiteral(Boolean literal) { this.literal = literal; }
+
+ public Boolean getFilter() { return filter; }
+ public void setFilter(Boolean filter) { this.filter = filter; }
+
+ public Boolean getNormal() { return normal; }
+ public void setNormal(Boolean n) { this.normal = n; }
+
+ public void apply(SDField field) {
+ if (literal != null) {
+ field.getRanking().setLiteral(literal);
+ }
+ if (filter != null) {
+ field.getRanking().setFilter(filter);
+ }
+ if (normal != null) {
+ field.getRanking().setNormal(normal);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java
new file mode 100644
index 00000000000..640ccc48818
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class RankTypeOperation implements FieldOperation {
+ private String indexName;
+ private RankType type;
+
+ public String getIndexName() {
+ return indexName;
+ }
+ public void setIndexName(String indexName) {
+ this.indexName = indexName;
+ }
+
+ public RankType getType() {
+ return type;
+ }
+ public void setType(RankType type) {
+ this.type = type;
+ }
+
+ public void apply(SDField field) {
+ if (indexName == null) {
+ field.setRankType(type); // Set default if the index is not specified.
+ } else {
+ Index index = field.getIndex(indexName);
+ if (index == null) {
+ index = new Index(indexName);
+ field.addIndex(index);
+ }
+ index.setRankType(type);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java
new file mode 100644
index 00000000000..e9225f0def5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SortingOperation implements FieldOperation {
+ private String attributeName;
+ private Boolean ascending;
+ private Boolean descending;
+ private Sorting.Function function;
+ private Sorting.Strength strength;
+ private String locale;
+
+ public SortingOperation(String attributeName) {
+ this.attributeName = attributeName;
+ }
+
+ public String getAttributeName() {
+ return attributeName;
+ }
+
+ public Boolean getAscending() {
+ return ascending;
+ }
+
+ public void setAscending() {
+ this.ascending = true;
+ }
+
+ public Boolean getDescending() {
+ return descending;
+ }
+
+ public void setDescending() {
+ this.descending = true;
+ }
+
+ public Sorting.Function getFunction() {
+ return function;
+ }
+
+ public void setFunction(Sorting.Function function) {
+ this.function = function;
+ }
+
+ public Sorting.Strength getStrength() {
+ return strength;
+ }
+
+ public void setStrength(Sorting.Strength strength) {
+ this.strength = strength;
+ }
+
+ public String getLocale() {
+ return locale;
+ }
+
+ public void setLocale(String locale) {
+ this.locale = locale;
+ }
+
+ public void apply(SDField field) {
+ Attribute attribute = field.getAttributes().get(attributeName);
+ if (attribute == null) {
+ attribute = new Attribute(attributeName, field.getDataType());
+ field.addAttribute(attribute);
+ }
+ Sorting sorting = attribute.getSorting();
+
+ if (ascending != null) {
+ sorting.setAscending();
+ }
+ if (descending != null) {
+ sorting.setDescending();
+ }
+ if (function != null) {
+ sorting.setFunction(function);
+ }
+ if (strength != null) {
+ sorting.setStrength(strength);
+ }
+ if (locale != null) {
+ sorting.setLocale(locale);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java
new file mode 100644
index 00000000000..19b760386a1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StemmingOperation implements FieldOperation {
+ private String setting;
+
+ public String getSetting() {
+ return setting;
+ }
+
+ public void setSetting(String setting) {
+ this.setting = setting;
+ }
+
+ public void apply(SDField field) {
+ field.setStemming(Stemming.get(setting));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java
new file mode 100644
index 00000000000..f0c3d964934
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructFieldOperation implements FieldOperation, FieldOperationContainer {
+ private String structFieldName;
+ private List<FieldOperation> pendingOperations = new LinkedList<>();
+
+ public StructFieldOperation(String structFieldName) {
+ this.structFieldName = structFieldName;
+ }
+
+ public void apply(SDField field) {
+ SDField structField = field.getStructField(structFieldName);
+ if (structField == null ) {
+ throw new IllegalArgumentException("Struct field '" + structFieldName + "' has not been defined in struct " +
+ "for field '" + field.getName() + "'.");
+ }
+
+ applyOperations(structField);
+ }
+
+ public void addOperation(FieldOperation op) {
+ pendingOperations.add(op);
+ }
+
+ public void applyOperations(SDField field) {
+ if (pendingOperations.isEmpty()) {
+ return;
+ }
+ ListIterator<FieldOperation> ops = pendingOperations.listIterator();
+ while (ops.hasNext()) {
+ FieldOperation op = ops.next();
+ ops.remove();
+ op.apply(field);
+ }
+ }
+
+ public String getName() {
+ return structFieldName;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java
new file mode 100644
index 00000000000..f320ef649b4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SummaryInFieldLongOperation extends SummaryInFieldOperation {
+ private DataType type;
+ private Boolean bold;
+ private Set<String> destinations = new java.util.LinkedHashSet<>();
+ private List<SummaryField.Property> properties = new ArrayList<>();
+
+ public SummaryInFieldLongOperation(String name) {
+ super(name);
+ }
+
+ public SummaryInFieldLongOperation() {
+ super(null);
+ }
+
+ public void setType(DataType type) {
+ this.type = type;
+ }
+
+ public void setBold(Boolean bold) {
+ this.bold = bold;
+ }
+
+ public void addDestination(String destination) {
+ destinations.add(destination);
+ }
+
+ public Iterator<String> destinationIterator() {
+ return destinations.iterator();
+ }
+
+
+ public void addProperty(SummaryField.Property property) {
+ properties.add(property);
+ }
+
+ public void apply(SDField field) {
+ if (type == null) {
+ type = field.getDataType();
+ }
+ SummaryField summary = new SummaryField(name, type);
+ applyToSummary(summary);
+ field.addSummaryField(summary);
+ }
+
+ public void applyToSummary(SummaryField summary) {
+ if (transform != null) {
+ summary.setTransform(transform);
+ }
+
+ if (bold != null) {
+ summary.setTransform(bold ? summary.getTransform().bold() : summary.getTransform().unbold());
+ }
+
+ for (SummaryField.Source source : sources) {
+ summary.addSource(source);
+ }
+
+ for (String destination : destinations) {
+ summary.addDestination(destination);
+ }
+
+ for (SummaryField.Property prop : properties) {
+ summary.addProperty(prop.getName(), prop.getValue());
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java
new file mode 100644
index 00000000000..e1abbcc93b5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class SummaryInFieldOperation implements FieldOperation {
+ protected String name;
+ protected SummaryTransform transform;
+ protected Set<SummaryField.Source> sources = new java.util.LinkedHashSet<>();
+
+ public SummaryInFieldOperation(String name) {
+ this.name = name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setTransform(SummaryTransform transform) {
+ this.transform = transform;
+ }
+
+ public SummaryTransform getTransform() {
+ return transform;
+ }
+
+ public void addSource(String name) {
+ sources.add(new SummaryField.Source(name));
+ }
+
+ public void addSource(SummaryField.Source source) {
+ sources.add(source);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java
new file mode 100644
index 00000000000..1ffb8d9a83e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SummaryInFieldShortOperation extends SummaryInFieldOperation {
+ public SummaryInFieldShortOperation(String name) {
+ super(name);
+ }
+
+ public void apply(SDField field) {
+ SummaryField ret = field.getSummaryField(name);
+ if (ret == null) {
+ ret = new SummaryField(name, field.getDataType());
+ ret.addSource(field.getName());
+ ret.addDestination("default");
+ }
+ ret.setImplicit(false);
+
+ ret.setTransform(transform);
+ for (SummaryField.Source source : sources) {
+ ret.addSource(source);
+ }
+ field.addSummaryField(ret);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java
new file mode 100644
index 00000000000..f392b2fdcc8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SummaryToOperation implements FieldOperation {
+ private Set<String> destinations = new java.util.LinkedHashSet<>();
+ private String name;
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void addDestination(String destination) {
+ destinations.add(destination);
+ }
+
+ public void apply(SDField field) {
+ SummaryField summary;
+ summary = field.getSummaryField(name);
+ if (summary == null) {
+ summary = new SummaryField(field);
+ summary.addSource(field.getName());
+ summary.addDestination("default");
+ field.addSummaryField(summary);
+ }
+ summary.setImplicit(false);
+
+ for (String destination : destinations) {
+ summary.addDestination(destination);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java
new file mode 100644
index 00000000000..3cbd557b850
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightOperation implements FieldOperation {
+ private int weight;
+
+ public int getWeight() {
+ return weight;
+ }
+
+ public void setWeight(int weight) {
+ this.weight = weight;
+ }
+
+ public void apply(SDField field) {
+ field.setWeight(weight);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java
new file mode 100644
index 00000000000..c1721a4dbb5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.document.WeightedSetDataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightedSetOperation implements FieldOperation {
+
+ private Boolean createIfNonExistent;
+ private Boolean removeIfZero;
+
+ public Boolean getCreateIfNonExistent() {
+ return createIfNonExistent;
+ }
+
+ public void setCreateIfNonExistent(Boolean createIfNonExistent) {
+ this.createIfNonExistent = createIfNonExistent;
+ }
+
+ public Boolean getRemoveIfZero() {
+ return removeIfZero;
+ }
+
+ public void setRemoveIfZero(Boolean removeIfZero) {
+ this.removeIfZero = removeIfZero;
+ }
+
+ public void apply(SDField field) {
+ WeightedSetDataType ctype = (WeightedSetDataType) field.getDataType();
+
+ if (createIfNonExistent != null) {
+ field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), createIfNonExistent,
+ ctype.removeIfZero()));
+ }
+
+ ctype = (WeightedSetDataType) field.getDataType();
+ if (removeIfZero != null) {
+ field.setDataType(DataType.getWeightedSet(ctype.getNestedType(),
+ ctype.createIfNonExistent(), removeIfZero));
+ }
+
+ ctype = (WeightedSetDataType) field.getDataType();
+ for (Object o : field.getAttributes().values()) {
+ Attribute attribute = (Attribute) o;
+ attribute.setRemoveIfZero(ctype.removeIfZero());
+ attribute.setCreateIfNonExistent(ctype.createIfNonExistent());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "WeightedSetOperation{" +
+ "createIfNonExistent=" + createIfNonExistent +
+ ", removeIfZero=" + removeIfZero +
+ '}';
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java
new file mode 100644
index 00000000000..b925b4f2caa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.parser;
+
+import com.yahoo.javacc.FastCharStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings("deprecation")
+public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream,
+ com.yahoo.vespa.indexinglanguage.parser.CharStream
+{
+ public SimpleCharStream(String input) {
+ super(input);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
new file mode 100644
index 00000000000..5d39c70e3b9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * This processor creates a {@link com.yahoo.searchdefinition.document.SDDocumentType} for each {@link Search} object which holds all the data that search
+ * associates with a document described in a search definition file. This includes all extra fields, summary fields and
+ * implicit fields. All non-indexed and non-summary fields are discarded.
+ */
+public class AddExtraFieldsToDocument extends Processor {
+
+ public AddExtraFieldsToDocument(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ SDDocumentType document = search.getDocument();
+ if (document != null) {
+ for (Field field : search.extraFieldList()) {
+ addSdField(search, document, (SDField)field);
+ }
+ for (SummaryField field : search.getSummary("default").getSummaryFields()) {
+ addSummaryField(search, document, field);
+ }
+ }
+ }
+
+ private void addSdField(Search search, SDDocumentType document, SDField field) {
+ if (field.getIndexToCount() == 0 && field.getAttributes().isEmpty()) {
+ return;
+ }
+ for (Attribute atr : field.getAttributes().values()) {
+ if (atr.getName().equals(field.getName() + "_position")) {
+ DataType type = PositionDataType.INSTANCE;
+ if (atr.getCollectionType().equals(Attribute.CollectionType.ARRAY)) {
+ type = DataType.getArray(type);
+ }
+ addField(search, document, new SDField(document, atr.getName(), type));
+ } else if (!atr.getName().equals(field.getName())) {
+ addField(search, document, new SDField(document, atr.getName(), atr.getDataType()));
+ }
+ }
+ addField(search, document, field);
+ }
+
+ private void addSummaryField(Search search, SDDocumentType document, SummaryField field) {
+ Field docField = document.getField(field.getName());
+ if (docField == null) {
+ SDField newField = search.getField(field.getName());
+ if (newField == null) {
+ newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true);
+ newField.setIsExtraField(true);
+ }
+ document.addField(newField);
+ } else if (!docField.getDataType().equals(field.getDataType())) {
+ throw newProcessException(search, field, "Summary field has conflicting type.");
+ }
+ }
+
+ private void addField(Search search, SDDocumentType document, Field field) {
+ if (document.getField(field.getName()) != null) {
+ throw newProcessException(search, field, "Field shadows another.");
+ }
+ document.addField(field);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
new file mode 100644
index 00000000000..9df77c09373
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Checks that attribute properties only are set for attributes that have data (are created by an indexing statement).
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class AttributeProperties extends Processor {
+
+ public AttributeProperties(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ String fieldName = field.getName();
+
+ // For each attribute, check if the attribute has been created
+ // by an indexing statement.
+ for (Attribute attribute : field.getAttributes().values()) {
+ if (attributeCreated(field, attribute.getName())) {
+ continue;
+ }
+ // Check other fields or statements that may have created this attribute.
+ boolean created = false;
+ for (SDField f : search.allFieldsList()) {
+ // Checking against the field we are looking at
+ if (!f.getName().equals(fieldName)) {
+ if (attributeCreated(f, attribute.getName())) {
+ created = true;
+ break;
+ }
+ }
+ }
+ if (!created) {
+ throw new IllegalArgumentException("Attribute '" + attribute.getName() + "' in field '" +
+ field.getName() + "' is not created by the indexing statement");
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks if the attribute has been created bye an indexing statement in this field.
+ *
+ * @param field a searchdefinition field
+ * @param attributeName name of the attribute
+ * @return true if the attribute has been created by this field, else false
+ */
+ static boolean attributeCreated(SDField field, String attributeName) {
+ if (!field.doesAttributing()) {
+ return false;
+ }
+ for (Attribute attribute : field.getAttributes().values()) {
+ if (attribute.getName().equals(attributeName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
new file mode 100644
index 00000000000..774160793cb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Fields that derive to attribute(s) and no indices should use the WORD indexing form,
+ * in a feeble attempt to match the most peoples expectations as closely as possible.
+ *
+ * @author vegardh
+ *
+ */
+public class AttributesImplicitWord extends Processor {
+
+ public AttributesImplicitWord(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (fieldImplicitlyWordMatch(field)) {
+ field.getMatching().setType(Matching.Type.WORD);
+ }
+ }
+ }
+
+ private boolean fieldImplicitlyWordMatch(SDField field) {
+ // numeric types should not trigger exact-match query parsing
+ DataType dt = field.getDataType().getPrimitiveType();
+ if (dt != null && dt instanceof NumericDataType) {
+ return false;
+ }
+ return (field.getIndexToCount() == 0
+ && field.getAttributes().size() > 0
+ && field.getIndices().isEmpty()
+ && !field.getMatching().isTypeUserSet());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
new file mode 100644
index 00000000000..2e2e665eb34
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Checks that bolding or dynamic summary is turned on only for text fields. Throws exception if it is turned on for any
+ * other fields (otherwise will cause indexing failure)
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class Bolding extends Processor {
+
+ public Bolding(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ for (SummaryField summary : field.getSummaryFields()) {
+ if (summary.getTransform().isBolded() &&
+ !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI)))
+ {
+ throw new IllegalArgumentException("'bolding: on' for non-text field " +
+ "'" + field.getName() + "'" +
+ " (" + summary.getDataType() + ")" +
+ " is not allowed");
+ } else if (summary.getTransform().isDynamic() &&
+ !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI)))
+ {
+ throw new IllegalArgumentException("'summary: dynamic' for non-text field " +
+ "'" + field.getName() + "'" +
+ " (" + summary.getDataType() + ")" +
+ " is not allowed");
+ }
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
new file mode 100644
index 00000000000..4ad9b5304a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document
+ * @author vegardh
+ *
+ */
+public class BuiltInFieldSets extends Processor {
+
+ private static final String DOC_FIELDSET_NAME = "[document]";
+ public static final String SEARCH_FIELDSET_NAME = "[search]"; // Public due to oddities in position handling.
+ public static final String INTERNAL_FIELDSET_NAME = "[internal]"; // This one populated from misc places
+
+ public BuiltInFieldSets(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ addDocumentFieldSet();
+ addSearchFieldSet();
+ // "Hook" the field sets on search onto the document types, since we will include them
+ // on the document configs
+ search.getDocument().setFieldSets(search.fieldSets());
+ }
+
+ private void addSearchFieldSet() {
+ for (SDField searchField : search.extraFieldList()) {
+ search.fieldSets().addBuiltInFieldSetItem(SEARCH_FIELDSET_NAME, searchField.getName());
+ }
+ }
+
+ private void addDocumentFieldSet() {
+ for (Field docField : search.getDocument().fieldSet()) {
+ search.fieldSets().addBuiltInFieldSetItem(DOC_FIELDSET_NAME, docField.getName());
+ }
+ }
+
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
new file mode 100644
index 00000000000..0aabd4dc192
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
@@ -0,0 +1,201 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+
+/**
+ * Adds a "fieldName.zcurve" long attribute and a "fieldName.distance" summary field to all position type fields.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class CreatePositionZCurve extends Processor {
+
+ public CreatePositionZCurve(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ DataType fieldType = field.getDataType();
+ if ( ! isSupportedPositionType(fieldType)) continue;
+
+ if (field.doesIndexing()) {
+ fail(search, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " +
+ "replace 'index' statement with 'attribute'.");
+ }
+
+ if ( ! field.doesAttributing()) continue;
+
+ boolean doesSummary = field.doesSummarying();
+
+ String fieldName = field.getName();
+ field.getAttributes().remove(fieldName);
+
+ String zName = PositionDataType.getZCurveFieldName(fieldName);
+ SDField zCurveField = createZCurveField(field, zName);
+ search.addExtraField(zCurveField);
+ search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName());
+
+ // configure summary
+ Collection<String> summaryTo = removeSummaryTo(field);
+ ensureCompatibleSummary(field, zName,
+ PositionDataType.getPositionSummaryFieldName(fieldName),
+ DataType.getArray(DataType.STRING), // will become "xmlstring"
+ SummaryTransform.POSITIONS, summaryTo);
+ ensureCompatibleSummary(field, zName,
+ PositionDataType.getDistanceSummaryFieldName(fieldName),
+ DataType.INT,
+ SummaryTransform.DISTANCE, summaryTo);
+ // clear indexing script
+ field.setIndexingScript(null);
+ SDField posX = field.getStructField(PositionDataType.FIELD_X);
+ if (posX != null) {
+ posX.setIndexingScript(null);
+ }
+ SDField posY = field.getStructField(PositionDataType.FIELD_Y);
+ if (posY != null) {
+ posY.setIndexingScript(null);
+ }
+ if (doesSummary) ensureCompatibleSummary(field, zName,
+ field.getName(),
+ field.getDataType(),
+ SummaryTransform.GEOPOS, summaryTo);
+ }
+ }
+
+ private SDField createZCurveField(SDField inputField, String fieldName) {
+ if (search.getField(fieldName) != null || search.getAttribute(fieldName) != null) {
+ throw newProcessException(search, null, "Incompatible position attribute '" + fieldName +
+ "' already created.");
+ }
+ boolean isArray = inputField.getDataType() instanceof ArrayDataType;
+ SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG);
+ Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY :
+ Attribute.CollectionType.SINGLE);
+ attribute.setPosition(true);
+ attribute.setFastSearch(true);
+ field.addAttribute(attribute);
+
+ ScriptExpression script = inputField.getIndexingScript();
+ script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script);
+ script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script);
+ field.setIndexingScript(script);
+ return field;
+ }
+
+ private void ensureCompatibleSummary(SDField field, String sourceName, String summaryName, DataType summaryType,
+ SummaryTransform summaryTransform, Collection<String> summaryTo) {
+ SummaryField summary = search.getSummaryField(summaryName);
+ if (summary == null) {
+ summary = new SummaryField(summaryName, summaryType, summaryTransform);
+ summary.addDestination("default");
+ summary.addDestinations(summaryTo);
+ field.addSummaryField(summary);
+ } else if (!summary.getDataType().equals(summaryType)) {
+ fail(search, field, "Incompatible summary field '" + summaryName + "' type "+summary.getDataType()+" already created.");
+ } else if (summary.getTransform() == SummaryTransform.NONE) {
+ summary.setTransform(summaryTransform);
+ summary.addDestination("default");
+ summary.addDestinations(summaryTo);
+ } else if (summary.getTransform() != summaryTransform) {
+ deployLogger.log(Level.WARNING, "Summary field " + summaryName + " has wrong transform: " + summary.getTransform());
+ return;
+ }
+ SummaryField.Source source = new SummaryField.Source(sourceName);
+ summary.getSources().clear();
+ summary.addSource(source);
+ }
+
+ private Set<String> removeSummaryTo(SDField field) {
+ Set<String> summaryTo = new HashSet<>();
+ Collection<SummaryField> summaryFields = field.getSummaryFields();
+ for (SummaryField summary : summaryFields) {
+ summaryTo.addAll(summary.getDestinations());
+ }
+ summaryFields.clear();
+ return summaryTo;
+ }
+
+ private static boolean isSupportedPositionType(DataType dataType) {
+ if (dataType instanceof ArrayDataType) {
+ dataType = ((ArrayDataType)dataType).getNestedType();
+ }
+ return dataType.equals(PositionDataType.INSTANCE);
+ }
+
+ private static class RemoveSummary extends ExpressionConverter {
+
+ final String find;
+
+ RemoveSummary(String find) {
+ this.find = find;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ if (!(exp instanceof SummaryExpression)) {
+ return false;
+ }
+ String fieldName = ((SummaryExpression)exp).getFieldName();
+ return fieldName == null || fieldName.equals(find);
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ return null;
+ }
+ }
+
+ private static class PerformZCurve extends ExpressionConverter {
+
+ final String find;
+ final String replace;
+ final boolean isArray;
+
+ PerformZCurve(SDField find, String replace) {
+ this.find = find.getName();
+ this.replace = replace;
+ this.isArray = find.getDataType() instanceof ArrayDataType;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ if (!(exp instanceof AttributeExpression)) {
+ return false;
+ }
+ String fieldName = ((AttributeExpression)exp).getFieldName();
+ return fieldName == null || fieldName.equals(find);
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ return new StatementExpression(
+ isArray ? new ForEachExpression(new ZCurveExpression()) :
+ new ZCurveExpression(), new AttributeExpression(replace));
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
new file mode 100644
index 00000000000..6d5b3ff8936
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+public class DeprecateAttributePrefetch extends Processor {
+
+ public DeprecateAttributePrefetch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ for (Attribute a : field.getAttributes().values()) {
+ if (Boolean.TRUE.equals(a.getPrefetchValue())) {
+ warn(search, field, "Attribute prefetch is deprecated. Use an explicitly defined document summary with all desired fields defined as attribute.");
+ }
+ if (Boolean.FALSE.equals(a.getPrefetchValue())) {
+ warn(search, field, "Attribute prefetch is deprecated. no-prefetch can be removed.");
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
new file mode 100644
index 00000000000..dbe83143189
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Non-primitive key types for map and weighted set forbidden (though OK in document model)
+ * @author vegardh
+ *
+ */
+public class DisallowComplexMapAndWsetKeyTypes extends Processor {
+
+ public DisallowComplexMapAndWsetKeyTypes(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do
+ // not start using the static stuff on SDDocumentTypes any more.
+ for (SDField field : search.allFieldsList()) {
+ if (field.getDataType() instanceof WeightedSetDataType) {
+ DataType nestedType = ((WeightedSetDataType)field.getDataType()).getNestedType();
+ if (!(nestedType instanceof PrimitiveDataType)) {
+ fail(search, field, "Weighted set must have a primitive key type.");
+ }
+ } else if (field.getDataType() instanceof MapDataType) {
+ if (!(((MapDataType)field.getDataType()).getKeyType() instanceof PrimitiveDataType)) {
+ fail(search, field, "Map key type must be a primitive type");
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
new file mode 100644
index 00000000000..49701f3e443
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Created by balder on 3/10/15.
+ */
+public class DiversitySettingsValidator extends Processor {
+ public DiversitySettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) {
+ if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) {
+ validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity());
+ }
+ }
+ }
+ private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) {
+ String attributeName = settings.getAttribute();
+ new AttributeValidator(search.getName(), rankProfile.getName(),
+ search.getAttribute(attributeName), attributeName).validate();
+ }
+
+ private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator {
+ public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) {
+ super(searchName, rankProfileName, attribute, attributeName);
+ }
+
+ protected void validateThatAttributeIsSingleAndNotPredicate() {
+ if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) ||
+ attribute.getType().equals(Attribute.Type.PREDICATE))
+ {
+ failValidation("must be single value numeric, or enumerated attribute, but it is '"
+ + attribute.getDataType().getName() + "'");
+ }
+ }
+
+ @Override
+ public void validate() {
+ validateThatAttributeExists();
+ validateThatAttributeIsSingleAndNotPredicate();
+ }
+
+ @Override
+ public String getValidationType() {
+ return "diversity";
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
new file mode 100644
index 00000000000..d53be4bd18d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.indexinglanguage.ExpressionSearcher;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of exact matching
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ExactMatch extends Processor {
+
+ public static final String DEFAULT_EXACT_TERMINATOR = "@@";
+
+ public ExactMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ Matching.Type matching = field.getMatching().getType();
+ if (matching.equals(Matching.Type.EXACT) ||
+ matching.equals(Matching.Type.WORD))
+ {
+ implementExactMatch(field, search);
+ } else if (field.getMatching().getExactMatchTerminator() != null) {
+ warn(search, field, "exact-terminator requires 'exact' matching to have any effect.");
+ }
+ }
+ }
+
+ private void implementExactMatch(SDField field, Search search) {
+ field.setStemming(Stemming.NONE);
+ field.getNormalizing().inferLowercase();
+
+ if (field.getMatching().getType().equals(Matching.Type.WORD)) {
+ field.addQueryCommand("word");
+ } else { // exact
+ String exactTerminator = DEFAULT_EXACT_TERMINATOR;
+ if (field.getMatching().getExactMatchTerminator() != null &&
+ ! field.getMatching().getExactMatchTerminator().equals(""))
+ {
+ exactTerminator = field.getMatching().getExactMatchTerminator();
+ } else {
+ warn(search, field,
+ "With 'exact' matching, an exact-terminator is needed (using \""
+ + exactTerminator +"\" as terminator)");
+ }
+ field.addQueryCommand("exact " + exactTerminator);
+
+ // The following part illustrates how nice it would have been with canonical
+ // representation of indices
+ if (field.doesIndexing()) {
+ exactMatchSettingsForField(field);
+ }
+ }
+ ScriptExpression script = field.getIndexingScript();
+ if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) {
+ field.setIndexingScript((ScriptExpression)new MyProvider(search).convert(field.getIndexingScript()));
+ }
+ }
+
+ private void exactMatchSettingsForField(SDField field) {
+ field.getRanking().setFilter(true);
+ }
+
+ private static class MyProvider extends TypedTransformProvider {
+
+ MyProvider(Search search) {
+ super(ExactExpression.class, search);
+ }
+
+ @Override
+ protected boolean requiresTransform(Expression exp, DataType fieldType) {
+ return exp instanceof OutputExpression;
+ }
+
+ @Override
+ protected Expression newTransform(DataType fieldType) {
+ Expression exp = new ExactExpression();
+ if (fieldType instanceof CollectionDataType) {
+ exp = new ForEachExpression(exp);
+ }
+ return exp;
+ }
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
new file mode 100644
index 00000000000..e809fe90c7f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.NormalizeLevel;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Computes the right "index commands" for each fieldset in a search definition.
+ *
+ *
+ * @author vegardh
+ * @author bratseth
+ */
+public class FieldSetValidity extends Processor {
+
+ public FieldSetValidity(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) {
+ checkFieldNames(search, fieldSet);
+ checkMatching(search, fieldSet);
+ checkNormalization(search, fieldSet);
+ checkStemming(search, fieldSet);
+ }
+ }
+
+ private void checkFieldNames(Search search, FieldSet fieldSet) {
+ for (String fld : fieldSet.getFieldNames()) {
+ SDField field = search.getField(fld);
+ if (field == null) {
+ throw new IllegalArgumentException("For search '" + search.getName() + "': Field '"+ fld + "' in " + fieldSet + " does not exist.");
+ }
+ }
+ }
+
+ private void checkMatching(Search search, FieldSet fieldSet) {
+ Matching fsMatching = null;
+ for (String fld : fieldSet.getFieldNames()) {
+ SDField field = search.getField(fld);
+ Matching fieldMatching = field.getMatching();
+ if (fsMatching==null) {
+ fsMatching = fieldMatching;
+ } else {
+ if (fsMatching!=null && !fsMatching.equals(fieldMatching)) {
+ warn(search, field, "The matching settings for the fields in " + fieldSet + " are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues.");
+ return;
+ }
+ }
+ }
+ }
+
+ private void checkNormalization(Search search, FieldSet fieldSet) {
+ NormalizeLevel.Level fsNorm = null;
+ for (String fld : fieldSet.getFieldNames()) {
+ SDField field = search.getField(fld);
+ NormalizeLevel.Level fieldNorm = field.getNormalizing().getLevel();
+ if (fsNorm==null) {
+ fsNorm = fieldNorm;
+ } else {
+ if (fsNorm!=null && !fsNorm.equals(fieldNorm)) {
+ warn(search, field, "The normalization settings for the fields in " + fieldSet + " are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues.");
+ return;
+ }
+ }
+ }
+ }
+
+ private void checkStemming(Search search, FieldSet fieldSet) {
+ Stemming fsStemming = null;
+ for (String fld : fieldSet.getFieldNames()) {
+ SDField field = search.getField(fld);
+ Stemming fieldStemming = field.getStemming();
+ if (fsStemming==null) {
+ fsStemming = fieldStemming;
+ } else {
+ if (fsStemming!=null && !fsStemming.equals(fieldStemming)) {
+ warn(search, field, "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+"' are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues.");
+ return;
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
new file mode 100644
index 00000000000..1df8c642750
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.logging.Level;
+
+/**
+ * Takes the fields and indexes that are of type rank filter, and stores those names on all rank profiles
+ * @author vegardh
+ *
+ */
+public class FilterFieldNames extends Processor {
+
+ public FilterFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField f : search.allFieldsList()) {
+ if (f.getRanking().isFilter()) {
+ filterField(f.getName());
+ }
+ }
+ for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) {
+ Set<String> filterFields = new LinkedHashSet<>();
+ findFilterFields(search, profile, filterFields);
+ for (Iterator<String> itr = filterFields.iterator(); itr.hasNext(); ) {
+ String fieldName = itr.next();
+ profile.filterFields().add(fieldName);
+ profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.EMPTY);
+ }
+ }
+ }
+
+ private void filterField(String f) {
+ for (RankProfile rp : rankProfileRegistry.localRankProfiles(search)) {
+ rp.filterFields().add(f);
+ }
+ }
+
+ private void findFilterFields(Search search, RankProfile profile, Set<String> filterFields) {
+ for (Iterator<RankProfile.RankSetting> itr = profile.declaredRankSettingIterator(); itr.hasNext(); ) {
+ RankProfile.RankSetting setting = itr.next();
+ if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) &&
+ ((Boolean)setting.getValue()).booleanValue())
+ {
+ String fieldName = setting.getFieldName();
+ if (search.getField(fieldName) != null) {
+ if (!profile.filterFields().contains(fieldName)) {
+ filterFields.add(fieldName);
+ }
+ } else {
+ deployLogger.log(Level.WARNING, "For rank profile '" + profile.getName() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'");
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
new file mode 100644
index 00000000000..fa7725843b1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
@@ -0,0 +1,222 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import java.util.logging.Level;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Makes implicitly defined summaries into explicit summaries
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ImplicitSummaries extends Processor {
+
+ public ImplicitSummaries(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ DocumentSummary defaultSummary=search.getSummary("default");
+ if (defaultSummary==null) {
+ defaultSummary=new DocumentSummary("default");
+ search.addSummary(defaultSummary);
+ }
+
+ for (SDField field : search.allFieldsList()) {
+ collectSummaries(field,search);
+ }
+
+ for (DocumentSummary documentSummary : search.getSummaries().values()) {
+ documentSummary.purgeImplicits();
+ }
+ }
+
+ private void addSummaryFieldSources(SummaryField summaryField, SDField sdField) {
+ sdField.addSummaryFieldSources(summaryField);
+ }
+
+ private void collectSummaries(SDField field,Search search) {
+ SummaryField addedSummaryField=null;
+
+ // Implicit
+ String fieldName = field.getName();
+ SummaryField fieldSummaryField=field.getSummaryField(fieldName);
+ if (fieldSummaryField == null && field.doesSummarying()) {
+ fieldSummaryField=new SummaryField(fieldName, field.getDataType());
+ fieldSummaryField.setImplicit(true);
+ addSummaryFieldSources(fieldSummaryField, field);
+ fieldSummaryField.addDestination("default");
+ field.addSummaryField(fieldSummaryField);
+ addedSummaryField = fieldSummaryField;
+ }
+ if (fieldSummaryField != null) {
+ for (String dest : fieldSummaryField.getDestinations()) {
+ DocumentSummary summary = search.getSummary(dest);
+ if (summary != null) {
+ summary.add(fieldSummaryField);
+ }
+ }
+ }
+
+ // Attribute prefetch
+ for (Attribute attribute : field.getAttributes().values()) {
+ if (attribute.getName().equals(fieldName)) {
+ if (addedSummaryField != null) {
+ addedSummaryField.setTransform(SummaryTransform.ATTRIBUTE);
+ }
+ if (attribute.isPrefetch()) {
+ addPrefetchAttribute(attribute, field, search);
+ }
+ }
+ }
+
+ // Position attributes
+ if (field.doesSummarying()) {
+ for (Attribute attribute : field.getAttributes().values()) {
+ if (!attribute.isPosition()) {
+ continue;
+ }
+ DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(search);
+ attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)));
+ attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)));
+ }
+ }
+
+ // Explicits
+ for (SummaryField summaryField : field.getSummaryFields()) {
+ // Make sure we fetch from attribute here too
+ Attribute attribute=field.getAttributes().get(fieldName);
+ if (attribute!=null && summaryField.getTransform()==SummaryTransform.NONE) {
+ summaryField.setTransform(SummaryTransform.ATTRIBUTE);
+ }
+
+ if (isValid(summaryField,search)) {
+ addToDestinations(summaryField, search);
+ }
+ }
+
+ }
+
+ private DocumentSummary getOrCreateAttributePrefetchSummary(Search search) {
+ DocumentSummary summary = search.getSummary("attributeprefetch");
+ if (summary == null) {
+ summary = new DocumentSummary("attributeprefetch");
+ search.addSummary(summary);
+ }
+ return summary;
+ }
+
+
+ private void addPrefetchAttribute(Attribute attribute,SDField field,Search search) {
+ if (attribute.getPrefetchValue()==null) { // Prefetch by default - unless any summary makes this dynamic
+ // Check if there is an implicit dynamic definition
+ SummaryField fieldSummaryField=field.getSummaryField(attribute.getName());
+ if (fieldSummaryField!=null && fieldSummaryField.getTransform().isDynamic()) return;
+
+ // Check if an explicit class makes it dynamic (first is enough, as all must be the same, checked later)
+ SummaryField explicitSummaryField=search.getExplicitSummaryField(attribute.getName());
+ if (explicitSummaryField!=null && explicitSummaryField.getTransform().isDynamic()) return;
+ }
+
+ DocumentSummary summary = getOrCreateAttributePrefetchSummary(search);
+ SummaryField attributeSummaryField = new SummaryField(attribute.getName(), attribute.getDataType());
+ attributeSummaryField.addSource(attribute.getName());
+ attributeSummaryField.addDestination("attributeprefetch");
+ attributeSummaryField.setTransform(SummaryTransform.ATTRIBUTE);
+ summary.add(attributeSummaryField);
+ }
+
+ // Returns whether this is valid. Warns if invalid and ignorable. Throws if not ignorable.
+ private boolean isValid(SummaryField summaryField,Search search) {
+ if (summaryField.getTransform() == SummaryTransform.DISTANCE ||
+ summaryField.getTransform() == SummaryTransform.POSITIONS)
+ {
+ int sourceCount = summaryField.getSourceCount();
+ if (sourceCount != 1) {
+ throw newProcessException(search.getName(), summaryField.getName(),
+ "Expected 1 source field, got " + sourceCount + ".");
+ }
+ String sourceName = summaryField.getSingleSource();
+ if (search.getAttribute(sourceName) == null) {
+ throw newProcessException(search.getName(), summaryField.getName(),
+ "Summary source attribute '" + sourceName + "' not found.");
+ }
+ return true;
+ }
+
+ String fieldName = summaryField.getSourceField();
+ SDField sourceField = search.getField(fieldName);
+ if (sourceField == null) {
+ throw newProcessException(search, summaryField, "Source field '" + fieldName + "' does not exist.");
+ }
+ if (!sourceField.doesSummarying() &&
+ !summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) &&
+ !summaryField.getTransform().equals(SummaryTransform.GEOPOS))
+ {
+ // Summary transform attribute may indicate that the ilscript was rewritten to remove summary
+ // by another search that uses this same field in inheritance.
+ deployLogger.log(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField +
+ " is not creating a summary value in its indexing statement");
+ return false;
+ }
+
+ if (summaryField.getTransform().isDynamic()
+ && summaryField.getName().equals(sourceField.getName())
+ && sourceField.doesAttributing()) {
+ Attribute attribute=sourceField.getAttributes().get(sourceField.getName());
+ if (attribute!=null) {
+ String destinations="document summary 'default'";
+ if (summaryField.getDestinations().size()>0) {
+ destinations = "document summaries " + summaryField.getDestinations();
+ }
+ deployLogger.log(Level.WARNING, "Will fetch the disk summary value of " + sourceField + " in " + destinations +
+ " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " +
+ "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " +
+ "to this attribute, remove any bolding and dynamic snippeting from this field");
+ // Note: The dynamic setting has already overridden the attribute map setting,
+ // so we do not need to actually do attribute.setSummary(false) here
+ // Also, we can not do this, since it makes it impossible to fetch this attribute
+ // in another summary
+ }
+ }
+
+ return true;
+ }
+
+ private void addToDestinations(SummaryField summaryField,Search search) {
+ if (summaryField.getDestinations().size()==0) {
+ addToDestination("default",summaryField,search);
+ }
+ else {
+ for (String destinationName : summaryField.getDestinations())
+ addToDestination(destinationName,summaryField,search);
+ }
+ }
+
+ private void addToDestination(String destinationName,SummaryField summaryField,Search search) {
+ DocumentSummary destination=search.getSummary(destinationName);
+ if (destination==null) {
+ destination=new DocumentSummary(destinationName);
+ search.addSummary(destination);
+ destination.add(summaryField);
+ }
+ else {
+ SummaryField existingField=
+ destination.getSummaryField(summaryField.getName());
+ SummaryField merged=summaryField.mergeWith(existingField);
+ destination.add(merged);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
new file mode 100644
index 00000000000..b95f2469cd8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * This processor adds all implicit summary fields to all registered document summaries. If another field has already
+ * been registered with one of the implicit names, this processor will throw an {@link IllegalStateException}.
+ */
+public class ImplicitSummaryFields extends Processor {
+
+ public ImplicitSummaryFields(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (DocumentSummary docsum : search.getSummaries().values()) {
+ addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES));
+ addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES));
+ }
+ }
+
+ private void addField(DocumentSummary docsum, SummaryField field) {
+ if (docsum.getSummaryField(field.getName()) != null) {
+ throw new IllegalStateException("Summary class '" + docsum.getName() + "' uses reserved field name '" +
+ field.getName() + "'.");
+ }
+ docsum.add(field);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
new file mode 100644
index 00000000000..2c18c58d3dc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Because of the way the parser works (allowing any token as identifier),
+ * it is not practical to limit the syntax of field names there, do it here.
+ * Important to disallow dash, has semantic in IL.
+ * @author vegardh
+ *
+ */
+public class IndexFieldNames extends Processor {
+
+ private static final String FIELD_NAME_REGEXP = "[a-zA-Z]\\w*";
+
+ public IndexFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (!field.getName().matches(FIELD_NAME_REGEXP) && !legalDottedPositionField(field)) {
+ fail(search, field, " Not a legal field name. Legal expression: " + FIELD_NAME_REGEXP);
+ }
+ }
+ }
+
+ /**
+ * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. For 6.0, rename
+ * to _position and _distance and delete this method.
+ * @param field an {@link com.yahoo.searchdefinition.document.SDField}
+ * @return true if allowed
+ */
+ private boolean legalDottedPositionField(SDField field) {
+ return field.getName().endsWith(".position") || field.getName().endsWith(".distance");
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
new file mode 100644
index 00000000000..fecc5e2309d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+
+/**
+ * Fail if:
+ * 1) There are index: settings without explicit index names (name same as field name)
+ * 2) All the index-to indexes differ from the field name.
+ * @author vegardh
+ *
+ */
+public class IndexSettingsNonFieldNames extends Processor {
+
+ public IndexSettingsNonFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ boolean fieldNameUsed = false;
+ for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) {
+ String iName = (String)(i.next());
+ if (iName.equals(field.getName())) {
+ fieldNameUsed = true;
+ }
+ }
+ if (!fieldNameUsed) {
+ for (Index index : field.getIndices().values()) {
+ if (index.getName().equals(field.getName())) {
+ throw new IllegalArgumentException("Error in " + field + " in " + search +
+ ": When all index names differ from field name, index parameter settings must specify index name explicitly.");
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java
new file mode 100644
index 00000000000..3148273ddc1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * @author balder
+ */
+public class IndexTo2FieldSet extends Processor {
+
+ public IndexTo2FieldSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ processField(field);
+ }
+ }
+ private void processField(SDField field) {
+ if (field.usesStructOrMap()) {
+ for (SDField subField : field.getStructFields()) {
+ processField(subField);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
new file mode 100644
index 00000000000..c2e889f7993
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * <p>This processor modifies all indexing scripts so that they input the value of the owning field by default. It also
+ * ensures that all fields used as input exist.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexingInputs extends Processor {
+
+ public IndexingInputs(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ ScriptExpression script = field.getIndexingScript();
+ if (script == null) {
+ continue;
+ }
+ String fieldName = field.getName();
+ script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script);
+ script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script);
+ new VerifyInputExpression(search, field).visit(script);
+ field.setIndexingScript(script);
+ }
+ }
+
+ private static class DefaultToCurrentField extends ExpressionConverter {
+
+ final String fieldName;
+
+ DefaultToCurrentField(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ return exp instanceof InputExpression && ((InputExpression)exp).getFieldName() == null;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ return new InputExpression(fieldName);
+ }
+ }
+
+ private static class EnsureInputExpression extends ExpressionConverter {
+
+ final String fieldName;
+
+ EnsureInputExpression(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ return exp instanceof StatementExpression;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ if (exp.requiredInputType() != null) {
+ return new StatementExpression(new InputExpression(fieldName), exp);
+ } else {
+ return exp;
+ }
+ }
+ }
+
+ private class VerifyInputExpression extends ExpressionVisitor {
+
+ private final Search search;
+ private final SDField field;
+
+ public VerifyInputExpression(Search search, SDField field) {
+ this.search = search;
+ this.field = field;
+ }
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (!(exp instanceof InputExpression)) {
+ return;
+ }
+ SDDocumentType docType = search.getDocument();
+ String inputField = ((InputExpression)exp).getFieldName();
+ if (docType.getField(inputField) != null) {
+ return;
+ }
+ fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " +
+ "in document type '" + docType.getName() + "'.");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
new file mode 100644
index 00000000000..55e7dabf9ba
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.*;
+
+/**
+ * <p>This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents
+ * any output expression from writing to any field except for the owning field. Finally, for <tt>SummaryExpression</tt>,
+ * this processor expands to write all appropriate summary fields.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexingOutputs extends Processor {
+
+ public IndexingOutputs(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ ScriptExpression script = field.getIndexingScript();
+ if (script == null) {
+ continue;
+ }
+ Set<String> summaryFields = new TreeSet<>();
+ findSummaryTo(search, field, summaryFields, summaryFields);
+ MyConverter converter = new MyConverter(search, field, summaryFields);
+ field.setIndexingScript((ScriptExpression)converter.convert(script));
+ }
+ }
+
+ public void findSummaryTo(Search search, SDField field, Set<String> dynamicSummary,
+ Set<String> staticSummary) {
+ Map<String, SummaryField> summaryFields = search.getSummaryFields(field);
+ if (summaryFields.isEmpty()) {
+ fillSummaryToFromField(field, dynamicSummary, staticSummary);
+ } else {
+ fillSummaryToFromSearch(search, field, summaryFields, dynamicSummary, staticSummary);
+ }
+ }
+
+ private void fillSummaryToFromSearch(Search search, SDField field, Map<String, SummaryField> summaryFields,
+ Set<String> dynamicSummary, Set<String> staticSummary) {
+ for (SummaryField summaryField : summaryFields.values()) {
+ fillSummaryToFromSummaryField(search, field, summaryField, dynamicSummary, staticSummary);
+ }
+ }
+
+ private void fillSummaryToFromSummaryField(Search search, SDField field, SummaryField summaryField, Set<String> dynamicSummary, Set<String> staticSummary) {
+ SummaryTransform summaryTransform = summaryField.getTransform();
+ String summaryName = summaryField.getName();
+ if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) {
+ // Avoid writing to summary fields that have more than a single input field, as that is handled by the
+ // summary rewriter in the search core.
+ return;
+ }
+ if (summaryTransform.isDynamic()) {
+ DataType fieldType = field.getDataType();
+ if (fieldType != DataType.URI && fieldType != DataType.STRING) {
+ warn(search, field, "Dynamic summaries are only supported for fields of type " +
+ "string, ignoring summary field '" + summaryField.getName() +
+ "' for sd field '" + field.getName() + "' of type " +
+ fieldType.getName() + ".");
+ return;
+ }
+ dynamicSummary.add(summaryName);
+ } else if (summaryTransform != SummaryTransform.ATTRIBUTE) {
+ staticSummary.add(summaryName);
+ }
+ }
+
+ private static void fillSummaryToFromField(SDField field, Set<String> dynamicSummary, Set<String> staticSummary) {
+ for (SummaryField summaryField : field.getSummaryFields()) {
+ String summaryName = summaryField.getName();
+ if (summaryField.getTransform().isDynamic()) {
+ dynamicSummary.add(summaryName);
+ } else {
+ staticSummary.add(summaryName);
+ }
+ }
+ }
+
+ private class MyConverter extends ExpressionConverter {
+
+ final Search search;
+ final Field field;
+ final Set<String> summaryFields;
+
+ MyConverter(Search search, Field field, Set<String> summaryFields) {
+ this.search = search;
+ this.field = field;
+ this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ if (!(exp instanceof OutputExpression)) {
+ return false;
+ }
+ String fieldName = ((OutputExpression)exp).getFieldName();
+ if (fieldName == null) {
+ return true; // inject appropriate field name
+ }
+ if (!fieldName.equals(field.getName())) {
+ fail(search, field, "Indexing expression '" + exp + "' attempts to write to a field other than '" +
+ field.getName() + "'.");
+ }
+ return false;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ List<Expression> ret = new LinkedList<>();
+ if (exp instanceof AttributeExpression) {
+ ret.add(new AttributeExpression(field.getName()));
+ } else if (exp instanceof IndexExpression) {
+ ret.add(new IndexExpression(field.getName()));
+ } else if (exp instanceof SummaryExpression) {
+ for (String fieldName : summaryFields) {
+ ret.add(new SummaryExpression(fieldName));
+ }
+ } else {
+ throw new UnsupportedOperationException(exp.getClass().getName());
+ }
+ return new StatementExpression(ret);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
new file mode 100644
index 00000000000..af04deb5347
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexingValidation extends Processor {
+
+ public IndexingValidation(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ VerificationContext ctx = new VerificationContext(new MyAdapter(search));
+ for (SDField field : search.allFieldsList()) {
+ ScriptExpression script = field.getIndexingScript();
+ try {
+ script.verify(ctx);
+ MyConverter converter = new MyConverter();
+ for (StatementExpression exp : script) {
+ converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch
+ }
+ } catch (VerificationException e) {
+ fail(search, field, "For expression '" + e.getExpression() + "': " + e.getMessage());
+ }
+ }
+ }
+
+ private static class MyConverter extends ExpressionConverter {
+
+ final Set<String> outputs = new HashSet<>();
+ final Set<String> prevNames = new HashSet<>();
+
+ @Override
+ protected ExpressionConverter branch() {
+ MyConverter ret = new MyConverter();
+ ret.outputs.addAll(outputs);
+ ret.prevNames.addAll(prevNames);
+ return ret;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ if (exp instanceof OutputExpression) {
+ String fieldName = ((OutputExpression)exp).getFieldName();
+ if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) {
+ throw new VerificationException(exp, "Attempting to assign conflicting values to field '" +
+ fieldName + "'.");
+ }
+ outputs.add(fieldName);
+ prevNames.add(fieldName);
+ }
+ if (exp.createdOutputType() != null) {
+ prevNames.clear();
+ }
+ return false;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class MyAdapter implements FieldTypeAdapter {
+
+ final Search search;
+
+ public MyAdapter(Search search) {
+ this.search = search;
+ }
+
+ @Override
+ public DataType getInputType(Expression exp, String fieldName) {
+ SDField field = search.getDocumentField(fieldName);
+ if (field == null) {
+ throw new VerificationException(exp, "Input field '" + fieldName + "' not found.");
+ }
+ return field.getDataType();
+ }
+
+ @Override
+ public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+ String fieldDesc;
+ DataType fieldType;
+ if (exp instanceof AttributeExpression) {
+ Attribute attribute = search.getAttribute(fieldName);
+ if (attribute == null) {
+ throw new VerificationException(exp, "Attribute '" + fieldName + "' not found.");
+ }
+ fieldDesc = "attribute";
+ fieldType = attribute.getDataType();
+ } else if (exp instanceof IndexExpression) {
+ SDField field = search.getField(fieldName);
+ if (field == null) {
+ throw new VerificationException(exp, "Index field '" + fieldName + "' not found.");
+ }
+ fieldDesc = "index field";
+ fieldType = field.getDataType();
+ } else if (exp instanceof SummaryExpression) {
+ SummaryField field = search.getSummaryField(fieldName);
+ if (field == null) {
+ throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
+ }
+ fieldDesc = "summary field";
+ fieldType = field.getDataType();
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ if (!fieldType.isAssignableFrom(valueType) &&
+ !fieldType.isAssignableFrom(createCompatType(valueType)))
+ {
+ throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
+ " '" + fieldName + "' which is " + fieldType.getName() + ".");
+ }
+ }
+
+ private static DataType createCompatType(DataType origType) {
+ if (origType instanceof ArrayDataType) {
+ return DataType.getArray(createCompatType(((ArrayDataType)origType).getNestedType()));
+ } else if (origType instanceof MapDataType) {
+ MapDataType mapType = (MapDataType)origType;
+ return DataType.getMap(createCompatType(mapType.getKeyType()),
+ createCompatType(mapType.getValueType()));
+ } else if (origType instanceof WeightedSetDataType) {
+ return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType()));
+ } else if (origType == PositionDataType.INSTANCE) {
+ return DataType.LONG;
+ } else {
+ return origType;
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
new file mode 100644
index 00000000000..ff8be71f8c3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class IndexingValues extends Processor {
+
+ public IndexingValues(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (Field field : search.getDocument().fieldSet()) {
+ SDField sdField = (SDField)field;
+ if (!sdField.isExtraField()) {
+ new RequireThatDocumentFieldsAreImmutable(field).convert(sdField.getIndexingScript());
+ }
+ }
+ }
+
+ private class RequireThatDocumentFieldsAreImmutable extends ExpressionConverter {
+
+ final Field field;
+ Expression mutatedBy;
+
+ RequireThatDocumentFieldsAreImmutable(Field field) {
+ this.field = field;
+ }
+
+ @Override
+ public ExpressionConverter branch() {
+ return clone();
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ if (exp instanceof OutputExpression && mutatedBy != null) {
+ throw newProcessException(search, field,
+ "Indexing expression '" + mutatedBy + "' modifies the value of the " +
+ "document field '" + field.getName() + "'. This is no longer supported -- " +
+ "declare such fields outside the document.");
+ }
+ if (exp instanceof InputExpression && ((InputExpression)exp).getFieldName().equals(field.getName())) {
+ mutatedBy = null;
+ } else if (exp.createdOutputType() != null) {
+ mutatedBy = exp;
+ }
+ return false;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
new file mode 100644
index 00000000000..a29896ce3e8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Replaces the 'index' statement of all numerical fields to 'attribute' because we no longer support numerical indexes.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class IntegerIndex2Attribute extends Processor {
+
+ public IntegerIndex2Attribute(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) {
+ // Avoid changing for example RISE fields
+ if (field.getIndex(field.getName()) != null && !(field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue;
+ ScriptExpression script = field.getIndexingScript();
+ Set<String> attributeNames = new HashSet<>();
+ new MyVisitor(attributeNames).visit(script);
+ field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script));
+ warn(search, field, "Changed to attribute because numerical indexes (field has type "+field.getDataType().getName()+") is not currently supported." +
+ " Index-only settings may fail. Ignore this warning for streaming search.");
+ }
+ }
+ }
+
+ private static class MyVisitor extends ExpressionVisitor {
+
+ final Set<String> attributeNames;
+
+ public MyVisitor(Set<String> attributeNames) {
+ this.attributeNames = attributeNames;
+ }
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (exp instanceof AttributeExpression) {
+ attributeNames.add(((AttributeExpression)exp).getFieldName());
+ }
+ }
+ }
+
+ private static class MyConverter extends ExpressionConverter {
+
+ final Set<String> attributeNames;
+
+ public MyConverter(Set<String> attributeNames) {
+ this.attributeNames = attributeNames;
+ }
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ return exp instanceof IndexExpression;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ String indexName = ((IndexExpression)exp).getFieldName();
+ if (attributeNames.contains(indexName)) {
+ return null;
+ }
+ return new AttributeExpression(indexName);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
new file mode 100644
index 00000000000..e4e96bb51ab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+
+/**
+ * Expresses literal boosts in terms of extra indices with rank boost.
+ * One extra index named <i>indexname</i>_exact is added for each index having
+ * a fields with literal-boosts of zero or more (zero to support other
+ * rank profiles setting a literal boost). Complete boost values in to fields
+ * are translated to rank boosts to the implementation indices.
+ * These indices has no positional
+ * or phrase support and contains concatenated versions of each field value
+ * of complete-boosted fields indexed to <i>indexname</i>. A search for indexname
+ * will be rewritten to also search <i>indexname</i>_exaxt
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class LiteralBoost extends Processor {
+
+ public LiteralBoost(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ /** Adds extra search fields and indices to express literal boosts */
+ @Override
+ public void process() {
+ checkRankModifierRankType(search);
+ addLiteralBoostsToFields(search);
+ reduceFieldLiteralBoosts(search);
+ }
+
+ /** Checks if literal boost is given using rank: , and set the actual literal boost accordingly. */
+ private void checkRankModifierRankType(Search search) {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getLiteralBoost() > -1) continue; // Let explicit value take precedence
+ if (field.getRanking().isLiteral())
+ field.setLiteralBoost(100);
+ }
+ }
+
+ /**
+ * Ensures there are field boosts for all literal boosts mentioned in rank profiles.
+ * This is required because boost indices will only be generated by looking
+ * at field boosts
+ */
+ private void addLiteralBoostsToFields(Search search) {
+ Iterator i = matchingRankSettingsIterator(search, RankProfile.RankSetting.Type.LITERALBOOST);
+ while (i.hasNext()) {
+ RankProfile.RankSetting setting = (RankProfile.RankSetting)i.next();
+ SDField field = search.getField(setting.getFieldName());
+ if (field == null) continue;
+ if (field.getLiteralBoost() < 0)
+ field.setLiteralBoost(0);
+ }
+ }
+
+ private void reduceFieldLiteralBoosts(Search search) {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getLiteralBoost()<0) continue;
+ reduceFieldLiteralBoost(field,search);
+ }
+ }
+
+ private void reduceFieldLiteralBoost(SDField field,Search search) {
+ SDField literalField = addField(search, field, "literal",
+ "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }",
+ "literal-boost");
+ literalField.setWeight(field.getWeight() + field.getLiteralBoost());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
new file mode 100644
index 00000000000..856b0cf3348
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Takes the aliases set on field by parser and sets them on correct Index or Attribute
+ * @author vegardh
+ *
+ */
+public class MakeAliases extends Processor {
+
+ public MakeAliases(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ List<String> usedAliases = new ArrayList<>();
+ for (SDField field : search.allFieldsList()) {
+ for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
+ String alias = e.getKey();
+ String name = e.getValue();
+ String errMsg = "For search '"+search.getName()+"': alias '"+alias+"' ";
+ if (search.existsIndex(alias)) {
+ throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an index.");
+ }
+ if (search.getAttribute(alias)!=null) {
+ throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an attribute.");
+ }
+ if (usedAliases.contains(alias)) {
+ throw new IllegalArgumentException(errMsg+"specified more than once.");
+ }
+ usedAliases.add(alias);
+
+ Index index = field.getIndex(name);
+ Attribute attribute = field.getAttributes().get(name);
+ if (index != null) {
+ index.addAlias(alias); // alias will be for index in this case, since it is the one used in a search
+ } else if (attribute != null && !field.doesIndexing()) {
+ attribute.getAliases().add(alias);
+ } else {
+ index = new Index(name);
+ index.addAlias(alias);
+ field.addIndex(index);
+ }
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
new file mode 100644
index 00000000000..a9f013daa98
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * <p>All summary fields which are not attributes
+ * must currently be present in the default summary class,
+ * since the default summary class also defines the docsum.dat format.
+ * This processor adds any missing summaries to the default summary.
+ * When that is decoupled from the actual summaries returned, this
+ * processor can be removed. Note: the StreamingSummary also takes advantage of
+ * the fact that default is the superset.</p>
+ *
+ * <p>All other summary logic should work unchanged without this processing step
+ * except that IndexStructureValidator.validateSummaryFields must be changed to
+ * consider all summaries, not just the default, i.e change to
+ * if (search.getSummaryField(expr.getFieldName()) == null)</p>
+ *
+ * <p>This must be done after other summary processors.</p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class MakeDefaultSummaryTheSuperSet extends Processor {
+
+ public MakeDefaultSummaryTheSuperSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ DocumentSummary defaultSummary=search.getSummary("default");
+ for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) {
+ if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue;
+ if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue;
+
+ defaultSummary.add(summaryField.clone());
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
new file mode 100644
index 00000000000..f21670e6f78
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.Matching.Type;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Warn on inconsistent match settings for any index
+ *
+ * @author vegardh
+ */
+public class MatchConsistency extends Processor {
+
+ public MatchConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ Map<String, Matching.Type> types = new HashMap<>();
+ for (SDField field : search.allFieldsList()) {
+ new MyVisitor(search, field, types).visit(field.getIndexingScript());
+ }
+ }
+
+ void checkMatching(Search search, SDField field, Map<String, Type> types, String indexTo) {
+ Type prevType = types.get(indexTo);
+ if (prevType == null) {
+ types.put(indexTo, field.getMatching().getType());
+ } else if (!field.getMatching().getType().equals(prevType)) {
+ warn(search, field, "The matching type for index '" + indexTo + "' (got " + field.getMatching().getType() +
+ ") is inconsistent with that given for the same index in a previous field (had " +
+ prevType + ").");
+ }
+ }
+
+ private class MyVisitor extends ExpressionVisitor {
+
+ final Search search;
+ final SDField field;
+ final Map<String, Type> types;
+
+ public MyVisitor(Search search, SDField field, Map<String, Type> types) {
+ this.search = search;
+ this.field = field;
+ this.types = types;
+ }
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (exp instanceof IndexExpression) {
+ checkMatching(search, field, types, ((IndexExpression)exp).getFieldName());
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
new file mode 100644
index 00000000000..efceeca0af0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Validates the match phase settings for all registered rank profiles.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class MatchPhaseSettingsValidator extends Processor {
+
+ public MatchPhaseSettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) {
+ RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings();
+ if (settings != null) {
+ validateMatchPhaseSettings(rankProfile, settings);
+ }
+ }
+ }
+
+ private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) {
+ String attributeName = settings.getAttribute();
+ new AttributeValidator(search.getName(), rankProfile.getName(),
+ search.getAttribute(attributeName), attributeName).validate();
+ }
+
+ public static class AttributeValidator {
+
+ private final String searchName;
+ private final String rankProfileName;
+ protected final Attribute attribute;
+ private final String attributeName;
+
+ public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) {
+ this.searchName = searchName;
+ this.rankProfileName = rankProfileName;
+ this.attribute = attribute;
+ this.attributeName = attributeName;
+ }
+
+ public void validate() {
+ validateThatAttributeExists();
+ validateThatAttributeIsSingleNumeric();
+ validateThatAttributeIsFastSearch();
+ }
+
+ protected void validateThatAttributeExists() {
+ if (attribute == null) {
+ failValidation("does not exists");
+ }
+ }
+
+ protected void validateThatAttributeIsSingleNumeric() {
+ if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) ||
+ attribute.getType().equals(Attribute.Type.STRING) ||
+ attribute.getType().equals(Attribute.Type.PREDICATE))
+ {
+ failValidation("must be single value numeric, but it is '"
+ + attribute.getDataType().getName() + "'");
+ }
+ }
+
+ protected void validateThatAttributeIsFastSearch() {
+ if (!attribute.isFastSearch()) {
+ failValidation("must be fast-search, but it is not");
+ }
+ }
+
+ protected void failValidation(String what) {
+ throw new IllegalArgumentException(createMessagePrefix() + what);
+ }
+
+ public String getValidationType() { return "match-phase"; }
+
+ private String createMessagePrefix() {
+ return "In search definition '" + searchName +
+ "', rank-profile '" + rankProfileName +
+ "': " + getValidationType() + " attribute '" + attributeName + "' ";
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
new file mode 100644
index 00000000000..61a11fd3a4e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.processing.multifieldresolver.IndexCommandResolver;
+import com.yahoo.searchdefinition.processing.multifieldresolver.RankTypeResolver;
+import com.yahoo.searchdefinition.processing.multifieldresolver.StemmingResolver;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Ensures that there are no conflicting types or field settings
+ * in multifield indices, either by changing settings or by splitting
+ * conflicting fields in multiple ones with different settings.
+ *
+ * @author bratseth
+ */
+public class MultifieldIndexHarmonizer extends Processor {
+
+ /** A map from index names to a List of fields going to that index */
+ private Map<String,List<SDField>> indexToFields=new java.util.HashMap<>();
+
+ public MultifieldIndexHarmonizer(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ populateIndexToFields(search);
+ resolveAllConflicts(search);
+ }
+
+ private void populateIndexToFields(Search search) {
+ for (SDField field : search.allFieldsList() ) {
+ if (!field.doesIndexing()) {
+ continue;
+ }
+ for (Iterator j = field.getFieldNameAsIterator(); j.hasNext();) {
+ String indexName = (String)j.next();
+ addIndexField(indexName, field);
+ }
+ }
+ }
+
+ private void addIndexField(String indexName,SDField field) {
+ List<SDField> fields=indexToFields.get(indexName);
+ if (fields==null) {
+ fields=new java.util.ArrayList<>();
+ indexToFields.put(indexName,fields);
+ }
+ fields.add(field);
+ }
+
+ private void resolveAllConflicts(Search search) {
+ for (Map.Entry<String, List<SDField>> entry : indexToFields.entrySet()) {
+ String indexName = entry.getKey();
+ List<SDField> fields = entry.getValue();
+ if (fields.size() == 1) continue; // It takes two to make a conflict
+ resolveConflicts(indexName, fields, search);
+ }
+ }
+
+ /**
+ * Resolves all conflicts for one index
+ *
+ * @param indexName the name of the index in question
+ * @param fields all the fields indexed to this index
+ * @param search the search definition having this
+ */
+ private void resolveConflicts(String indexName,List<SDField> fields,Search search) {
+ new StemmingResolver(indexName, fields, search, deployLogger).resolve();
+ new IndexCommandResolver(indexName, fields, search, deployLogger).resolve();
+ new RankTypeResolver(indexName, fields, search, deployLogger).resolve();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
new file mode 100644
index 00000000000..8e5122cc158
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of "gram" matching - splitting the incoming text and the queries into
+ * n-grams for matching. This will also validate the gram settings.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class NGramMatch extends Processor {
+
+ public static final int DEFAULT_GRAM_SIZE = 2;
+
+ public NGramMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getMatching().getType().equals(Matching.Type.GRAM))
+ implementGramMatch(search, field);
+ else if (field.getMatching().getGramSize()>=0)
+ throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'");
+ }
+ }
+
+ private void implementGramMatch(Search search, SDField field) {
+ if (field.doesAttributing())
+ throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' not 'attribute' in indexing");
+
+ int n = field.getMatching().getGramSize();
+ if (n<0)
+ n=DEFAULT_GRAM_SIZE; // not set - use default gram size
+ if (n==0)
+ throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1");
+ field.getNormalizing().inferCodepoint();
+ field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing
+ field.addQueryCommand("ngram " + n);
+ field.setIndexingScript((ScriptExpression)new MyProvider(search, n).convert(field.getIndexingScript()));
+ }
+
+ private static class MyProvider extends TypedTransformProvider {
+
+ final int ngram;
+
+ MyProvider(Search search, int ngram) {
+ super(NGramExpression.class, search);
+ this.ngram = ngram;
+ }
+
+ @Override
+ protected boolean requiresTransform(Expression exp, DataType fieldType) {
+ return exp instanceof OutputExpression;
+ }
+
+ @Override
+ protected Expression newTransform(DataType fieldType) {
+ Expression exp = new NGramExpression(null, ngram);
+ if (fieldType instanceof CollectionDataType) {
+ exp = new ForEachExpression(exp);
+ }
+ return exp;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
new file mode 100644
index 00000000000..078e9400ec5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionOptimizer;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Run ExpressionOptimizer on all scripts, to get rid of expressions that have no effect.
+ */
+public class OptimizeIlscript extends Processor {
+ public OptimizeIlscript(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ ScriptExpression script = field.getIndexingScript();
+ if (script == null) {
+ continue;
+ }
+ field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script));
+ if (!field.getIndexingScript().toString().equals(script.toString())) {
+ warn(search, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString());
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
new file mode 100644
index 00000000000..a156a98a584
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Validates the predicate fields.
+ *
+ * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a>
+ */
+public class PredicateProcessor extends Processor {
+
+ public PredicateProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getDataType() == DataType.PREDICATE) {
+ if (field.doesIndexing()) {
+ fail(search, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded.");
+ }
+ if (field.doesAttributing()) {
+ Attribute attribute = field.getAttributes().get(field.getName());
+ for (Index index : field.getIndices().values()) {
+ BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton();
+ if (booleanDefinition == null || !booleanDefinition.hasArity()) {
+ fail(search, field, "Missing arity value in predicate field.");
+ }
+ if (booleanDefinition.getArity() < 2) {
+ fail(search, field, "Invalid arity value in predicate field, must be greater than 1.");
+ }
+ double threshold = booleanDefinition.getDensePostingListThreshold();
+ if (threshold <= 0 || threshold > 1) {
+ fail(search, field, "Invalid dense-posting-list-threshold value in predicate field. " +
+ "Value must be in range (0..1].");
+ }
+
+ attribute.setArity(booleanDefinition.getArity());
+ attribute.setLowerBound(booleanDefinition.getLowerBound());
+ attribute.setUpperBound(booleanDefinition.getUpperBound());
+
+ attribute.setDensePostingListThreshold(threshold);
+ addPredicateOptimizationIlScript(field, booleanDefinition);
+ }
+ DocumentSummary summary = search.getSummary("attributeprefetch");
+ if (summary != null) {
+ summary.remove(attribute.getName());
+ }
+ for (SummaryField summaryField : search.getSummaryFields(field).values()) {
+ summaryField.setTransform(SummaryTransform.NONE);
+ }
+ }
+ } else if (field.getDataType().getPrimitiveType() == DataType.PREDICATE) {
+ fail(search, field, "Collections of predicates are not allowed.");
+ } else if (field.getDataType() == DataType.RAW && field.doesIndexing()) {
+ fail(search, field, "Indexing of RAW fields are not supported. If you are using RAW fields for boolean search, use predicate data type instead.");
+ } else {
+ // if field is not a predicate, disallow predicate-related index parameters
+ for (Index index : field.getIndices().values()) {
+ if (index.getBooleanIndexDefiniton() != null) {
+ BooleanIndexDefinition def = index.getBooleanIndexDefiniton();
+ if (def.hasArity()) {
+ fail(search, field, "Arity parameter is used only for predicate type fields.");
+ } else if (def.hasLowerBound() || def.hasUpperBound()) {
+ fail(search, field, "Parameters lower-bound and upper-bound are used only for predicate type fields.");
+ } else if (def.hasDensePostingListThreshold()) {
+ fail(search, field, "Parameter dense-posting-list-threshold is used only for predicate type fields.");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void addPredicateOptimizationIlScript(SDField field, BooleanIndexDefinition booleanIndexDefiniton) {
+ Expression script = field.getIndexingScript();
+ if (script == null) {
+ return;
+ }
+ script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script);
+
+ ExpressionConverter converter = new PredicateOutputTransformer(search);
+ field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script)));
+ }
+
+ private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) {
+ List<Expression> expressions = new ArrayList<>();
+ expressions.add(new SetValueExpression(new IntegerFieldValue(options.getArity())));
+ expressions.add(new SetVarExpression("arity"));
+ if (options.hasLowerBound()) {
+ expressions.add(new SetValueExpression(new LongFieldValue(options.getLowerBound())));
+ expressions.add(new SetVarExpression("lower_bound"));
+ }
+ if (options.hasUpperBound()) {
+ expressions.add(new SetValueExpression(new LongFieldValue(options.getUpperBound())));
+ expressions.add(new SetVarExpression("upper_bound"));
+ }
+ return new StatementExpression(expressions);
+ }
+
+ private static class PredicateOutputTransformer extends TypedTransformProvider {
+
+ PredicateOutputTransformer(Search search) {
+ super(OptimizePredicateExpression.class, search);
+ }
+
+ @Override
+ protected boolean requiresTransform(Expression exp, DataType fieldType) {
+ return exp instanceof OutputExpression && fieldType == DataType.PREDICATE;
+ }
+
+ @Override
+ protected Expression newTransform(DataType fieldType) {
+ return new OptimizePredicateExpression();
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
new file mode 100644
index 00000000000..74f359602bd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Executor of processors. This defines the right order of processor execution.
+ *
+ * @author bratseth
+ */
+public class Processing {
+
+ /**
+ * Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly
+ * exchanging it with another</b>, as well as its document types.
+ *
+ * @param search The search to process.
+ * @param deployLogger The log to log messages and warnings for application deployment to
+ * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
+ * @param queryProfiles The query profiles contained in the application this search is part of.
+ */
+ public static void process(Search search,
+ DeployLogger deployLogger,
+ RankProfileRegistry rankProfileRegistry,
+ QueryProfiles queryProfiles) {
+ search.process();
+ new UrlFieldValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new BuiltInFieldSets(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SearchMustHaveDocument(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new ReservedDocumentNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IndexFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IntegerIndex2Attribute(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new MakeAliases(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SetLanguage(search, deployLogger, rankProfileRegistry, queryProfiles).process(); // Needs to come before UriHack, see ticket 6405470
+ new UriHack(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new LiteralBoost(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IndexTo2FieldSet(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new TagType(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IndexingInputs(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new OptimizeIlscript(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new ValidateFieldWithIndexSettingsCreatesIndex(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new AttributesImplicitWord(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new CreatePositionZCurve(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new WordMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new DeprecateAttributePrefetch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new ImplicitSummaries(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new ImplicitSummaryFields(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SummaryConsistency(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SummaryNamesFieldCollisions(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new MakeDefaultSummaryTheSuperSet(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new Bolding(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new AttributeProperties(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SetRankTypeEmptyOnFilters(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IndexSettingsNonFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SummaryDynamicStructsArrays(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new StringSettingsOnNonStringFields(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new ExactMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new NGramMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new TextMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new MultifieldIndexHarmonizer(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new FilterFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new MatchConsistency(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new ValidateFieldTypes(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new DisallowComplexMapAndWsetKeyTypes(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new SortingSettings(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new FieldSetValidity(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new AddExtraFieldsToDocument(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new PredicateProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new MatchPhaseSettingsValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new DiversitySettingsValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new TensorFieldProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new RankProfileTypeSettingsProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+
+ // These two should be last.
+ new IndexingValidation(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ new IndexingValues(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
new file mode 100644
index 00000000000..a3e0484b2fd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Abstract superclass of all search definition processors.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public abstract class Processor {
+
+ protected final Search search;
+ protected DeployLogger deployLogger;
+ protected final RankProfileRegistry rankProfileRegistry;
+ protected final QueryProfiles queryProfiles;
+
+ /**
+ * Base constructor
+ * @param search the search to process
+ * @param deployLogger Logger du use when logging deploy output.
+ * @param rankProfileRegistry Registry with all rank profiles, used for lookup and insertion.
+ * @param queryProfiles The query profiles contained in the application this search is part of.
+ */
+ public Processor(Search search, DeployLogger deployLogger,
+ RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ this.search = search;
+ this.deployLogger = deployLogger;
+ this.rankProfileRegistry = rankProfileRegistry;
+ this.queryProfiles = queryProfiles;
+ }
+
+ /**
+ * Processes the input search definition by <b>modifying</b> the input search and its documents, and returns the
+ * input search definition.
+ */
+ public abstract void process();
+
+ /**
+ * Convenience method for adding a no-strings-attached implementation field for a regular field
+ *
+ * @param search the search definition in question
+ * @param field the field to add an implementation field for
+ * @param suffix the suffix of the added implementation field (without the underscore)
+ * @param indexing the indexing statement of the field
+ * @param queryCommand the query command of the original field, or null if none
+ * @return the implementation field which is added to the search
+ */
+ protected SDField addField(Search search, SDField field, String suffix, String indexing, String queryCommand) {
+ SDField implementationField = search.getField(field.getName() + "_" + suffix);
+ if (implementationField != null) {
+ deployLogger.log(Level.WARNING, "Implementation field " + implementationField + " added twice");
+ } else {
+ implementationField = new SDField(search.getDocument(), field.getName() + "_" + suffix, DataType.STRING);
+ }
+ implementationField.setRankType(RankType.EMPTY);
+ implementationField.setStemming(Stemming.NONE);
+ implementationField.getNormalizing().inferCodepoint();
+ implementationField.parseIndexingScript(indexing);
+ for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) {
+ String indexName = (String)i.next();
+ String implementationIndexName = indexName + "_" + suffix;
+ Index implementationIndex = new Index(implementationIndexName);
+ search.addIndex(implementationIndex);
+ }
+ if (queryCommand != null) {
+ field.addQueryCommand(queryCommand);
+ }
+ search.addExtraField(implementationField);
+ search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, implementationField.getName());
+ return implementationField;
+ }
+
+ /**
+ * Returns an iterator of all the rank settings with given type in all the rank profiles in this search
+ * definition.
+ */
+ protected Iterator<RankProfile.RankSetting> matchingRankSettingsIterator(
+ Search search, RankProfile.RankSetting.Type type)
+ {
+ List<RankProfile.RankSetting> someRankSettings = new java.util.ArrayList<>();
+
+ for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) {
+ for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) {
+ RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next();
+ if (setting.getType().equals(type)) {
+ someRankSettings.add(setting);
+ }
+ }
+ }
+ return someRankSettings.iterator();
+ }
+
+ protected String formatError(String searchName, String fieldName, String msg) {
+ return "For search '" + searchName + "', field '" + fieldName + "': " + msg;
+ }
+
+ protected RuntimeException newProcessException(String searchName, String fieldName, String msg) {
+ return new IllegalArgumentException(formatError(searchName, fieldName, msg));
+ }
+
+ protected RuntimeException newProcessException(Search search, Field field, String msg) {
+ return newProcessException(search.getName(), field.getName(), msg);
+ }
+
+ public void fail(Search search, Field field, String msg) {
+ throw newProcessException(search, field, msg);
+ }
+
+ protected void warn(String searchName, String fieldName, String msg) {
+ String fullMsg = formatError(searchName, fieldName, msg);
+ deployLogger.log(Level.WARNING, fullMsg);
+ }
+
+ protected void warn(Search search, Field field, String msg) {
+ warn(search.getName(), field.getName(), msg);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
new file mode 100644
index 00000000000..74bf2d35bac
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ReservedDocumentNames extends Processor {
+
+ private static final Set<String> RESERVED_NAMES = new HashSet<>();
+ static {
+ for (SDDocumentType dataType : SDDocumentType.VESPA_DOCUMENT.getTypes()) {
+ RESERVED_NAMES.add(dataType.getName());
+ }
+ }
+
+ public ReservedDocumentNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ String docName = search.getDocument().getName();
+ if (RESERVED_NAMES.contains(docName)) {
+ throw new IllegalArgumentException("For search '" + search.getName() + "': Document name '" + docName +
+ "' is reserved.");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
new file mode 100644
index 00000000000..5b45aecc257
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * A search must have a document definition of the same name inside of it, otherwise crashes may occur as late as
+ * during feeding
+ * @author vegardh
+ *
+ */
+public class SearchMustHaveDocument extends Processor {
+
+ public SearchMustHaveDocument(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ if (search.getDocument()==null) {
+ throw new IllegalArgumentException("For search '" + search.getName() + "': A search specification must have an equally named document inside of it.");
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
new file mode 100644
index 00000000000..764bb1602a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Check that no text field appears before a field that sets language.
+ *
+ * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ */
+public class SetLanguage extends Processor {
+
+ public SetLanguage(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ List<String> textFieldsWithoutLanguage = new ArrayList<>();
+
+ for (SDField field : search.allFieldsList()) {
+ if (fieldMustComeAfterLanguageSettingField(field)) {
+ textFieldsWithoutLanguage.add(field.getName());
+ }
+ if (field.containsExpression(SetLanguageExpression.class) && !textFieldsWithoutLanguage.isEmpty()) {
+ StringBuffer fieldString = new StringBuffer();
+ for (String fieldName : textFieldsWithoutLanguage) {
+ fieldString.append(fieldName).append(" ");
+ }
+ warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " +
+ "and should be defined as the first field in the searchdefinition. If you have both header and body fields, this field "+
+ "should be header, if you require it to affect subsequent header fields and/or any body fields. " +
+ "Preceding text fields that will not have their language set: " +
+ fieldString.toString() +
+ " (This warning is omitted for any subsequent fields that also do set_language.)");
+ return;
+ }
+ }
+ }
+
+ private boolean fieldMustComeAfterLanguageSettingField(SDField field) {
+ return (!field.containsExpression(SetLanguageExpression.class) &&
+ (field.getDataType() == DataType.STRING));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
new file mode 100644
index 00000000000..bc676b64a71
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * All rank: filter fields should have rank type empty.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SetRankTypeEmptyOnFilters extends Processor {
+
+ public SetRankTypeEmptyOnFilters(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getRanking().isFilter()) {
+ field.setRankType(RankType.EMPTY);
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
new file mode 100644
index 00000000000..6e77f48b4a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Validate conflicting settings for sorting
+ * @author vegardh
+ *
+ */
+public class SortingSettings extends Processor {
+
+ public SortingSettings(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ for (Attribute attribute : field.getAttributes().values()) {
+ Sorting sorting = attribute.getSorting();
+ if (sorting.getFunction()!=Sorting.Function.UCA) {
+ if (sorting.getStrength()!=null && sorting.getStrength()!=Sorting.Strength.PRIMARY) {
+ warn(search, field, "Sort strength only works for sort function 'uca'.");
+ }
+ if (sorting.getLocale()!=null && !"".equals(sorting.getLocale())) {
+ warn(search, field, "Sort locale only works for sort function 'uca'.");
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
new file mode 100644
index 00000000000..48c9af5556a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+public class StringSettingsOnNonStringFields extends Processor {
+
+ public StringSettingsOnNonStringFields(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (!doCheck(field)) continue;
+ if (field.getMatching().isTypeUserSet()) {
+ warn(search, field, "Matching type "+field.getMatching().getType()+" is only allowed for string fields.");
+ }
+ if (field.getRanking().isLiteral()) {
+ warn(search, field, "Rank type literal only applies to string fields");
+ }
+ }
+ }
+
+ private boolean doCheck(SDField field) {
+ if (field.getDataType() instanceof NumericDataType) return true;
+ if (field.getDataType() instanceof CollectionDataType) {
+ if (((CollectionDataType)field.getDataType()).getNestedType() instanceof NumericDataType) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
new file mode 100644
index 00000000000..20eeb5f0810
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Ensure that summary field transforms for fields having the same name
+ * are consistent across summary classes
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SummaryConsistency extends Processor {
+
+ public SummaryConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (DocumentSummary summary : search.getSummaries().values()) {
+ if (summary.getName().equals("default")) continue;
+ for (SummaryField summaryField : summary.getSummaryFields() ) {
+ assertConsistency(summaryField,search);
+ makeAttributeTransformIfAppropriate(summaryField,search);
+ }
+ }
+ }
+
+ /** If the source is an attribute, make this use the attribute transform */
+ private void makeAttributeTransformIfAppropriate(SummaryField summaryField,Search search) {
+ if (summaryField.getTransform()!=SummaryTransform.NONE) return;
+ Attribute attribute=search.getAttribute(summaryField.getSingleSource());
+ if (attribute==null) return;
+ summaryField.setTransform(SummaryTransform.ATTRIBUTE);
+ }
+
+ private void assertConsistency(SummaryField summaryField,Search search) {
+ SummaryField existingDefault=search.getSummary("default").getSummaryField(summaryField.getName()); // Compare to default
+ if (existingDefault!=null) {
+ assertConsistentTypes(existingDefault,summaryField);
+ makeConsistentWithDefaultOrThrow(existingDefault,summaryField);
+ }
+ else {
+ // If no default, compare to whichever definition of the field
+ SummaryField existing=search.getExplicitSummaryField(summaryField.getName());
+ if (existing==null) return;
+ assertConsistentTypes(existing,summaryField);
+ makeConsistentOrThrow(existing,summaryField,search);
+ }
+ }
+
+ private void assertConsistentTypes(SummaryField field1,SummaryField field2) {
+ if (field1.getDataType() instanceof WeightedSetDataType && field2.getDataType() instanceof WeightedSetDataType &&
+ ((WeightedSetDataType)field1.getDataType()).getNestedType().equals(((WeightedSetDataType)field2.getDataType()).getNestedType()))
+ return; // Disregard create-if-nonexistent and create-if-zero distinction
+ if ( ! field1.getDataType().equals(field2.getDataType()))
+ throw new IllegalArgumentException(field1.toLocateString() + " is inconsistent with " + field2.toLocateString() + ": All declarations of the same summary field must have the same type");
+ }
+
+ private void makeConsistentOrThrow(SummaryField field1, SummaryField field2,Search search) {
+ if (field2.getTransform()==SummaryTransform.ATTRIBUTE && field1.getTransform()==SummaryTransform.NONE) {
+ Attribute attribute=search.getAttribute(field1.getName());
+ if (attribute != null) {
+ field1.setTransform(SummaryTransform.ATTRIBUTE);
+ }
+ }
+
+ if (field2.getTransform().equals(SummaryTransform.NONE)) {
+ field2.setTransform(field1.getTransform());
+ }
+ else { // New field sets an explicit transform - must be the same
+ assertEqualTransform(field1,field2);
+ }
+ }
+ private void makeConsistentWithDefaultOrThrow(SummaryField defaultField,SummaryField newField) {
+ if (newField.getTransform().equals(SummaryTransform.NONE)) {
+ newField.setTransform(defaultField.getTransform());
+ }
+ else { // New field sets an explicit transform - must be the same
+ assertEqualTransform(defaultField,newField);
+ }
+ }
+
+
+ private void assertEqualTransform(SummaryField field1,SummaryField field2) {
+ if (!field2.getTransform().equals(field1.getTransform())) {
+ throw new IllegalArgumentException("Conflicting summary transforms. " + field2 +" is already defined as " +
+ field1 + ". A field with the same name " +
+ "can not have different transforms in different summary classes");
+ }
+ }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
new file mode 100644
index 00000000000..f97d5345cab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Fail if:
+ * An SD field explicitly says summary:dynamic , but the field is wset, array or struct.
+ * If there is an explicitly defined summary class, saying dynamic in one of its summary
+ * fields is always legal.
+ * @author vegardh
+ *
+ */
+public class SummaryDynamicStructsArrays extends Processor {
+
+ public SummaryDynamicStructsArrays(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ DataType type = field.getDataType();
+ if (type instanceof ArrayDataType || type instanceof WeightedSetDataType
+ || type instanceof StructDataType) {
+ for (SummaryField sField : field.getSummaryFields()) {
+ if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) {
+ throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " +
+ "for fields of type struct, array or weighted set. Use an explicit summary class with explicit summary fields sourcing from" +
+ " the array/struct/weighted set.");
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
new file mode 100644
index 00000000000..0d76ada0d52
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Verifies that the source fields actually refers to a valid field.
+ *
+ * @author balder
+ *
+ */
+public class SummaryFieldsMustHaveValidSource extends Processor {
+ SummaryFieldsMustHaveValidSource(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+ @Override
+ public void process() {
+ for (DocumentSummary summary : search.getSummaries().values()) {
+ for (SummaryField summaryField : summary.getSummaryFields()) {
+ if (summaryField.getSources().isEmpty()) {
+ if ((summaryField.getTransform() != SummaryTransform.RANKFEATURES) &&
+ (summaryField.getTransform() != SummaryTransform.SUMMARYFEATURES))
+ {
+ verifySource(summaryField.getName(), summaryField, summary);
+ }
+ } else if (summaryField.getSourceCount() == 1) {
+ verifySource(summaryField.getSingleSource(), summaryField, summary);
+ } else {
+ for (SummaryField.Source source : summaryField.getSources()) {
+ if ( ! source.getName().equals(summaryField.getName()) ) {
+ verifySource(source.getName(), summaryField, summary);
+ }
+ }
+ }
+ }
+ }
+
+ }
+
+ private boolean isValid(String source, SummaryField summaryField, DocumentSummary summary) {
+ return isDocumentField(source) ||
+ (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) ||
+ (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) ||
+ ("documentid".equals(source));
+ }
+
+ private void verifySource(String source, SummaryField summaryField, DocumentSummary summary) {
+ if ( ! isValid(source, summaryField, summary) ) {
+ throw new IllegalArgumentException("For search '" + search.getName() + "', summary class '" + summary.getName() + "'," +
+ " summary field '" + summaryField.getName() + "': there is no valid source '" + source + "'.");
+ }
+ }
+
+ private boolean isNotInThisSummaryClass(DocumentSummary summary, String name) {
+ return summary.getSummaryField(name) == null;
+ }
+ private boolean isInThisSummaryClass(DocumentSummary summary, String name) {
+ return summary.getSummaryField(name) != null;
+ }
+ private boolean isDocumentField(String name) {
+ return search.getField(name) != null;
+ }
+
+ private boolean isSummaryField(String name) {
+ return search.getSummaryField(name) != null;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
new file mode 100644
index 00000000000..8d99611d697
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryField.Source;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Verifies that equally named summary fields in different summary classes don't use different fields for source.
+ * The summarymap config doesn't model this.
+ *
+ * @author vegardh
+ *
+ */
+public class SummaryNamesFieldCollisions extends Processor {
+
+ public SummaryNamesFieldCollisions(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>();
+ for (DocumentSummary summary : search.getSummaries().values()) {
+ if ("default".equals(summary.getName())) continue;
+ for (SummaryField summaryField : summary.getSummaryFields() ) {
+ if (summaryField.isImplicit()) continue;
+ Pair<String, String> prevClassAndSource = fieldToClassAndSource.get(summaryField.getName());
+ for (Source source : summaryField.getSources()) {
+ if (prevClassAndSource!=null) {
+ String prevClass = prevClassAndSource.getFirst();
+ String prevSource = prevClassAndSource.getSecond();
+ if (!prevClass.equals(summary.getName())) {
+ if (!prevSource.equals(source.getName())) {
+ throw new IllegalArgumentException("For search '"+search.getName()+"', summary class '"+summary.getName()+"'," +
+ " summary field '"+summaryField.getName()+"':" +
+ " Can not use source '"+source.getName()+"' for this summary field, an equally named field in summary class '" +
+ prevClass + "' uses a different source: '"+prevSource+"'.");
+ }
+ }
+ } else {
+ fieldToClassAndSource.put(summaryField.getName(), new Pair<>(summary.getName(), source.getName()));
+ }
+ }
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
new file mode 100644
index 00000000000..3e211f4f632
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of the tag datatype
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class TagType extends Processor {
+
+ public TagType(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) {
+ implementTagType(field);
+ }
+ }
+ }
+
+ private void implementTagType(SDField field) {
+ field.setDataType(DataType.getWeightedSet(DataType.STRING,true,true));
+ // Don't set matching and ranking if this field is not attribute nor index
+ if (!field.doesIndexing() && !field.doesAttributing()) return;
+ Matching m = field.getMatching();
+ if (!m.isTypeUserSet()) {
+ m.setType(Matching.Type.WORD);
+ }
+ if (field.getRankType()==null || field.getRankType()== RankType.DEFAULT)
+ field.setRankType((RankType.TAGS));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
new file mode 100644
index 00000000000..eb4efbe377f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Class that processes and validates tensor fields.
+ *
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldProcessor extends Processor {
+
+ public TensorFieldProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getDataType() == DataType.TENSOR) {
+ warnUseOfTensorFieldAsAttribute(field);
+ validateIndexingScripsForTensorField(field);
+ validateAttributeSettingForTensorField(field);
+ } else {
+ validateDataTypeForField(field);
+ }
+ }
+ }
+
+ private void warnUseOfTensorFieldAsAttribute(SDField field) {
+ if (field.doesAttributing()) {
+ // TODO (geirst): Remove when no longer beta
+ warn(search, field, "An attribute of type 'tensor' is currently beta, and re-feeding data between Vespa versions might be required.");
+ }
+ }
+
+ private void validateIndexingScripsForTensorField(SDField field) {
+ if (field.doesIndexing()) {
+ fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field.");
+ }
+ }
+
+ private void validateAttributeSettingForTensorField(SDField field) {
+ if (field.doesAttributing()) {
+ Attribute attribute = field.getAttributes().get(field.getName());
+ if (attribute != null && attribute.isFastSearch()) {
+ fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'.");
+ }
+ }
+ }
+
+ private void validateDataTypeForField(SDField field) {
+ if (field.getDataType().getPrimitiveType() == DataType.TENSOR) {
+ fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
new file mode 100644
index 00000000000..21723639ece
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class TextMatch extends Processor {
+
+ public TextMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.getMatching().getType() != Matching.Type.TEXT) {
+ continue;
+ }
+ ScriptExpression script = field.getIndexingScript();
+ if (script == null) {
+ continue;
+ }
+ DataType fieldType = field.getDataType();
+ if (fieldType instanceof CollectionDataType) {
+ fieldType = ((CollectionDataType)fieldType).getNestedType();
+ }
+ if (fieldType != DataType.STRING) {
+ continue;
+ }
+ Set<String> dynamicSummary = new TreeSet<>();
+ Set<String> staticSummary = new TreeSet<>();
+ new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(search, field, dynamicSummary, staticSummary);
+ MyVisitor visitor = new MyVisitor(dynamicSummary);
+ visitor.visit(script);
+ if (!visitor.requiresTokenize) {
+ continue;
+ }
+ ExpressionConverter converter = new MyStringTokenizer(search, findAnnotatorConfig(search, field));
+ field.setIndexingScript((ScriptExpression)converter.convert(script));
+ }
+ }
+
+ private static AnnotatorConfig findAnnotatorConfig(Search search, SDField field) {
+ AnnotatorConfig ret = new AnnotatorConfig();
+ Stemming activeStemming = field.getStemming();
+ if (activeStemming == null) {
+ activeStemming = search.getStemming();
+ }
+ ret.setStemMode(activeStemming.toStemMode());
+ ret.setRemoveAccents(field.getNormalizing().doRemoveAccents());
+ return ret;
+ }
+
+ private static class MyVisitor extends ExpressionVisitor {
+
+ final Set<String> dynamicSummaryFields;
+ boolean requiresTokenize = false;
+
+ MyVisitor(Set<String> dynamicSummaryFields) {
+ this.dynamicSummaryFields = dynamicSummaryFields;
+ }
+
+ @Override
+ protected void doVisit(Expression exp) {
+ if (exp instanceof IndexExpression) {
+ requiresTokenize = true;
+ }
+ if (exp instanceof SummaryExpression &&
+ dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName()))
+ {
+ requiresTokenize = true;
+ }
+ }
+ }
+
+ private static class MyStringTokenizer extends TypedTransformProvider {
+
+ final AnnotatorConfig annotatorCfg;
+
+ MyStringTokenizer(Search search, AnnotatorConfig annotatorCfg) {
+ super(TokenizeExpression.class, search);
+ this.annotatorCfg = annotatorCfg;
+ }
+
+ @Override
+ protected boolean requiresTransform(Expression exp, DataType fieldType) {
+ return exp instanceof OutputExpression;
+ }
+
+ @Override
+ protected Expression newTransform(DataType fieldType) {
+ Expression exp = new TokenizeExpression(null, annotatorCfg);
+ if (fieldType instanceof CollectionDataType) {
+ exp = new ForEachExpression(exp);
+ }
+ return exp;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java
new file mode 100644
index 00000000000..0fa9cbfa05f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.indexinglanguage.ValueTransformProvider;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class TypedTransformProvider extends ValueTransformProvider {
+
+ private final Search search;
+ private DataType fieldType;
+
+ TypedTransformProvider(Class<? extends Expression> transformClass, Search search) {
+ super(transformClass);
+ this.search = search;
+ }
+
+ @Override
+ protected final boolean requiresTransform(Expression exp) {
+ if (exp instanceof OutputExpression) {
+ String fieldName = ((OutputExpression)exp).getFieldName();
+ if (exp instanceof AttributeExpression) {
+ Attribute attribute = search.getAttribute(fieldName);
+ if (attribute == null) {
+ throw new IllegalArgumentException("Attribute '" + fieldName + "' not found.");
+ }
+ fieldType = attribute.getDataType();
+ } else if (exp instanceof IndexExpression) {
+ Field field = search.getField(fieldName);
+ if (field == null) {
+ throw new IllegalArgumentException("Index field '" + fieldName + "' not found.");
+ }
+ fieldType = field.getDataType();
+ } else if (exp instanceof SummaryExpression) {
+ Field field = search.getSummaryField(fieldName);
+ if (field == null) {
+ throw new IllegalArgumentException("Summary field '" + fieldName + "' not found.");
+ }
+ fieldType = field.getDataType();
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+ return requiresTransform(exp, fieldType);
+ }
+
+ @Override
+ protected final Expression newTransform() {
+ return newTransform(fieldType);
+ }
+
+ protected abstract boolean requiresTransform(Expression exp, DataType fieldType);
+
+ protected abstract Expression newTransform(DataType fieldType);
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
new file mode 100644
index 00000000000..8f9b88b0268
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author balder
+ */
+public class UriHack extends Processor {
+ private static final List<String> URL_SUFFIX =
+ Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname");
+ public UriHack(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (field.doesIndexing()) {
+ DataType fieldType = field.getDataType();
+ if (fieldType instanceof CollectionDataType) {
+ fieldType = ((CollectionDataType)fieldType).getNestedType();
+ }
+ if (fieldType == DataType.URI) {
+ processField(search, field);
+ }
+ }
+ }
+ }
+
+ private void processField(Search search, SDField uriField) {
+ String uriName = uriField.getName();
+ uriField.setStemming(Stemming.NONE);
+ DataType generatedType = DataType.STRING;
+ if (uriField.getDataType() instanceof ArrayDataType) {
+ generatedType = new ArrayDataType(DataType.STRING);
+ } else if (uriField.getDataType() instanceof WeightedSetDataType) {
+ WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType();
+ generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero());
+ }
+ for (String suffix : URL_SUFFIX) {
+ String partName = uriName + "." + suffix;
+ // I wonder if this is explicit in qrs or implicit in backend?
+ // search.addFieldSetItem(uriName, partName);
+ SDField partField = new SDField(partName, generatedType, true);
+ partField.setIndexStructureField(uriField.doesIndexing());
+ partField.setRankType(uriField.getRankType());
+ partField.setStemming(Stemming.NONE);
+ partField.getNormalizing().inferLowercase();
+ if (uriField.getIndex(suffix) != null) {
+ partField.addIndex(uriField.getIndex(suffix));
+ }
+ search.addExtraField(partField);
+ search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, partField.getName());
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
new file mode 100644
index 00000000000..b01a82b0131
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * @author bratseth
+ */
+public class UrlFieldValidator extends Processor {
+
+ public UrlFieldValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if ( ! field.getDataType().equals(DataType.URI)) continue;
+
+ if (field.doesAttributing())
+ throw new IllegalArgumentException("Error in " + field + " in " + search + ": " +
+ "uri type fields cannot be attributes");
+ }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
new file mode 100644
index 00000000000..38c7338be3c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This Processor checks to make sure all fields with the same name have the same {@link DataType}. This check
+ * explicitly disregards whether a field is an index field, an attribute or a summary field. This is a requirement if we
+ * hope to move to a model where index fields, attributes and summary fields share a common field class.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ValidateFieldTypes extends Processor {
+
+ public ValidateFieldTypes(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ String searchName = search.getName();
+ Map<String, DataType> fieldTypes = new HashMap<>();
+ for (SDField field : search.allFieldsList()) {
+ checkFieldType(searchName, "index field", field.getName(), field.getDataType(), fieldTypes);
+ for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) {
+ checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), fieldTypes);
+ }
+ }
+ for (DocumentSummary summary : search.getSummaries().values()) {
+ for (SummaryField field : summary.getSummaryFields()) {
+ checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), fieldTypes);
+ }
+ }
+ }
+
+ private void checkFieldType(String searchName, String fieldDesc, String fieldName, DataType fieldType,
+ Map<String, DataType> fieldTypes)
+ {
+ DataType prevType = fieldTypes.get(fieldName);
+ if (prevType == null) {
+ fieldTypes.put(fieldName, fieldType);
+ } else if (!equalTypes(prevType, fieldType)) {
+ throw newProcessException(searchName, fieldName, "Duplicate field name with different types. Expected " + prevType.getName() + " for " + fieldDesc +
+ " '" + fieldName + "', got " + fieldType.getName() + ".");
+ }
+ }
+
+ private boolean equalTypes(DataType d1, DataType d2) {
+ if ("tag".equals(d1.getName())) {
+ return "tag".equals(d2.getName()) || "WeightedSet<string>".equals(d2.getName());
+ }
+ if ("tag".equals(d2.getName())) {
+ return "tag".equals(d1.getName()) || "WeightedSet<string>".equals(d1.getName());
+ }
+ return d1.equals(d2);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
new file mode 100644
index 00000000000..a5c7d25532d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.Ranking;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Check that fields with index settings actually creates an index or attribute
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor {
+
+ public ValidateFieldWithIndexSettingsCreatesIndex(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ Matching defaultMatching = new Matching();
+ Ranking defaultRanking = new Ranking();
+ for (SDField field : search.allFieldsList()) {
+ if (field.doesIndexing()) {
+ continue;
+ }
+ if (field.doesAttributing()) {
+ continue;
+ }
+ if (!(field.getRanking().equals(defaultRanking))) {
+ fail(search, field, "Fields which are not creating an index or attribute can not contain rank settings.");
+ }
+ if (!(field.getMatching().equals(defaultMatching))) {
+ fail(search, field, "Fields which are not creating an index or attribute can not contain match settings.");
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
new file mode 100644
index 00000000000..24b811fc7fc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of word matching - with word matching the field is assumed to contain a single "word" - some
+ * contiguous sequence of word and number characters - but without changing the data at the indexing side (as with text
+ * matching) to enforce this. Word matching is thus almost like exact matching on the indexing side (no action taken),
+ * and like text matching on the query side. This may be suitable for attributes, where people both expect the data to
+ * be left as in the input document, and trivially written queries to work by default. However, this may easily lead to
+ * data which cannot be matched at all as the indexing and query side does not agree.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class WordMatch extends Processor {
+
+ public WordMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ public void process() {
+ for (SDField field : search.allFieldsList()) {
+ if (!field.getMatching().getType().equals(Matching.Type.WORD)) {
+ continue;
+ }
+ field.setStemming(Stemming.NONE);
+ field.getNormalizing().inferLowercase();
+ field.addQueryCommand("word");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java
new file mode 100644
index 00000000000..158515fd05e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Resolver-class for harmonizing index-commands in multifield indexes
+ */
+public class IndexCommandResolver extends MultiFieldResolver {
+
+ /** Commands which don't have to be harmonized between fields */
+ private static List<String> ignoredCommands = new ArrayList<>();
+
+ /** Commands which must be harmonized between fields */
+ private static List<String> harmonizedCommands = new ArrayList<>();
+
+ static {
+ String[] ignore = { "complete-boost", "literal-boost", "highlight" };
+ ignoredCommands.addAll(Arrays.asList(ignore));
+ String[] harmonize = { "stemming", "normalizing" };
+ harmonizedCommands.addAll(Arrays.asList(harmonize));
+ }
+
+ public IndexCommandResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+ super(indexName, fields, search, logger);
+ }
+
+ /**
+ * Check index-commands for each field, report and attempt to fix any
+ * inconsistencies
+ */
+ public void resolve() {
+ for (SDField field : fields) {
+ for (String command : field.getQueryCommands()) {
+ if (!ignoredCommands.contains(command))
+ checkCommand(command);
+ }
+ }
+ }
+
+ private void checkCommand(String command) {
+ for (SDField field : fields) {
+ if (!field.hasQueryCommand(command)) {
+ if (harmonizedCommands.contains(command)) {
+ deployLogger.log(Level.WARNING, command + " must be added to all fields going to the same index (" + indexName + ")" +
+ ", adding to field " + field.getName());
+ field.addQueryCommand(command);
+ } else {
+ deployLogger.log(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() +
+ "\' doesn't contain command \'" + command+"\'");
+ }
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java
new file mode 100644
index 00000000000..943a788fc75
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import java.util.List;
+
+/**
+ * Abstract superclass of all multifield conflict resolvers
+ */
+public abstract class MultiFieldResolver {
+
+ protected String indexName;
+ protected List<SDField> fields;
+ protected Search search;
+
+ protected DeployLogger deployLogger;
+
+ public MultiFieldResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+ this.indexName = indexName;
+ this.fields = fields;
+ this.search = search;
+ this.deployLogger = logger;
+ }
+
+ /**
+ * Checks the list of fields for specific conflicts, and reports and/or
+ * attempts to correct them
+ */
+ public abstract void resolve();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
new file mode 100644
index 00000000000..d98dd97ff83
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.TensorFieldType;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.processing.Processor;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Class that processes a search instance and sets type settings on all rank profiles.
+ *
+ * Currently, type settings are limited to the type of tensor attribute fields and tensor query features.
+ *
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class RankProfileTypeSettingsProcessor extends Processor {
+
+ private static final Pattern queryFeaturePattern = Pattern.compile("query\\((\\w+)\\)$");
+
+ public RankProfileTypeSettingsProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process() {
+ processAttributeFields();
+ processQueryProfileTypes();
+
+ }
+
+ private void processAttributeFields() {
+ for (SDField field : search.allFieldsList()) {
+ Attribute attribute = field.getAttributes().get(field.getName());
+ if (attribute != null && attribute.tensorType().isPresent()) {
+ addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString());
+ }
+ }
+ }
+
+ private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) {
+ for (RankProfile profile : rankProfileRegistry.allRankProfiles()) {
+ profile.addAttributeType(attributeName, attributeType);
+ }
+ }
+
+ private void processQueryProfileTypes() {
+ for (QueryProfileType queryProfileType : queryProfiles.getRegistry().getTypeRegistry().allComponents()) {
+ for (Map.Entry<String, FieldDescription> fieldDescEntry : queryProfileType.fields().entrySet()) {
+ processFieldDescription(fieldDescEntry.getValue());
+ }
+ }
+ }
+
+ private void processFieldDescription(FieldDescription fieldDescription) {
+ String fieldName = fieldDescription.getName();
+ FieldType fieldType = fieldDescription.getType();
+ if (fieldType instanceof TensorFieldType) {
+ TensorFieldType tensorFieldType = (TensorFieldType)fieldType;
+ Matcher matcher = queryFeaturePattern.matcher(fieldName);
+ if (tensorFieldType.type().isPresent() && matcher.matches()) {
+ String queryFeature = matcher.group(1);
+ addQueryFeatureTypeToRankProfiles(queryFeature, tensorFieldType.type().get().toString());
+ }
+ }
+ }
+
+ private void addQueryFeatureTypeToRankProfiles(String queryFeature, String queryFeatureType) {
+ for (RankProfile profile : rankProfileRegistry.allRankProfiles()) {
+ profile.addQueryFeatureType(queryFeature, queryFeatureType);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java
new file mode 100644
index 00000000000..787a70862a9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Checks if fields have defined different rank types for the same
+ * index (typically in an index-to statement), and if they have
+ * output a warning and use the first ranktype.
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class RankTypeResolver extends MultiFieldResolver {
+
+ public RankTypeResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+ super(indexName, fields, search, logger);
+ }
+
+ public void resolve() {
+ RankType rankType = null;
+ if (fields.size() > 0) {
+ boolean first = true;
+ for (SDField field : fields) {
+ if (first) {
+ rankType = fields.get(0).getRankType();
+ first = false;
+ } else if (!field.getRankType().equals(rankType)) {
+ deployLogger.log(Level.WARNING, "In field '" + field.getName() + "' " +
+ field.getRankType() + " for index '" + indexName +
+ "' conflicts with " + rankType +
+ " defined for the same index in field '" +
+ field.getName() + "'. Using " +
+ rankType + ".");
+ field.setRankType(rankType);
+ }
+ }
+ }
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java
new file mode 100644
index 00000000000..f1e1899391d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.language.Linguistics;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.searchdefinition.processing.BuiltInFieldSets;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Class resolving conflicts when fields with different stemming-settings are
+ * combined into the same index
+ */
+public class StemmingResolver extends MultiFieldResolver {
+
+ public StemmingResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+ super(indexName, fields, search, logger);
+ }
+
+ @Override
+ public void resolve() {
+ checkStemmingForIndexFields(indexName, fields);
+ }
+
+ private void checkStemmingForIndexFields(String indexName, List<SDField> fields) {
+ Stemming stemming = null;
+ SDField stemmingField = null;
+ for (SDField field : fields) {
+ if (stemming == null && stemmingField==null) {
+ stemming = field.getStemming(search);
+ stemmingField = field;
+ } else if (stemming != field.getStemming(search)) {
+ deployLogger.log(Level.WARNING, "Field '" + field.getName() + "' has " + field.getStemming(search) +
+ ", whereas field '" + stemmingField.getName() + "' has " + stemming +
+ ". All fields indexing to the index '" + indexName + "' must have the same stemming." +
+ " This should be corrected as it will make indexing fail in a few cases.");
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java
new file mode 100644
index 00000000000..9915e11b28d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Classes in this package (processors) implements some search
+ * definition features by reducing them to simpler features.
+ * The processors are run after parsing of the search definition,
+ * before creating the derived model.
+ *
+ * For simplicity, features should always be implemented here
+ * rather than in the derived model if possible.
+ *
+ * New processors must be added to the list in Processing.
+ */
+@com.yahoo.api.annotations.PackageMarker
+package com.yahoo.searchdefinition.processing;
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
new file mode 100644
index 00000000000..29c0c4e8136
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -0,0 +1,156 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configmodel.producers;
+
+import com.yahoo.document.config.DocumentmanagerConfig;
+import static com.yahoo.document.config.DocumentmanagerConfig.*;
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.documentmodel.DataTypeCollection;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author balder
+ * @since 2010-02-19
+ */
+public class DocumentManager {
+
+ public DocumentmanagerConfig.Builder produce(DocumentModel model, DocumentmanagerConfig.Builder docman) {
+ docman.enablecompression(false);
+ Set<DataType> handled = new HashSet<>();
+ for(NewDocumentType documentType : model.getDocumentManager().getTypes()) {
+ handle(documentType, docman, handled);
+ handleAnnotations(documentType.getAnnotations(), docman);
+ if ( documentType != VespaDocumentType.INSTANCE) {
+ DocumentmanagerConfig.Datatype.Builder dt = new DocumentmanagerConfig.Datatype.Builder();
+ docman.datatype(dt);
+ handleDataType(documentType, dt);
+ }
+ }
+ return docman;
+ }
+
+ private void handle(DataTypeCollection type, DocumentmanagerConfig.Builder docman, Set<DataType> handled) {
+ for (DataType dataType : type.getTypes()) {
+ if (handled.contains(dataType)) continue;
+ handled.add(dataType);
+ if (dataType instanceof TemporaryStructuredDataType) continue;
+ if ((dataType.getId() < 0) || (DataType.lastPredefinedDataTypeId() < dataType.getId())) {
+ Datatype.Builder dtc = new Datatype.Builder();
+ docman.datatype(dtc);
+ handleDataType(dataType, dtc);
+ }
+ }
+ }
+
+ private void handleAnnotation(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) {
+ atb.
+ id(type.getId()).
+ name(type.getName());
+ if (type.getDataType() != null) {
+ atb.datatype(type.getDataType().getId());
+ }
+ if ( ! type.getInheritedTypes().isEmpty()) {
+ for (AnnotationType inherited : type.getInheritedTypes()) {
+ atb.inherits(new DocumentmanagerConfig.Annotationtype.Inherits.Builder().id(inherited.getId()));
+ }
+ }
+ }
+ private void handleAnnotations(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
+ for (AnnotationType type : types) {
+ DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
+ handleAnnotation(type, atb);
+ builder.annotationtype(atb);
+ }
+ }
+
+ private void handleDataType(DataType type, Datatype.Builder dtc) {
+ dtc.id(type.getId());
+ if (type instanceof ArrayDataType) {
+ CollectionDataType dt = (CollectionDataType) type;
+ dtc.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId()));
+ } else if (type instanceof WeightedSetDataType) {
+ WeightedSetDataType dt = (WeightedSetDataType) type;
+ dtc.weightedsettype(new Datatype.Weightedsettype.Builder().
+ datatype(dt.getNestedType().getId()).
+ createifnonexistant(dt.createIfNonExistent()).
+ removeifzero(dt.removeIfZero()));
+ } else if (type instanceof MapDataType) {
+ MapDataType mtype = (MapDataType) type;
+ dtc.maptype(new Datatype.Maptype.Builder().
+ keytype(mtype.getKeyType().getId()).
+ valtype(mtype.getValueType().getId()));
+ } else if (type instanceof DocumentType) {
+ DocumentType dt = (DocumentType) type;
+ Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
+ dtc.documenttype(doc);
+ doc.
+ name(dt.getName()).
+ headerstruct(dt.getHeaderType().getId()).
+ bodystruct(dt.getBodyType().getId());
+ for (DocumentType inherited : dt.getInheritedTypes()) {
+ doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
+ }
+ } else if (type instanceof NewDocumentType) {
+ NewDocumentType dt = (NewDocumentType) type;
+ Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
+ dtc.documenttype(doc);
+ doc.
+ name(dt.getName()).
+ headerstruct(dt.getHeader().getId()).
+ bodystruct(dt.getBody().getId());
+ for (NewDocumentType inherited : dt.getInherited()) {
+ doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
+ }
+ handleFieldSets(dt.getFieldSets(), doc);
+ } else if (type instanceof TemporaryStructuredDataType) {
+ //Ignored
+ } else if (type instanceof StructDataType) {
+ StructDataType dt = (StructDataType) type;
+ Datatype.Structtype.Builder st = new Datatype.Structtype.Builder();
+ dtc.structtype(st);
+ st.name(dt.getName());
+ if (dt.getCompressionConfig().type.getCode() != 0) {
+ st.
+ compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(dt.getCompressionConfig().type.toString())).
+ compresslevel(dt.getCompressionConfig().compressionLevel).
+ compressthreshold((int)dt.getCompressionConfig().threshold).
+ compressminsize((int)dt.getCompressionConfig().minsize);
+ }
+ for (com.yahoo.document.Field field : dt.getFieldsThisTypeOnly()) {
+ Datatype.Structtype.Field.Builder fb = new Datatype.Structtype.Field.Builder();
+ st.field(fb);
+ fb.name(field.getName());
+ if (field.hasForcedId()) {
+ fb.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId()));
+ }
+ fb.datatype(field.getDataType().getId());
+ }
+ for (StructDataType inherited : dt.getInheritedTypes()) {
+ st.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName()));
+ }
+ } else if (type instanceof AnnotationReferenceDataType) {
+ AnnotationReferenceDataType annotationRef = (AnnotationReferenceDataType) type;
+ dtc.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName()));
+ } else {
+ throw new IllegalArgumentException("Can not handle datatype '" + type.getName());
+ }
+ }
+
+ private void handleFieldSets(Set<FieldSet> fieldSets, Datatype.Documenttype.Builder doc) {
+
+ for (FieldSet builtinFs : fieldSets) {
+ handleFieldSet(builtinFs, doc);
+ }
+ }
+
+ private void handleFieldSet(FieldSet fs, Datatype.Documenttype.Builder doc) {
+ doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
new file mode 100644
index 00000000000..92b3c13d383
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -0,0 +1,158 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.configmodel.producers;
+
+import com.yahoo.document.*;
+import com.yahoo.document.DocumenttypesConfig.Builder;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.documentmodel.DataTypeCollection;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.FieldSets;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import java.util.*;
+
+/**
+ * @author balder
+ */
+public class DocumentTypes {
+
+ public DocumenttypesConfig.Builder produce(DocumentModel model, DocumenttypesConfig.Builder builder) {
+ Map<NewDocumentType.Name, NewDocumentType> produced = new HashMap<>();
+ for(NewDocumentType documentType : model.getDocumentManager().getTypes()) {
+ produceInheritOrder(documentType, builder, produced);
+ }
+ return builder;
+ }
+
+ private void produceInheritOrder(NewDocumentType documentType, DocumenttypesConfig.Builder builder, Map<NewDocumentType.Name, NewDocumentType> produced) {
+ if ( ! produced.containsKey(documentType.getFullName())) {
+ for (NewDocumentType inherited : documentType.getInherited()) {
+ produceInheritOrder(inherited, builder, produced);
+ }
+ handle(documentType, builder);
+ produced.put(documentType.getFullName(), documentType);
+ }
+ }
+
+ private void handle(NewDocumentType documentType, DocumenttypesConfig.Builder builder) {
+ if (documentType == VespaDocumentType.INSTANCE) {
+ return;
+ }
+ DocumenttypesConfig.Documenttype.Builder db = new DocumenttypesConfig.Documenttype.Builder();
+ db.
+ id(documentType.getId()).
+ name(documentType.getName()).
+ headerstruct(documentType.getHeader().getId()).
+ bodystruct(documentType.getBody().getId());
+ Set<Integer> handled = new HashSet<>();
+ for (NewDocumentType inherited : documentType.getInherited()) {
+ db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId()));
+ markAsHandled(handled, inherited.getAllTypes());
+ }
+ for (DataType dt : documentType.getTypes()) {
+ handle(dt, db, handled);
+ }
+ for(AnnotationType annotation : documentType.getAnnotations()) {
+ DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder();
+ db.annotationtype(atb);
+ handle(annotation, atb);
+ }
+ handleFieldSets(documentType.getFieldSets(), db);
+ builder.documenttype(db);
+ }
+
+ private void handleFieldSets(Set<FieldSet> fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) {
+ for (FieldSet fs : fieldSets) {
+ handleFieldSet(fs, db);
+ }
+ }
+
+ private void handleFieldSet(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) {
+ db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
+ }
+
+ private void markAsHandled(Set<Integer> handled, DataTypeCollection typeCollection) {
+ for (DataType type : typeCollection.getTypes()) {
+ handled.add(type.getId());
+ }
+ }
+
+ private void handle(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) {
+ builder.
+ id(annotation.getId()).
+ name(annotation.getName());
+ DataType dt = annotation.getDataType();
+ if (dt!=null) {
+ builder.datatype(dt.getId());
+ }
+ for (AnnotationType inherited : annotation.getInheritedTypes()) {
+ builder.inherits(new DocumenttypesConfig.Documenttype.Annotationtype.Inherits.Builder().id(inherited.getId()));
+ }
+ }
+
+ private void handle(DataType type, DocumenttypesConfig.Documenttype.Builder db, Set<Integer> handled) {
+ if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! handled.contains(type.getId())) {
+ handled.add(type.getId());
+ DocumenttypesConfig.Documenttype.Datatype.Builder dtb = new DocumenttypesConfig.Documenttype.Datatype.Builder();
+ dtb.id(type.getId());
+ if (type instanceof StructDataType) {
+ dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("STRUCT"));
+ StructDataType dt = (StructDataType) type;
+ DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder sb = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder();
+ dtb.sstruct(sb);
+ sb.name(dt.getName());
+ if (dt.getCompressionConfig().type.getCode() != 0) {
+ sb.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder().
+ type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(dt.getCompressionConfig().type.toString())).
+ level(dt.getCompressionConfig().compressionLevel).
+ threshold((int)dt.getCompressionConfig().threshold).
+ minsize((int)dt.getCompressionConfig().minsize));
+ }
+ for (com.yahoo.document.Field field : dt.getFields()) {
+ sb.field(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder().
+ name(field.getName()).
+ id(field.getId()).
+ id_v6(field.getIdV6()).
+ datatype(field.getDataType().getId()));
+ handle(field.getDataType(), db, handled);
+ }
+ } else if (type instanceof ArrayDataType) {
+ dtb.
+ type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ARRAY")).
+ array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder().
+ element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(((ArrayDataType)type).getNestedType().getId())));
+ handle(((ArrayDataType)type).getNestedType(), db, handled);
+ } else if (type instanceof WeightedSetDataType) {
+ dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("WSET")).
+ wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder().
+ key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder().
+ id(((WeightedSetDataType)type).getNestedType().getId())).
+ createifnonexistent(((WeightedSetDataType)type).createIfNonExistent()).
+ removeifzero(((WeightedSetDataType)type).removeIfZero()));
+ handle(((WeightedSetDataType)type).getNestedType(), db, handled);
+ } else if (type instanceof MapDataType) {
+ dtb.
+ type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("MAP")).
+ map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder().
+ key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder().
+ id(((MapDataType)type).getKeyType().getId())).
+ value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder().
+ id(((MapDataType)type).getValueType().getId())));
+ handle(((MapDataType)type).getKeyType(), db, handled);
+ handle(((MapDataType)type).getValueType(), db, handled);
+ } else if (type instanceof AnnotationReferenceDataType) {
+ dtb.
+ type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ANNOTATIONREF")).
+ annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder().
+ annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder().
+ id(((AnnotationReferenceDataType)type).getAnnotationType().getId())));
+ } else {
+ return;
+ }
+ db.datatype(dtb);
+ }
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java
new file mode 100644
index 00000000000..6c8206d30f2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.documentmodel.DocumentTypeRepo;
+
+/**
+ * DocumentModel represents everything derived from a set of search definitions.
+ * It contains a document manager managing all defined document types.
+ * It contains a search manager managing all specified search definitions.
+ * It contains a storage manager managing all specified storage definitions.
+ *
+ * @author balder
+ * @since 2010-02-19
+ */
+public class DocumentModel {
+ private DocumentTypeRepo documentMan = new DocumentTypeRepo();
+ private SearchManager searchMan = new SearchManager();
+
+ /**
+ *
+ * @return Returns the DocumentManager
+ */
+ public DocumentTypeRepo getDocumentManager() { return documentMan; }
+
+ /**
+ *
+ * @return Returns the SearchManager
+ */
+ public SearchManager getSearchManager() { return searchMan; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java
new file mode 100644
index 00000000000..42fa7b04cf6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.Field;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A document summary definition - a list of summary fields.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class DocumentSummary extends FieldView {
+
+
+ /**
+ * Will create a DocumentSummary with the given name.
+ * @param name The name to use for this summary.
+ */
+ public DocumentSummary(String name) {
+ super(name);
+ }
+
+ /**
+ * The model is constrained to ensure that summary fields of the same name
+ * in different classes have the same summary transform, because this is
+ * what is supported by the backend currently.
+ * @param summaryField The summaryfield to add
+ */
+ public void add(SummaryField summaryField) {
+ summaryField.addDestination(getName());
+ super.add(summaryField);
+ }
+
+ public SummaryField getSummaryField(String name) {
+ return (SummaryField) get(name);
+ }
+
+ public Collection<SummaryField> getSummaryFields() {
+ ArrayList<SummaryField> fields = new ArrayList<>(getFields().size());
+ for(Field f : getFields()) {
+ fields.add((SummaryField) f);
+ }
+ return fields;
+ }
+
+ /**
+ * Removes implicit fields which shouldn't be included.
+ * This is implicitly added fields which are sources for
+ * other fields. We then assume they are not intended to be added
+ * implicitly in additon.
+ * This should be called when this summary is complete.
+ */
+ public void purgeImplicits() {
+ List<SummaryField> falseImplicits = new ArrayList<>();
+ for (SummaryField summaryField : getSummaryFields() ) {
+ if (summaryField.isImplicit()) continue;
+ for (Iterator<SummaryField.Source> j = summaryField.sourceIterator(); j.hasNext(); ) {
+ String sourceName = j.next().getName();
+ if (sourceName.equals(summaryField.getName())) continue;
+ SummaryField sourceField=getSummaryField(sourceName);
+ if (sourceField==null) continue;
+ if (!sourceField.isImplicit()) continue;
+ falseImplicits.add(sourceField);
+ }
+ }
+ for (SummaryField field : falseImplicits) {
+ remove(field.getName());
+ }
+ }
+
+ public String toString() {
+ return "document summary '" + getName() + "'";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java
new file mode 100644
index 00000000000..dfb44aef917
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.Field;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author balder
+ * @since 2010-02-19
+ */
+public class FieldView implements Serializable {
+ private String name;
+ private Map<String, Field> fields = new LinkedHashMap<>();
+
+ /**
+ * Creates a view with a name
+ * @param name Name of the view.
+ */
+ public FieldView(String name) {
+ this.name = name;
+ }
+ public String getName() { return name; }
+ public Collection<Field> getFields() { return fields.values(); }
+ public Field get(String name) { return fields.get(name); }
+ public void remove(String name) { fields.remove(name); }
+
+ /**
+ * This method will add a field to a view. All fields must come from the same document type. Not enforced here.
+ * @param field The field to add.
+ * @return Itself for chaining purposes.
+ */
+ public FieldView add(Field field) {
+ if (fields.containsKey(field.getName())) {
+ if ( ! fields.get(field.getName()).equals(field)) {
+ throw new IllegalArgumentException(
+ "View '" + name + "' already contains a field with name '" +
+ field.getName() + "' and definition : " +
+ fields.get(field.getName()).toString() + ". Your is : " + field.toString());
+ }
+ } else {
+ fields.put(field.getName(), field);
+ }
+ return this;
+ }
+
+ /**
+ * This method will join the two views.
+ * @param view The view to be joined in to this.
+ * @return Itself for chaining.
+ */
+ public FieldView add(FieldView view) {
+ for(Field field : view.getFields()) {
+ add(field);
+ }
+ return this;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java
new file mode 100644
index 00000000000..07b7c973841
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * @author balder
+ * @since 2010-02-19
+ */
+public class SearchDef {
+ private final static Logger log = Logger.getLogger(SearchDef.class.getName());
+ /// Name of the searchdefinition
+ private String name;
+ /// These are the real backing documenttypes
+ private DocumentTypeManager sources = new DocumentTypeManager();
+ /// Map of all search fields
+ private Map<String, SearchField> fields = new HashMap<>();
+ /// Map of all views that can be searched.
+ private Map<String, FieldView> views = new HashMap<>();
+ /// Map of all aliases <alias, realname>
+ private Map<String, String> aliases = new HashMap<>();
+
+ /**
+ * Will create a SearchDef with the given name
+ * @param name The name of the searchdefinition
+ */
+ public SearchDef(String name) {
+ this.name = name;
+ }
+
+ /**
+ * This will provide you with the name of the searchdefinition.
+ * @return The name of the searchdefinition.
+ */
+ public String getName() { return name; }
+
+ public Map<String, SearchField> getFields() { return fields; }
+ public Map<String, FieldView> getViews() { return views; }
+
+ /**
+ * Adds a document that can be mapped to this search.
+ * @param source A document that can be mapped to this search.
+ * @return Itself for chaining.
+ */
+ public SearchDef add(DataType source) {
+ sources.register(source);
+ return this;
+ }
+
+ private void noShadowing(String name) {
+ noFieldShadowing(name);
+ noViewShadowing(name);
+ }
+
+ private void noFieldShadowing(String name) {
+ if (fields.containsKey(name)) {
+ throw new IllegalArgumentException("Searchdef '" + getName() + "' already contains the fields '" + fields.toString() +
+ "'. You are trying to add '" + name + "'. Shadowing is not supported");
+ }
+ }
+
+ private void noViewShadowing(String name) {
+ if (views.containsKey(name)) {
+ throw new IllegalArgumentException("Searchdef '" + getName() + "' already contains a view with name '" +
+ name + "'. Shadowing is not supported.");
+ }
+ }
+
+ /**
+ * Adds a search field to the definition.
+ * @param field The field to add.
+ * @return Itself for chaining.
+ */
+ public SearchDef add(SearchField field) {
+ try {
+ noFieldShadowing(field.getName());
+ fields.put(field.getName(), field);
+ } catch (IllegalArgumentException e) {
+ if (views.containsKey(field.getName())) {
+ throw e;
+ }
+ }
+ return this;
+ }
+
+ public SearchDef addAlias(String alias, String aliased) {
+ noShadowing(alias);
+ if (!fields.containsKey(aliased) && !views.containsKey(aliased)) {
+ if (aliased.contains(".")) {
+ // TODO Here we should nest ourself down to something that really exists.
+ log.warning("Aliased item '" + aliased + "' not verifiable. Allowing it to be aliased to '" + alias + " for now. Validation will come when URL/Position is structified.");
+ } else {
+ throw new IllegalArgumentException("Searchdef '" + getName() + "' has nothing named '" + aliased + "'to alias to '" + alias + "'.");
+ }
+ }
+ String oldAliased = aliases.get(alias);
+ if ((oldAliased != null)) {
+ if (oldAliased.equals(aliased)) {
+ throw new IllegalArgumentException("Searchdef '" + getName() + "' already has the alias '" + alias +
+ "' to '" + aliased + ". Why do you want to add it again.");
+
+ } else {
+ throw new IllegalArgumentException("Searchdef '" + getName() + "' already has the alias '" + alias +
+ "' to '" + oldAliased + ". Cannot change it to alias '" + aliased + "'.");
+ }
+ } else {
+ aliases.put(alias, aliased);
+ }
+ return this;
+ }
+
+ public SearchDef add(FieldView view) {
+ noViewShadowing(view.getName());
+ if (views.containsKey(view.getName())) {
+ views.get(view.getName()).add(view);
+ }
+ views.put(view.getName(), view);
+ return this;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java
new file mode 100644
index 00000000000..2db81861955
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author balder
+ * @since 2010-02-19
+ */
+public class SearchField extends Field {
+ /// Indicate if field shall be stored in memory for attribute usage.
+ private boolean attribute = false;
+ /// Indicate if the field is Vespa indexed.
+ private boolean indexed = false;
+ /// Indication to backend on how much optimization should be done.
+
+ /**
+ * This is a representation of features to generate for this field.
+ * It can be both optimize hints, and real functional hints.
+ */
+ public enum Feature {
+ WEIGHT_IN_ATTRIBUTE_POSTINGS("WeightInAttributePosting"), // Hint to put the weight in postings for attribute.
+ WORDPOS_IN_POSTINGS("WordPosInPosting"), // Default for generating posocc
+ FILTER_ONLY("FilterOnly"); // Might only generate bitvector
+ private String name;
+ Feature(String name) { this.name = name;}
+ public String getName() { return name; }
+ }
+ private List<Feature> featureList = new ArrayList<>();
+
+ public SearchField(Field field, boolean indexed, boolean attribute) {
+ this(field, indexed, attribute, null);
+ }
+ public SearchField(Field field, boolean indexed, boolean attribute, List<Feature> features) {
+ super(field.getName(), field);
+ this.attribute = attribute;
+ this.indexed = indexed;
+ if (features != null) {
+ featureList.addAll(features);
+ }
+ validate();
+ }
+
+ @SuppressWarnings({ "deprecation" })
+ private void validate() {
+ if (attribute || !indexed) {
+ return;
+ }
+ DataType fieldType = getDataType();
+ DataType primiType = fieldType.getPrimitiveType();
+ if (DataType.STRING.equals(primiType) || DataType.URI.equals(primiType)) {
+ return;
+ }
+ throw new IllegalStateException("Expected type " + DataType.STRING.getName() + " for indexed field '" +
+ getName() + "', got " + fieldType.getName() + ".");
+ }
+
+ public SearchField setIndexed() { indexed = true; validate(); return this; }
+ public SearchField setAttribute() { attribute = true; validate(); return this; }
+ public boolean isAttribute() { return attribute; }
+ /**
+ * True if field is Vespa indexed
+ * @return true if indexed
+ */
+ public boolean isIndexed() { return indexed; }
+ public SearchField addFeature(Feature feature) { featureList.add(feature); validate(); return this; }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java
new file mode 100644
index 00000000000..29a960f7e7b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import java.util.TreeMap;
+
+/**
+ * @author balder
+ * @since 2010-02-19
+ */
+public class SearchManager {
+ /// This is the list of all known search definitions
+ private TreeMap<String, SearchDef> defs = new TreeMap<>();
+
+ /**
+ * This will add a searchdefinition or throw an IllegalArgumentException if the name is already used
+ * @param def The searchdef to add
+ * @return itself for chaining purposes.
+ */
+ public SearchManager add(SearchDef def) {
+ if (defs.containsKey(def.getName())) {
+ throw new IllegalArgumentException("There already exist a searchdefinition with this content:\n" +
+ defs.get(def.getName()).toString() + "\n No room for : " + def.toString());
+ }
+ defs.put(def.getName(), def);
+ return this;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java
new file mode 100644
index 00000000000..f6db82785b0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java
@@ -0,0 +1,350 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.TypedKey;
+
+import java.io.Serializable;
+import java.util.*;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * A summary field
+ *
+ * @author bratseth
+ */
+public class SummaryField extends Field implements Cloneable, TypedKey {
+
+ /**
+ * This class represents a source (field name) and the type of the source (only used for smart summary)
+ */
+ public static class Source implements Serializable {
+ public static enum Type {
+ CONTEXTUAL("contextual"),
+ TITLE("title"),
+ STATIC("static"),
+ URL("url");
+ private final String name;
+ Type(String name) {
+ this.name = name;
+ }
+ public String getName() { return name; }
+ }
+ private String name;
+ private Type type;
+ private boolean override = false;
+ public Source(String name) {
+ this.name = name;
+ this.type = Type.CONTEXTUAL;
+ }
+ public Source(String name, Type type) {
+ this.name = name;
+ this.type = type;
+ }
+ public String getName() { return name; }
+ public Type getType() { return type; }
+ public void setOverride(boolean override) { this.override = override; }
+ public boolean getOverride() { return override; }
+ public int hashCode() {
+ return name.hashCode() + type.getName().hashCode() + Boolean.valueOf(override).hashCode();
+ }
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Source)) {
+ return false;
+ }
+ Source other = (Source)obj;
+ return name.equals(other.name) &&
+ type.getName().equals(other.type.getName()) &&
+ override == other.override;
+ }
+ public String toString() {
+ return name;
+ }
+ }
+
+ /** A name-value property (used for smart summary) */
+ public static class Property implements Serializable {
+ private String name;
+ private String value;
+ public Property(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+ public String getName() { return name; }
+ public String getValue() { return value; }
+ public @Override int hashCode() {
+ return name.hashCode() + 17*value.hashCode();
+ }
+ public @Override boolean equals(Object obj) {
+ if (!(obj instanceof Property)) {
+ return false;
+ }
+ Property other = (Property)obj;
+ return name.equals(other.name) && value.equals(other.value);
+ }
+ }
+
+ /** The transform to perform on the stored source */
+ private SummaryTransform transform=SummaryTransform.NONE;
+
+ /** The command used per field in vsmsummary */
+ private VsmCommand vsmCommand = VsmCommand.NONE;
+
+ /**
+ * The data sources for this output summary field, in prioritized order
+ * (use only second source if first yields no result after transformation
+ * and so on). If no sources are given explicitly, the field of the same
+ * name as this summary field is used
+ */
+ private Set<Source> sources = new java.util.LinkedHashSet<>();
+
+ private Set<String> destinations=new java.util.LinkedHashSet<>();
+
+ /** True if this field was defined implicitly */
+ private boolean implicit=false;
+
+ /** The list of properties for this summary field */
+ private List<Property> properties = new ArrayList<>();
+
+ /** Creates a summary field with NONE as transform */
+ public SummaryField(String name, DataType type) {
+ this(name,type,SummaryTransform.NONE);
+ }
+
+ /** Creates a summary field with NONE as transform */
+ public SummaryField(Field field) {
+ this(field,SummaryTransform.NONE);
+ }
+
+
+ public SummaryField(Field field,SummaryTransform transform) {
+ this(field.getName(), field.getDataType(), transform);
+ }
+
+ public SummaryField(String name,DataType type,SummaryTransform transform) {
+ super(name, type);
+ this.transform=transform;
+ }
+
+ public void setImplicit(boolean implicit) { this.implicit=implicit; }
+
+ @Override // override to make public
+ public void setDataType(DataType type) {
+ super.setDataType(type);
+ }
+
+ public boolean isImplicit() { return implicit; }
+
+ public void setTransform(SummaryTransform transform) {
+ this.transform=transform;
+ if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) {
+ // This is the kind of logic we want to have in processing,
+ // but can't because of deriveDocuments mode, which doesn't run
+ // processing.
+ setVsmCommand(VsmCommand.FLATTENJUNIPER);
+ }
+ }
+
+ public SummaryTransform getTransform() { return transform; }
+
+ /** Returns the first source field of this, or null if the source field is not present */
+ public String getSourceField() {
+ String sourceName=getName();
+ if (sources.size()>0)
+ sourceName=sources.iterator().next().getName();
+ return sourceName;
+ }
+
+ public void addSource(String name) {
+ sources.add(new Source(name));
+ }
+
+ public void addSource(Source source) {
+ sources.add(source);
+ }
+
+ public Iterator<Source> sourceIterator() {
+ return sources.iterator();
+ }
+
+ public int getSourceCount() {
+ return sources.size();
+ }
+
+ /** Returns a modifiable set of the sources of this */
+ public Set<Source> getSources() { return sources; }
+
+ /** Returns the first source name of this, or the field name if no source has been set */
+ public String getSingleSource() {
+ if (sources.size()==0) return getName();
+ return sources.iterator().next().getName();
+ }
+
+ public void addDestination(String name) {
+ destinations.add(name);
+ }
+
+ public final void addDestinations(Iterable<String> names) {
+ for (String name : names) {
+ addDestination(name);
+ }
+ }
+
+ /** Returns an modifiable view of the destination set owned by this */
+ public Set<String> getDestinations() {
+ return destinations;
+ }
+
+ private String toString(Collection<?> collection) {
+ StringBuffer buffer=new StringBuffer();
+ for (Iterator<?> i=collection.iterator(); i.hasNext(); ) {
+ buffer.append(i.next().toString());
+ if (i.hasNext())
+ buffer.append(", ");
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Returns a summary field which merges the settings in the given field
+ * into this field
+ *
+ * @param merge the field to merge with this, if null, the merged field is
+ * <code>this</code> field
+ * @throws RuntimeException if the two fields can not be merged
+ */
+ public SummaryField mergeWith(SummaryField merge) {
+ if (merge==null) return this;
+ if (this.isImplicit()) return merge;
+ if (merge.isImplicit()) return this;
+
+ if (!merge.getName().equals(getName()))
+ throw new IllegalArgumentException(merge + " conflicts with " + this +
+ ": different names");
+
+ if (!merge.getTransform().equals(getTransform()))
+ throw new IllegalArgumentException(merge + " conflicts with " + this +
+ ": different transforms");
+
+ if (!merge.getDataType().equals(getDataType()))
+ throw new IllegalArgumentException(merge + " conflicts with " + this +
+ ": different types");
+
+ if (!merge.isImplicit())
+ setImplicit(false);
+
+ if (isHeadOf(this.sourceIterator(),merge.sourceIterator())) {
+ // Ok
+ }
+ else if (isHeadOf(merge.sourceIterator(),this.sourceIterator())) {
+ sources=new LinkedHashSet<>(merge.sources);
+ }
+ else {
+ throw new IllegalArgumentException(merge + " conflicts with " + this +
+ ": on source list must be the start of the other");
+ }
+
+ destinations.addAll(merge.destinations);
+
+ return this;
+ }
+
+ public boolean hasSource(String name) {
+ for (Source s : sources) {
+ if (s.getName().equals(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns true if the second list is the start of the first list
+ */
+ private boolean isHeadOf(Iterator<?> full, Iterator<?> head) {
+ while (head.hasNext()) {
+ if (!full.hasNext()) return false;
+
+ if (!full.next().equals(head.next())) return false;
+ }
+ return true;
+ }
+
+ private String getDestinationString()
+ {
+ StringBuilder destinationString = new StringBuilder("destinations(");
+ for (String destination : destinations) {
+ destinationString.append(destination).append(" ");
+ }
+ destinationString.append(")");
+ return destinationString.toString();
+ }
+
+ public String toString() {
+ return
+ "summary field '" + getName() + ' ' + getDestinationString() +
+ "' [type: '" + getDataType().getName() +
+ "' transform: '" + transform +
+ "', source: '" + toString(sources) +
+ "', to '" + toString(destinations) + "']";
+ }
+
+ /** returns a string which aids locating this field in the source search definition */
+ public String toLocateString() {
+ return "'summary " + getName() + " type " + toLowerCase(getDataType().getName()) + "' in '" + getDestinationString() + "'";
+ }
+
+ public SummaryField clone() {
+ try {
+ SummaryField clone=(SummaryField)super.clone();
+ if (this.sources!=null)
+ clone.sources=new LinkedHashSet<>(this.sources);
+ if (this.destinations!=null)
+ clone.destinations=new LinkedHashSet<>(destinations);
+ return clone;
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Programming error");
+ }
+ }
+
+ public VsmCommand getVsmCommand() {
+ return vsmCommand;
+ }
+
+ public void setVsmCommand(VsmCommand vsmCommand) {
+ this.vsmCommand = vsmCommand;
+ }
+
+ /** Adds a property to this summary field */
+ public void addProperty(String name, String value) {
+ properties.add(new Property(name, value));
+ }
+
+ public List<Property> getProperties() {
+ return properties;
+ }
+
+ /**
+ * The command used when using data from this SummaryField to generate StreamingSummary config (vsmsummary).
+ * Not used for ordinary Summary config.
+ * @author vegardh
+ *
+ */
+ public enum VsmCommand {
+ NONE("NONE"),
+ FLATTENSPACE("FLATTENSPACE"),
+ FLATTENJUNIPER("FLATTENJUNIPER");
+
+ private String cmd="";
+ private VsmCommand(String cmd) {
+ this.cmd=cmd;
+ }
+ @Override
+ public String toString() {
+ return cmd;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
new file mode 100644
index 00000000000..05092d50951
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+/**
+ * A value class representing a search time
+ * transformation on a summary field.
+ *
+ * @author bratseth
+ */
+public enum SummaryTransform {
+
+ NONE("none"),
+ ATTRIBUTE("attribute"),
+ BOLDED("bolded"),
+ DISTANCE("distance"),
+ DYNAMICBOLDED("dynamicbolded"),
+ DYNAMICTEASER("dynamicteaser"),
+ POSITIONS("positions"),
+ RANKFEATURES("rankfeatures"),
+ SUMMARYFEATURES("summaryfeatures"),
+ TEXTEXTRACTOR("textextractor"),
+ GEOPOS("geopos");
+
+ private String name;
+
+ private SummaryTransform(String name) {
+ this.name=name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /** Returns the bolded version of this transform if possible, throws if not */
+ public SummaryTransform bold() {
+ switch (this) {
+ case NONE:
+ case BOLDED:
+ return BOLDED;
+
+ case DYNAMICBOLDED:
+ case DYNAMICTEASER:
+ return DYNAMICBOLDED;
+
+ default:
+ throw new IllegalArgumentException("Can not bold a '" + this + "' field.");
+ }
+ }
+
+ /** Returns the unbolded version of this transform */
+ public SummaryTransform unbold() {
+ switch (this) {
+ case NONE:
+ case BOLDED:
+ return NONE;
+
+ case DYNAMICBOLDED:
+ return DYNAMICTEASER;
+
+ default:
+ return this;
+ }
+ }
+
+ /** Returns whether this value is bolded */
+ public boolean isBolded() {
+ return this==BOLDED || this==DYNAMICBOLDED;
+ }
+
+ /** Whether this is dynamically generated, both teasers and bolded fields are dynamic */
+ public boolean isDynamic() {
+ return this==BOLDED || this==DYNAMICBOLDED || this==DYNAMICTEASER;
+ }
+
+ /** Returns whether this is a teaser, not the complete field value */
+ public boolean isTeaser() {
+ return this==DYNAMICBOLDED || this==DYNAMICTEASER;
+ }
+
+ /** Returns whether this transform always gets its value by accessing memory only */
+ public boolean isInMemory() {
+ switch (this) {
+ case ATTRIBUTE:
+ case DISTANCE:
+ case POSITIONS:
+ case GEOPOS:
+ case RANKFEATURES:
+ case SUMMARYFEATURES:
+ return true;
+
+ default:
+ return false;
+ }
+ }
+
+ public String toString() {
+ return name;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
new file mode 100644
index 00000000000..13d85c3a955
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -0,0 +1,516 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.config.model.api.PortInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.filedistribution.PathDoesNotExistException;
+
+import java.util.*;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Superclass for all Processes.
+ *
+ * @author gjoranv
+ */
+public abstract class AbstractService extends AbstractConfigProducer<AbstractConfigProducer<?>> implements Service {
+
+ private static final long serialVersionUID = 1L;
+
+ // The physical host this Service runs on.
+ private HostResource hostResource = null;
+
+ /**
+ * Identifier that ensures that multiple instances of the same
+ * Service subclass will have unique names on the host. The first
+ * instance of one kind of Service will have the id 1, and the id
+ * will increase by 1 for each new instance.
+ * TODO: Do something more intelligent in Host?
+ */
+ private int id = 0;
+
+ /** The actual base port for this Service. */
+ private int basePort;
+
+ /** The ports allocated to this Service. */
+ private List<Integer> ports = new ArrayList<>();
+
+ /** The optional JVM execution args for this Service. */
+ // Please keep non-null, as passed to command line in service startup
+ private String jvmArgs = "";
+
+ /** The optional PRELOAD libraries for this Service. */
+ // Please keep non-null, as passed to command line in service startup
+ private String preload = Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so";
+
+ // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump.
+ private long mmapNoCoreLimit = -1l;
+
+ /** The ports metainfo object */
+ protected PortsMeta portsMeta = new PortsMeta();
+
+ /**
+ * Custom properties that a service may populate to communicate
+ * more key/value pairs to the service-list dump.
+ * Supported key datatypes are String, and values may be String or Integer.
+ */
+ private HashMap<String, Object> serviceProperties = new LinkedHashMap<>();
+
+ /** The affinity properties of this service. */
+ private Optional<Affinity> affinity = Optional.empty();
+
+ private boolean initialized = false;
+
+ /**
+ * Preferred constructor when building from XML. Use this if you are building
+ * in doBuild() in an AbstractConfigProducerBuilder.
+ * build() will call initService() in that case, after setting hostalias and baseport.
+ * @param parent Parent config producer in the model tree.
+ * @param name Name of this service.
+ */
+ public AbstractService(AbstractConfigProducer parent, String name) {
+ super(parent, name);
+ }
+
+ /**
+ * Only used for testing. Stay away.
+ * @param name Name of this service.
+ */
+ public AbstractService(String name) {
+ super(name);
+ }
+
+ /**
+ * Distribute affinity on a collection of services. Services that are located on the same host
+ * will be assigned a specific cpu socket on that host.
+ *
+ * @param services A {@link Collection} of services of the same type, not necessarily on the same host.
+ */
+ public static <SERVICE extends AbstractService> void distributeCpuSocketAffinity(Collection<SERVICE> services) {
+ Map<HostResource, List<SERVICE>> affinityMap = new HashMap<>();
+ for (SERVICE service : services) {
+ if (!affinityMap.containsKey(service.getHostResource())) {
+ affinityMap.put(service.getHostResource(), new ArrayList<SERVICE>());
+ }
+ int cpuSocket = affinityMap.get(service.getHostResource()).size();
+ affinityMap.get(service.getHostResource()).add(service);
+ service.setAffinity(new Affinity.Builder().cpuSocket(cpuSocket).build());
+ }
+ }
+
+ /**
+ * Helper method to avoid replicating code.
+ *
+ * @param hostResource The physical host on which this service should run.
+ * @param userPort The wanted port given by the user.
+ */
+ private void initService(HostResource hostResource, int userPort) {
+ if (initialized) {
+ throw new IllegalStateException("Service '" + getConfigId() + "' already initialized.");
+ }
+ if (hostResource == null) {
+ throw new RuntimeException("No host found for service '" + getServiceName() + "'. " +
+ "The hostalias is probably missing from hosts.xml.");
+ }
+ id = getIndex(hostResource);
+ ports = hostResource.allocateService(this, getInstanceWantedPort(userPort));
+ initialized = true;
+ }
+
+ /**
+ * Called by builder class which has not given the host or port in a constructor, hence
+ * initService is not yet run for this.
+ */
+ public void initService() {
+ initService(this.hostResource, this.basePort);
+ }
+
+ /**
+ * Returns the desired base port for the first instance of the
+ * service type. Returns '0' as default, which means that the
+ * service type should use the default port allocation mechanism.
+ *
+ * @return The desired base port for the first instance of the service type.
+ */
+ public int getWantedPort() {
+ return 0;
+ }
+
+ /**
+ * Returns the desired base port for this service instance, '0' if
+ * it should use the default port allocation mechanism.
+ *
+ * @param userWantedPort The wanted port given by the user.
+ * @return The desired base port for this service instance, '0' by default
+ */
+ private int getInstanceWantedPort(int userWantedPort) {
+ int wantedPort = 0;
+ if (userWantedPort == 0) {
+ if (requiresWantedPort())
+ wantedPort = getWantedPort();
+ else if (getWantedPort() > 0)
+ wantedPort = getWantedPort() + ((getId() - 1) * getPortCount());
+ } else {
+ // User defined from spec
+ wantedPort = userWantedPort;
+/* if ((wantedPort >= Host.BASE_PORT) &&
+ (wantedPort <= (Host.BASE_PORT + Host.MAX_PORTS))) {
+ throw new RuntimeException
+ ("Attribute 'basePort=" + wantedPort +
+ "' is not allowed to be inside Vespa's reserved port range "
+ + Host.BASE_PORT + "-"
+ + (Host.BASE_PORT + Host.MAX_PORTS) + ".");
+ }
+*/
+ }
+ return wantedPort;
+ }
+
+ /**
+ * Override if the desired base port (returned by getWantedPort()) is the only allowed base port.
+ *
+ * @return false by default
+ */
+ public boolean requiresWantedPort() {
+ return false;
+ }
+
+ /**
+ * Override if the services does not require consecutive port numbers. I.e. if any ports
+ * in addition to the baseport should be allocated from Vespa's default port range.
+ *
+ * @return true by default
+ */
+ public boolean requiresConsecutivePorts() {
+ return true;
+ }
+
+ /**
+ * Gets the ports metainfo object. The service implementation
+ * must populate this object in the constructor.
+ */
+ public PortsMeta getPortsMeta() {
+ return portsMeta;
+ }
+
+ /**
+ * Computes and returns the i'th port for this service, based on
+ * this Service's baseport.
+ *
+ * @param i The offset from 'basePort' of the port to return
+ * @return the i'th port relative to the base port.
+ * @throws IllegalStateException if i is out of range.
+ */
+ public int getRelativePort(int i) {
+ if (ports.size() < 1) {
+ throw new IllegalStateException
+ ("Requested port with offset " + i + " for service that " +
+ "has not reserved any ports: " + this);
+ }
+ if (i >= ports.size()) {
+ throw new IllegalStateException
+ ("Requested port with offset " + i + " for service that " +
+ "only has reserved " + ports.size() + " ports: " + this);
+ }
+ return ports.get(i);
+ }
+
+ /**
+ * Must be overridden by services that should be started by
+ * config-sentinel. The returned value will be used in
+ * config-sentinel configuration. Returns null by default.
+ *
+ * @return null by default.
+ */
+ public String getStartupCommand() {
+ return null;
+ }
+
+ public Optional<String> getPreShutdownCommand() {
+ return Optional.empty();
+ }
+
+ /**
+ * Tells if this service should be autostarted by
+ * config-sentinel. Returned value will be used to configure the
+ * config-sentinel.
+ *
+ * @return true by default.
+ */
+ public boolean getAutostartFlag() {
+ return true;
+ }
+
+ /**
+ * Tells if this service should be autorestarted by
+ * config-sentinel. Returned value will be used to configure the
+ * config-sentinel.
+ *
+ * @return true by default.
+ */
+ public boolean getAutorestartFlag() {
+ return true;
+ }
+
+ /**
+ * Returns the name that identifies this service for the config-sentinel.
+ *
+ * @return the name that identifies this service for the config-sentinel.
+ */
+ public String getServiceName() {
+ return getServiceType() + ((id == 1) ? "" : Integer.toString(id));
+ }
+
+ /**
+ * Returns the type of service. This is the class name without the
+ * package prefix by default.
+ */
+ public String getServiceType() {
+ return toLowerCase(getShortClassName());
+ }
+
+ /**
+ * Strips the package prefix and returns the short classname.
+ *
+ * @return classname without package prefix.
+ */
+ private String getShortClassName() {
+ Class myClass = getClass();
+ Package myPackage = myClass.getPackage();
+ return myClass.getName().substring(1 + myPackage.getName().length());
+ }
+
+ /**
+ * @return the physical host on which this service runs.
+ */
+ public Host getHost() {
+ if (hostResource != null) {
+ return hostResource.getHost();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * @return The hostname on which this service runs.
+ */
+ public String getHostName() {
+ return hostResource.getHostName();
+ }
+
+ /**
+ * @return The id (index) of this service on the host where it runs
+ */
+ public int getId() {
+ return id;
+ }
+
+ /**
+ * Computes a number that identifies the service on the given
+ * host. The number of services of the same type (Class) is
+ * counted and the number is returned.
+ *
+ * @param host the host on which the service will run
+ * @return id number for the given service.
+ */
+ // TODO: Do something more intelligent in the Host class..?
+ protected int getIndex(HostResource host) {
+ int i = 0;
+ for (Service s : host.getServices()) {
+ //if (s.getClass().equals(getClass()) && (s != this)) {
+ if (s.getServiceType().equals(getServiceType()) && (s != this)) {
+ i++;
+ }
+ }
+ return i + 1;
+ }
+
+ @Override
+ public ServiceInfo getServiceInfo() {
+ Set<PortInfo> portInfos = new LinkedHashSet<>();
+ for (int i = 0; i < portsMeta.getNumPorts(); i++) {
+ portInfos.add(new PortInfo(ports.get(i), new LinkedHashSet<>(portsMeta.getTagsAt(i))));
+ }
+ Map<String, String> properties = new LinkedHashMap<>();
+ for (Map.Entry<String, Object> prop : serviceProperties.entrySet()) {
+ properties.put(prop.getKey(), prop.getValue().toString());
+ }
+ return new ServiceInfo(getServiceName(), getServiceType(), portInfos, properties, getConfigId(), getHostName());
+ }
+
+ /**
+ * Sets a service property value for the given key.
+ *
+ * @param key a key used for this property
+ * @param value a String value associated with the key
+ * @return this service
+ */
+ public AbstractService setProp(String key, String value) {
+ serviceProperties.put(key, value);
+ return this;
+ }
+
+ /**
+ * Sets a service property value for the given key.
+ *
+ * @param key a key used for this property
+ * @param value an Integer value associated with the key
+ * @return this service
+ */
+ public AbstractService setProp(String key, Integer value) {
+ serviceProperties.put(key, value);
+ return this;
+ }
+
+ /**
+ * Gets a service property value mapped to the given key
+ * as a String, or null if no such key exists.
+ *
+ * @param key a key used for lookup in the service properties
+ * @return the associated String value for the given key, or null
+ */
+ public String getServicePropertyString(String key) {
+ return getServicePropertyString(key, null);
+ }
+
+ public String getServicePropertyString(String key, String defStr) {
+ Object result = serviceProperties.get(key);
+ return (result == null) ? defStr : result.toString();
+ }
+
+ /** Optional execution args for this service */
+ public String getJvmArgs() {
+ return jvmArgs;
+ }
+ public void setJvmArgs(String args) {
+ jvmArgs = (args == null) ? "" : args;
+ }
+ public void appendJvmArgs(String args) {
+ if ((args != null) && ! "".equals(args)) {
+ setJvmArgs(jvmArgs + getSeparator(jvmArgs) + args);
+ }
+ }
+ private static String getSeparator(String current) {
+ return ("".equals(current)) ? "" : " ";
+ }
+ public void prependJvmArgs(String args) {
+ if ((args != null) && ! "".equals(args)) {
+ setJvmArgs(args + getSeparator(jvmArgs) + jvmArgs);
+ }
+ }
+ public String getPreLoad() { return preload; }
+ public void setPreLoad(String preload) {
+ this.preload = preload;
+ }
+ public long getMMapNoCoreLimit() { return mmapNoCoreLimit; }
+ public void setMMapNoCoreLimit(long noCoreLimit) {
+ this.mmapNoCoreLimit = noCoreLimit;
+ }
+
+ public String getMMapNoCoreEnvVariable() {
+ return (getMMapNoCoreLimit() >= 0l)
+ ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " "
+ : "";
+ }
+
+ /**
+ * WARNING: should only be called before initService(), otherwise call at own risk!
+ */
+ public void setBasePort(int wantedPort) {
+ if (initialized && getPortCount() > 0) {
+ // This means initServices has been called already, so you are being nasty, trying to change ports!
+ // Try to allocate the new ports if they are available.
+ if (!hostResource.isPortRangeAvailable(wantedPort, 1) ||
+ (requiresConsecutivePorts() && ! hostResource.isPortRangeAvailable(wantedPort, getPortCount())))
+ throw new IllegalStateException("For service with id " + getConfigId() + ": setBasePort is called after initService, and ports are not available.");
+
+ hostResource.deallocatePorts(this);
+ ports = hostResource.allocatePorts(this, getInstanceWantedPort(wantedPort));
+ }
+
+ this.basePort = wantedPort;
+ }
+
+ /** Temporary hack: reserve port with index 0
+ * Must be done this way since the system test framework
+ * currently uses the first port as container http port.
+ */
+ public void reservePortPrepended(int port) {
+ hostResource.reservePort(this, port);
+ ports.add(0, port);
+ }
+
+ public void setHostResource(HostResource hostResource) {
+ this.hostResource = hostResource;
+ }
+
+ public boolean isInitialized() {
+ return initialized;
+ }
+
+ /**
+ * Add the given file to the application's file distributor.
+ *
+ * @param relativePath path to the file, relative to the app package.
+ * @return the file reference hash
+ */
+ public FileReference sendFile(String relativePath) {
+ try {
+ return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost());
+ } catch (PathDoesNotExistException e) {
+ throw new RuntimeException("File does not exist: '" + relativePath + "'.");
+ }
+ }
+
+ /**
+ * Sets up this service to be included when generating monitoring config.
+ * The ymon service name used will be {@link #getServiceType()}
+ */
+ public void monitorService() {
+ monitorService(getServiceType());
+ }
+
+ /**
+ * Sets up this service to be included when generating ymon config.
+ * @param ymonServiceName the ymon service name to be used
+ */
+ public void monitorService(String ymonServiceName) {
+ setProp("ymonService", ymonServiceName);
+ }
+
+ /**
+ *
+ * The service HTTP port for health status
+ * @return portnumber
+ */
+ public int getHealthPort() {return -1;}
+
+ /**
+ * Overridden by subclasses. List of default dimensions to be added to this services metrics
+ * @return The default dimensions for this service
+ */
+ public HashMap<String, String> getDefaultMetricDimensions(){
+ return new LinkedHashMap<>();
+ }
+
+ // For testing
+ public int getNumPortsAllocated() {
+ return ports.size();
+ }
+
+ public HostResource getHostResource() {
+ return hostResource;
+ }
+
+ public Optional<Affinity> getAffinity() {
+ return affinity;
+ }
+
+ public void setAffinity(Affinity affinity) {
+ this.affinity = Optional.ofNullable(affinity);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java b/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java
new file mode 100644
index 00000000000..d621c299a5d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+/**
+ * Represents a set of affinities that can be expressed by a service. Currently only supports
+ * CPU socket affinity.
+ *
+ * @author lulf
+ * @since 5.12
+ */
+public class Affinity {
+ private final int cpuSocket;
+
+ private Affinity(int cpuSocket) {
+ this.cpuSocket = cpuSocket;
+ }
+
+ public int cpuSocket() {
+ return cpuSocket;
+ }
+
+ public static Affinity none() {
+ return new Builder().build();
+ }
+
+ public static class Builder {
+ private int cpuSocket = -1;
+ public Builder cpuSocket(int cpuSocket) {
+ this.cpuSocket = cpuSocket;
+ return this;
+ }
+
+ public Affinity build() {
+ return new Affinity(cpuSocket);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Client.java b/config-model/src/main/java/com/yahoo/vespa/model/Client.java
new file mode 100644
index 00000000000..15685f5f669
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Client.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * This is a placeholder config producer that makes global configuration available through a single identifier. This
+ * is added directly to the {@link ApplicationConfigProducerRoot} producer, and so can be accessed by the simple "client" identifier.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Client extends AbstractConfigProducer {
+
+ /**
+ * Constructs a client config producer that is added as a child to
+ * the given config producer.
+ *
+ * @param parent The parent config producer.
+ */
+ public Client(AbstractConfigProducer parent) {
+ super(parent, "client");
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
new file mode 100644
index 00000000000..852e4e73331
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.ConfigInstance.Builder;
+import com.yahoo.config.model.producer.UserConfigRepo;
+
+/**
+ * Interface that should be implemented by all config producing modules
+ * in the vespa model.
+ *
+ * @author gjoranv
+ */
+public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer {
+
+ /**
+ * @return the configId of this ConfigProducer.
+ */
+ public String getConfigId();
+
+ /**
+ * @return The one and only HostSystem of the root node
+ */
+ public HostSystem getHostSystem();
+
+ /** Returns the user configs of this */
+ public UserConfigRepo getUserConfigs();
+
+ /**
+ * @return this ConfigProducer's children (only 1st level)
+ */
+ public Map<String,? extends ConfigProducer> getChildren();
+
+ /**
+ * @return a List of all Services that are descendants to this ConfigProducer
+ */
+ public List<Service> getDescendantServices();
+
+ /**
+ * Writes files that need to be written. The files will usually
+ * only be written when the Vespa model is generated through the
+ * deploy-application script.
+ * gv: This is primarily intended for debugging.
+ * @param directory directory to write files to
+ * @throws java.io.IOException if writing fails
+ */
+ public void writeFiles(File directory) throws IOException;
+
+ /**
+ * Dump the three of config producers to the specified stream.
+ * @param out The stream to print to, e.g. System.out
+ */
+ public void dump(PrintStream out);
+
+ /**
+ * Build config from this and all parent ConfigProducers,
+ * such that the root node's config will be added first, and this
+ * ConfigProducer's config last in the returned builder.
+ *
+ * @param builder The builder implemented by the concrete ConfigInstance class
+ * @return true if a model config producer was found, so config was applied
+ */
+ boolean cascadeConfig(Builder builder);
+
+ /**
+ * Adds user config override from this ConfigProducer to the existing builder
+ *
+ * @param builder The ConfigBuilder to add user config overrides.
+ * @return true if overrides were added, false if not.
+ */
+ public boolean addUserConfig(ConfigInstance.Builder builder);
+
+ /**
+ * check constraints depending on the state of the vespamodel graph.
+ * When overriding, you must invoke super.
+ */
+ public void validate() throws Exception;
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java
new file mode 100644
index 00000000000..d0bb89beca7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.filedistribution.FileDistributor;
+
+import java.util.Set;
+
+
+/**
+ * Intended to be used as an external interface to the vespa model root.
+ *
+ * @author tonytv
+ */
+public interface ConfigProducerRoot extends ConfigProducer {
+
+ /**
+ * Adds the given producer (at any depth level) as descendant to this root nodes.
+ *
+ * @param id string id of descendant
+ * @param descendant the producer to add to this root node
+ */
+ void addDescendant(String id, AbstractConfigProducer descendant);
+
+ /**
+ * @return an unmodifiable copy of the set of configIds in this root.
+ */
+ Set<String> getConfigIds();
+
+ ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId);
+
+ /**
+ * Resolves config of the given type and config id.
+ * @param clazz The type of config
+ * @param configId The config id
+ * @return A config instance of the given type
+ */
+ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId);
+
+ /**
+ * Get the global deploy state of this model.
+ */
+ DeployState getDeployState();
+
+ FileDistributor getFileDistributor();
+
+ Admin getAdmin();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java
new file mode 100644
index 00000000000..22fdca6af50
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+/**
+ * There is one config proxy running on each Vespa host, and one instance of
+ * this class is therefore created by each instance of class {@link
+ * com.yahoo.vespa.model.Host}.
+ *
+ * NOTE: The Config proxy is not started by the config system, and
+ * does not receive any config. It's included here so we know what host
+ * it runs on, and to give an error message if another service tries
+ * to reserve the port it is using.
+ *
+ * @author Vidar Larsen
+ * @author Harald Musum
+ */
+public class ConfigProxy extends AbstractService {
+
+ /**
+ * Creates a new ConfigProxy instance.
+ *
+ * @param host hostname
+ */
+ public ConfigProxy(Host host) {
+ super(host, "configproxy");
+ portsMeta.on(0).tag("rpc").tag("client").tag("status").tag("rpc").tag("admin");
+ setProp("clustertype", "hosts");
+ setProp("clustername", "admin");
+ }
+
+ /**
+ * Returns the desired base port for this service.
+ */
+ public int getWantedPort() { return 19090; }
+
+ /**
+ * The desired base port is the only allowed base port.
+ */
+ public boolean requiresWantedPort() { return true; }
+
+ /**
+ * ConfigProxy needs one rpc client port.
+ *
+ * @return The number of ports reserved by the config proxy
+ */
+ public int getPortCount() { return 1; }
+
+ /**
+ * The config proxy is not started by the config system!
+ */
+ public boolean getAutostartFlag() { return false; }
+
+ /**
+ * The config proxy is not started by the config system!
+ */
+ public boolean getAutorestartFlag() { return false; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
new file mode 100644
index 00000000000..adcb93bc539
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.cloud.config.SentinelConfig;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+
+/**
+ * There is one config-sentinel running on each Vespa host, and one
+ * instance of this class is therefore created by each instance of
+ * class {@link Host}.
+ *
+ * @author gjoranv
+ */
+public class ConfigSentinel extends AbstractService implements SentinelConfig.Producer {
+
+ private final ApplicationId applicationId;
+ private final Zone zone;
+
+ /**
+ * Constructs a new ConfigSentinel for the given host.
+ *
+ * @param host Physical host on which to run.
+ */
+ public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone) {
+ super(host, "sentinel");
+ this.applicationId = applicationId;
+ this.zone = zone;
+ portsMeta.on(0).tag("rpc").tag("notyet");
+ portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state");
+ setProp("clustertype", "hosts");
+ setProp("clustername", "admin");
+ }
+
+ /**
+ * Returns the desired base port for this service.
+ */
+ public int getWantedPort() { return 19097; }
+
+ /**
+ * The desired base port is the only allowed base port.
+ */
+ public boolean requiresWantedPort() { return true; }
+
+ /**
+ * @return The number of ports reserved by the Sentinel.
+ */
+ public int getPortCount() { return 2; }
+
+ @Override
+ public int getHealthPort() {return getRelativePort(1); }
+
+ /**
+ * Overrides parent method as this is named config-sentinel and not configsentinel all over Vespa
+ * @return service type for config-sentinel
+ */
+ public String getServiceType(){
+ return "config-sentinel";
+ }
+
+ @Override
+ public void getConfig(SentinelConfig.Builder builder) {
+ builder.application(getApplicationConfig());
+ for (Service s : getHostResource().getServices()) {
+ if (s.getStartupCommand() != null) {
+ builder.service(getServiceConfig(s));
+ }
+ }
+ }
+
+ private SentinelConfig.Application.Builder getApplicationConfig() {
+ SentinelConfig.Application.Builder builder = new SentinelConfig.Application.Builder();
+ builder.tenant(applicationId.tenant().value());
+ builder.name(applicationId.application().value());
+ builder.environment(zone.environment().value());
+ builder.region(zone.region().value());
+ builder.instance(applicationId.instance().value());
+ return builder;
+ }
+
+ private SentinelConfig.Service.Builder getServiceConfig(Service s) {
+ SentinelConfig.Service.Builder serviceBuilder = new SentinelConfig.Service.Builder();
+ serviceBuilder.command(s.getStartupCommand());
+ serviceBuilder.name(s.getServiceName());
+ serviceBuilder.autostart(s.getAutostartFlag());
+ serviceBuilder.autorestart(s.getAutorestartFlag());
+ serviceBuilder.id(s.getConfigId());
+ serviceBuilder.affinity(getServiceAffinity(s));
+ setPreShutdownCommand(serviceBuilder, s);
+ return serviceBuilder;
+ }
+
+ private void setPreShutdownCommand(SentinelConfig.Service.Builder serviceBuilder, Service service) {
+ if (service.getPreShutdownCommand().isPresent()) {
+ serviceBuilder.preShutdownCommand(service.getPreShutdownCommand().get());
+ }
+ }
+
+
+ private SentinelConfig.Service.Affinity.Builder getServiceAffinity(Service s) {
+ SentinelConfig.Service.Affinity.Builder builder = new SentinelConfig.Service.Affinity.Builder();
+ if (s.getAffinity().isPresent()) {
+ builder.cpuSocket(s.getAffinity().get().cpuSocket());
+ }
+ return builder;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
new file mode 100644
index 00000000000..99109a881a1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.logging.Level;
+import com.yahoo.cloud.config.SentinelConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * A physical host, running a set of services.
+ * The identity of a host is its hostname. Hosts are comparable on their host name.
+ *
+ * @author gjoranv
+ */
+public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?>> implements SentinelConfig.Producer, Comparable<Host> {
+
+ private ConfigSentinel configSentinel = null;
+ private final String hostname;
+ private final boolean multitenant;
+
+ /**
+ * Constructs a new Host instance.
+ *
+ * @param parent parent AbstractConfigProducer in the config model.
+ * @param hostname hostname for this host.
+ */
+ public Host(AbstractConfigProducer parent, String hostname) {
+ this(parent, hostname, false);
+ }
+
+ private Host(AbstractConfigProducer parent, String hostname, boolean multitenant) {
+ super(parent, hostname);
+ Objects.requireNonNull(hostname, "The host name of a host cannot be null");
+ this.multitenant = multitenant;
+ this.hostname = hostname;
+ if (parent instanceof HostSystem) {
+ checkName((HostSystem) parent, hostname);
+ }
+ }
+
+ private void checkName(HostSystem parent, String hostname) {
+ // Give a warning if the host does not exist
+ if (! parent.getIp(hostname).equals("0.0.0.0")) {
+ // Host exists - warn if given hostname is not a fully qualified one.
+ String canonical=hostname;
+ try {
+ canonical = parent.getCanonicalHostname(hostname);
+ } catch (UnknownHostException e) {
+ deployLogger().log(Level.WARNING, "Unable to find canonical hostname of host: " + hostname);
+ }
+ if ((null != canonical) && (! hostname.equals(canonical))) {
+ deployLogger().log(Level.WARNING, "Host named '" + hostname + "' will not receive any config " +
+ "since it does not match its canonical hostname: " + canonical);
+ }
+ }
+ }
+
+ public static Host createMultitenantHost(AbstractConfigProducer parent, String hostname) {
+ return new Host(parent, hostname, true);
+ }
+
+ // For testing
+ Host(AbstractConfigProducer parent) {
+ super(parent, "testhost");
+ hostname = "testhost";
+ configSentinel = null;
+ multitenant = false;
+ }
+
+ public String getHostName() {
+ return hostname;
+ }
+
+ public boolean isMultitenant() {
+ return multitenant;
+ }
+
+ /**
+ * Returns the string representation of this Host object.
+ * @return The string representation of this Host object.
+ */
+ public String toString() {
+ return "host '" + getHostName() + "'";
+ }
+
+ @Override
+ public void writeFiles(File directory) throws IOException {
+ }
+
+ @Override
+ public void getConfig(SentinelConfig.Builder builder) {
+ // TODO (MAJOR_RELEASE): This shouldn't really be here, but we need to make sure users can upgrade if we change sentinel to use hosts/<hostname>/sentinel instead of hosts/<hostname>
+ // as config id. We should probably wait for a major release
+ if (configSentinel != null) {
+ configSentinel.getConfig(builder);
+ }
+ }
+
+ public void setConfigSentinel(ConfigSentinel configSentinel) {
+ this.configSentinel = configSentinel;
+ }
+
+ @Override
+ public int hashCode() { return hostname.hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) return true;
+ if ( ! (other instanceof Host)) return false;
+ return ((Host)other).hostname.equals(hostname);
+ }
+
+ @Override
+ public int compareTo(Host other) {
+ return this.hostname.compareTo(other.hostname);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
new file mode 100644
index 00000000000..782487ee12c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
@@ -0,0 +1,251 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+
+import java.util.*;
+import java.util.logging.Level;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+
+/**
+ * A host representation. The identity of this is the identity of its Host.
+ * TODO: Merge with {@link Host}
+ * Host resources are ordered by their host order.
+ *
+ * @author lulf
+ * @since 5.12
+ */
+public class HostResource implements Comparable<HostResource> {
+
+ public final static int BASE_PORT = 19100;
+ public final static int MAX_PORTS = 799;
+ private final Host host;
+
+ // Map from "sentinel name" to service
+ private final Map<String,Service> services = new LinkedHashMap<>();
+ private final Map<Integer, Service> portDB = new LinkedHashMap<>();
+
+ private int allocatedPorts = 0;
+
+ /**
+ * Create a new {@link HostResource} bound to a specific {@link com.yahoo.vespa.model.Host}.
+ *
+ * @param host {@link com.yahoo.vespa.model.Host} object to bind to.
+ */
+ public HostResource(Host host) {
+ this.host = host;
+ }
+
+ /**
+ * Return the currently bounded {@link com.yahoo.vespa.model.Host}.
+ * @return the {@link com.yahoo.vespa.model.Host} if bound, null if not.
+ */
+ public Host getHost() { return host; }
+
+ /**
+ * Returns the baseport of the first available port range of length numPorts,
+ * or 0 if there is no range of that length available.
+ *
+ * @param numPorts The length of the desired port range.
+ * @return The baseport of the first available range, or 0 if no range is available.
+ */
+ public int nextAvailableBaseport(int numPorts) {
+ int range = 0;
+ int port = BASE_PORT;
+ for (; port < BASE_PORT + MAX_PORTS && (range < numPorts); port++) {
+ if (portDB.containsKey(port)) {
+ range = 0;
+ continue;
+ }
+ range++;
+ }
+ return range == numPorts ?
+ port - range :
+ 0;
+ }
+
+ boolean isPortRangeAvailable(int start, int numPorts) {
+ int range = 0;
+ int port = start;
+ for (; port < BASE_PORT + MAX_PORTS && (range < numPorts); port++) {
+ if (portDB.containsKey(port)) {
+ return false;
+ }
+ range++;
+ }
+ return range == numPorts;
+ }
+
+ /**
+ * Adds service and allocates resources for it.
+ *
+ * @param service The Service to allocate resources for
+ * @param wantedPort the wanted port for this service
+ * @return The allocated ports for the Service.
+ */
+ List<Integer> allocateService(AbstractService service, int wantedPort) {
+ List<Integer> ports = allocatePorts(service, wantedPort);
+ assert (getService(service.getServiceName()) == null) :
+ ("There is already a service with name '" + service.getServiceName() + "' registered on " + this +
+ ". Most likely a programming error - all service classes must have unique names, even in different packages!");
+
+ services.put(service.getServiceName(), service);
+ return ports;
+ }
+
+ // TODO: make private when LocalApplication and all model services has stopped reallocating ports _after_ allocateServices has been called
+ List<Integer> allocatePorts(AbstractService service, int wantedPort) {
+ List<Integer> ports = new ArrayList<>();
+ if (service.getPortCount() < 1)
+ return ports;
+
+ int serviceBasePort = BASE_PORT + allocatedPorts;
+ if (wantedPort > 0) {
+ if (service.getPortCount() < 1) {
+ throw new RuntimeException(service + " wants baseport " + wantedPort +
+ ", but it has not reserved any ports, so it cannot name a desired baseport.");
+ }
+ if (service.requiresWantedPort() || canUseWantedPort(service, wantedPort, serviceBasePort))
+ serviceBasePort = wantedPort;
+ }
+
+ reservePort(service, serviceBasePort);
+ ports.add(serviceBasePort);
+
+ int remainingPortsStart = service.requiresConsecutivePorts() ?
+ serviceBasePort + 1:
+ BASE_PORT + allocatedPorts;
+ for (int i = 0; i < service.getPortCount() - 1; i++) {
+ int port = remainingPortsStart + i;
+ reservePort(service, port);
+ ports.add(port);
+ }
+ return ports;
+ }
+
+ // TODO: this is a hack to allow calling AbstractService.setBasePort _after_ the services has been initialized,
+ // i.e. modifying the baseport. Done by e.g. LocalApplication. Try to remove usage of this method!
+ void deallocatePorts(AbstractService service) {
+ for (Iterator<Map.Entry<Integer,Service>> i=portDB.entrySet().iterator(); i.hasNext();) {
+ Map.Entry<Integer, Service> e = i.next();
+ Service s = e.getValue();
+ if (s.equals(service))
+ i.remove();
+ }
+ }
+
+ private boolean canUseWantedPort(AbstractService service, int wantedPort, int serviceBasePort) {
+ for (int i = 0; i < service.getPortCount(); i++) {
+ int port = wantedPort + i;
+ if (portDB.containsKey(port)) {
+ AbstractService s = (AbstractService)portDB.get(port);
+ s.getRoot().getDeployState().getDeployLogger().log(Level.WARNING, service.getServiceName() +" cannot reserve port " + port + " on " +
+ this + ": Already reserved for " + s.getServiceName() +
+ ". Using default port range from " + serviceBasePort);
+ return false;
+ }
+ if (!service.requiresConsecutivePorts()) break;
+ }
+ return true;
+ }
+
+ /**
+ * Reserves the desired port for the given service, or throws as exception if the port
+ * is not available.
+ *
+ * @param service the service that wishes to reserve the port.
+ * @param port the port to be reserved.
+ */
+ void reservePort(AbstractService service, int port) {
+ if (portDB.containsKey(port)) {
+ portAlreadyReserved(service, port);
+ } else {
+ if (inVespasPortRange(port)) {
+ allocatedPorts++;
+ if (allocatedPorts > MAX_PORTS) {
+ noMoreAvailablePorts();
+ }
+ }
+ portDB.put(port, service);
+ }
+ }
+
+ private boolean inVespasPortRange(int port) {
+ return port >= BASE_PORT &&
+ port < BASE_PORT + MAX_PORTS;
+ }
+ private void portAlreadyReserved(AbstractService service, int port) {
+ AbstractService otherService = (AbstractService)portDB.get(port);
+ int nextAvailablePort = nextAvailableBaseport(service.getPortCount());
+ if (nextAvailablePort == 0) {
+ noMoreAvailablePorts();
+ }
+ String msg = (service.getClass().equals(otherService.getClass()) && service.requiresWantedPort())
+ ? "You must set port explicitly for all instances of this service type, except the first one. "
+ : "";
+ throw new RuntimeException(service.getServiceName() + " cannot reserve port " + port +
+ " on " + this + ": Already reserved for " + otherService.getServiceName() +
+ ". " + msg + "Next available port is: " + nextAvailablePort);
+ }
+
+
+ private void noMoreAvailablePorts() {
+ throw new RuntimeException
+ ("Too many ports are reserved in Vespa's port range (" +
+ BASE_PORT + ".." + (BASE_PORT+MAX_PORTS) + ") on " + this +
+ ". Move one or more services to another host, or outside this port range.");
+ }
+
+
+ /**
+ * Returns the service with the given "sentinel name" on this Host,
+ * or null if the name does not match any service.
+ *
+ * @param sentinelName the sentinel name of the service we want to return
+ * @return The service with the given sentinel name
+ */
+ public Service getService(String sentinelName) {
+ return services.get(sentinelName);
+ }
+
+ /**
+ * Returns a List of all services running on this Host.
+ * @return a List of all services running on this Host.
+ */
+ public List<Service> getServices() {
+ return new ArrayList<>(services.values());
+ }
+
+ public HostInfo getHostInfo() {
+ return new HostInfo(getHostName(), services.values().stream()
+ .map(service -> service.getServiceInfo())
+ .collect(Collectors.toSet()));
+ }
+
+ @Override
+ public String toString() {
+ return "host '" + host.getHostName() + "'";
+ }
+
+ public String getHostName() {
+ return host.getHostName();
+ }
+
+ @Override
+ public int hashCode() { return host.hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) return true;
+ if ( ! (other instanceof HostResource)) return false;
+ return ((HostResource)other).getHost().equals(this.getHost());
+ }
+
+ @Override
+ public int compareTo(HostResource other) {
+ return this.host.compareTo(other.host);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
new file mode 100644
index 00000000000..42feee08f43
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
@@ -0,0 +1,203 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.test.MockRoot;
+import com.yahoo.config.provision.*;
+import com.yahoo.net.HostName;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+/**
+ * The parent node for all Host instances, and thus accessible
+ * to enable services to get their Host.
+ *
+ * @author gjoranv
+ */
+public class HostSystem extends AbstractConfigProducer<Host> {
+
+ private Map<String,String> ipAddresses = new LinkedHashMap<>();
+ private Map<String,String> hostnames = new LinkedHashMap<>();
+
+ private final Map<String, HostResource> hostname2host = new LinkedHashMap<>();
+ private final HostProvisioner provisioner;
+ private final Map<HostResource, Set<ClusterMembership>> mapping = new LinkedHashMap<>();
+
+ public HostSystem(AbstractConfigProducer parent, String name, HostProvisioner provisioner) {
+ super(parent, name);
+ this.provisioner = provisioner;
+ }
+
+ /**
+ * Returns the host with the given hostname.
+ *
+ * @param name the hostname of the host.
+ * @return the host with the given hostname.
+ */
+ public HostResource getHostByHostname(String name) {
+ // TODO: please eliminate the following ugly hack
+ if ("localhost.fortestingpurposesonly".equals(name)) {
+ String localhost = "localhost";
+ if ( ! getChildren().containsKey(localhost)) {
+ new Host(this, localhost);
+ }
+ return new HostResource(getChildren().get(localhost));
+ }
+ return hostname2host.get(name);
+ }
+
+ /**
+ * Returns the canonical name of a given host. This will cache names for faster lookup.
+ *
+ * @param hostname the hostname to retrieve the canonical hostname for.
+ * @return The canonical hostname, or null if unable to resolve.
+ * @throws UnknownHostException if the hostname cannot be resolved
+ */
+ public String getCanonicalHostname(String hostname) throws UnknownHostException {
+ if ( ! hostnames.containsKey(hostname)) {
+ hostnames.put(hostname, lookupCanonicalHostname(hostname));
+ }
+ return hostnames.get(hostname);
+ }
+
+ /**
+ * Static helper method that looks up the canonical name of a given host.
+ *
+ * @param hostname the hostname to retrieve the canonical hostname for.
+ * @return The canonical hostname, or null if unable to resolve.
+ * @throws UnknownHostException if the hostname cannot be resolved
+ */
+ public static String lookupCanonicalHostname(String hostname) throws UnknownHostException {
+ return java.net.InetAddress.getByName(hostname).getCanonicalHostName();
+ }
+
+ /**
+ * Returns the if address of a host.
+ *
+ * @param hostname the hostname to retrieve the ip address for.
+ * @return The string representation of the ip-address.
+ */
+ public String getIp(String hostname) {
+ if (ipAddresses.containsKey(hostname)) return ipAddresses.get(hostname);
+
+ String ipAddress;
+ if (hostname.startsWith(MockRoot.MOCKHOST)) {
+ ipAddress = "0.0.0.0";
+ } else {
+ try {
+ InetAddress address = InetAddress.getByName(hostname);
+ ipAddress = address.getHostAddress();
+ } catch (java.net.UnknownHostException e) {
+ log.warning("Unable to find valid IP address of host: " + hostname);
+ ipAddress = "0.0.0.0";
+ }
+ }
+ ipAddresses.put(hostname, ipAddress);
+ return ipAddress;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (HostResource host : mapping.keySet()) {
+ sb.append(host).append(",");
+ }
+ if (sb.length() > 0) sb.deleteCharAt(sb.length() - 1);
+ return sb.toString();
+ }
+
+ public HostResource getHost(String hostAlias) {
+ HostSpec hostSpec = provisioner.allocateHost(hostAlias);
+ for (Map.Entry<HostResource, Set<ClusterMembership>> entrySet : mapping.entrySet()) {
+ HostResource resource = entrySet.getKey();
+ if (resource.getHostName().equals(hostSpec.hostname())) {
+ entrySet.getValue().add(hostSpec.membership().orElse(null));
+ return resource;
+ }
+ }
+ return addNewHost(hostSpec);
+ }
+
+ private HostResource addNewHost(HostSpec hostSpec) {
+ Host host = new Host(this, hostSpec.hostname());
+ HostResource hostResource = new HostResource(host);
+ hostname2host.put(host.getHostName(), hostResource);
+ Set<ClusterMembership> hostMemberships = new LinkedHashSet<>();
+ if (hostSpec.membership().isPresent())
+ hostMemberships.add(hostSpec.membership().get());
+ mapping.put(hostResource, hostMemberships);
+ return hostResource;
+ }
+
+ public List<HostResource> getHosts() {
+ return mapping.keySet().stream()
+ .filter(host -> !host.getHost().isMultitenant())
+ .collect(Collectors.toList());
+ }
+
+ public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) {
+ List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger));
+ // TODO: Let hostresource own a membership rather than using a map?
+ Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>();
+ for (HostSpec host : allocatedHosts) {
+ // This is needed for single node host provisioner to work in unit tests for hosted vespa applications.
+ Optional<HostResource> existingHost = getExistingHost(host);
+ if (existingHost.isPresent()) {
+ retAllocatedHosts.put(existingHost.get(), host.membership().orElse(null));
+ } else {
+ retAllocatedHosts.put(addNewHost(host), host.membership().orElse(null));
+ }
+ }
+ return retAllocatedHosts;
+ }
+
+ private Optional<HostResource> getExistingHost(HostSpec key) {
+ List<HostResource> hosts = mapping.keySet().stream()
+ .filter(resource -> resource.getHostName().equals(key.hostname()))
+ .collect(Collectors.toList());
+ if (hosts.isEmpty()) {
+ return Optional.empty();
+ } else {
+ return Optional.of(hosts.get(0));
+ }
+ }
+
+ public void addBoundHost(HostResource host) {
+ mapping.put(host, new LinkedHashSet<>());
+ hostname2host.put(host.getHostName(), host);
+ }
+
+ Map<HostSpec, Set<ClusterMembership>> getHostToServiceSpecMapping() {
+ Map<HostSpec, Set<ClusterMembership>> specMapping = new LinkedHashMap<>();
+ for (Map.Entry<HostResource, Set<ClusterMembership>> entrySet : mapping.entrySet()) {
+ if (!entrySet.getKey().getHost().isMultitenant()) {
+ Optional<ClusterMembership> membership = entrySet.getValue().stream().filter(m -> m != null).findFirst();
+ specMapping.put(new HostSpec(entrySet.getKey().getHostName(), membership), new LinkedHashSet<>(entrySet.getValue()));
+ }
+ }
+ return specMapping;
+ }
+
+ /** A provision logger which forwards to a deploy logger */
+ private static class ProvisionDeployLogger implements ProvisionLogger {
+
+ private final DeployLogger deployLogger;
+
+ public ProvisionDeployLogger(DeployLogger deployLogger) {
+ this.deployLogger = deployLogger;
+ }
+
+ @Override
+ public void log(Level level, String message) {
+ deployLogger.log(level, message);
+ }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java
new file mode 100644
index 00000000000..9c761e425a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java
@@ -0,0 +1,196 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.*;
+import com.yahoo.config.codegen.CNode;
+import com.yahoo.config.codegen.ConfigGenerator;
+import com.yahoo.config.codegen.InnerCNode;
+import com.yahoo.config.codegen.LeafCNode;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.buildergen.ConfigDefinition;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.vespa.config.*;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * This class is capable of resolving config from a config model for a given request. It will handle
+ * incompatibilities of the def version in the request and the version of the config classes the model
+ * is using.
+ * </p>
+ * <p>
+ * This class is agnostic of transport protocol and server implementation.
+ * </p>
+ * <p>
+ * Thread safe.
+ * </p>
+ *
+ * @author vegardh
+ * @since 5.1.5
+ */
+// TODO This functionality should be on VespaModel itself, but we don't have a way right now to apply a config override to a ConfigInstance.Builder
+class InstanceResolver {
+
+ private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(InstanceResolver.class.getName());
+
+ /**
+ * Resolves this config key into a correctly typed ConfigInstance using the given config builder.
+ * FIXME: Make private once config overrides are deprecated.?
+ *
+ * @param key a ConfigKey
+ * @param builder a ConfigBuilder to create the instance from.
+ * @param targetDef the def to use
+ * @return the config instance or null of no producer for this found in model
+ */
+ static ConfigInstance resolveToInstance(ConfigKey<?> key, ConfigBuilder builder, InnerCNode targetDef) {
+ ConfigDefinitionKey defKey = new ConfigDefinitionKey(key);
+ try {
+ if (targetDef != null) applyDef(builder, targetDef);
+ ConfigInstance instance = getInstance(defKey, builder.getClass().getClassLoader());
+ Class<? extends ConfigInstance> clazz = instance.getClass();
+ return clazz.getConstructor(new Class<?>[]{builder.getClass()}).newInstance(builder);
+ } catch (Exception e) {
+ throw new ConfigurationRuntimeException(e);
+ }
+ }
+
+ /**
+ * Resolves this config key into a correctly typed ConfigBuilder using the given config model.
+ * FIXME: Make private once config overrides are deprecated.?
+ *
+ * @return the config builder or null if no producer for this found in model
+ */
+ static ConfigBuilder resolveToBuilder(ConfigKey<?> key, VespaModel model, ConfigDefinition targetDef) {
+ if (model == null) return null;
+ ConfigDefinitionKey defKey = new ConfigDefinitionKey(key);
+ ConfigInstance.Builder builder = model.createBuilder(defKey, targetDef);
+ model.getConfig(builder, key.getConfigId());
+ return builder;
+ }
+
+ /**
+ * If some fields on the builder are null now, set them from the def. Do recursively.
+ * <p>
+ * If the targetDef has some schema incompatibilities, they are not handled here
+ * (except logging in some cases), but in ConfigInstance.serialize().
+ *
+ * @param builder a {@link com.yahoo.config.ConfigBuilder}
+ * @param targetDef a config definition
+ * @throws Exception if applying values form config definitions fails
+ */
+ static void applyDef(ConfigBuilder builder, InnerCNode targetDef) throws Exception {
+ for (Map.Entry<String, CNode> e: targetDef.children().entrySet()) {
+ CNode node = e.getValue();
+ if (node instanceof LeafCNode) {
+ setLeafValueIfUnset(targetDef, builder, (LeafCNode)node);
+ } else if (node instanceof InnerCNode) {
+ // Is there a private field on the builder that matches this inner node in the def?
+ if (hasField(builder.getClass(), node.getName())) {
+ Field innerField = builder.getClass().getDeclaredField(node.getName());
+ innerField.setAccessible(true);
+ Object innerFieldVal = innerField.get(builder);
+ if (innerFieldVal instanceof List) {
+ // inner array? Check that list elems are ConfigBuilder
+ List<?> innerList = (List<?>) innerFieldVal;
+ for (Object b : innerList) {
+ if (b instanceof ConfigBuilder) {
+ applyDef((ConfigBuilder) b, (InnerCNode) node);
+ }
+ }
+ } else if (innerFieldVal instanceof ConfigBuilder) {
+ // Struct perhaps
+ applyDef((ConfigBuilder) innerFieldVal, (InnerCNode) node);
+ } else {
+ // Likely a config value mismatch. That is handled in ConfigInstance.serialize() (error message, omit from response.)
+ }
+ }
+ }
+ }
+ }
+
+ private static boolean hasField(Class<?> aClass, String name) {
+ for (Field field : aClass.getDeclaredFields()) {
+ if (name.equals(field.getName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static void setLeafValueIfUnset(InnerCNode targetDef, Object builder, LeafCNode node) throws Exception {
+ if (hasField(builder.getClass(), node.getName())) {
+ Field field = builder.getClass().getDeclaredField(node.getName());
+ field.setAccessible(true);
+ Object val = field.get(builder);
+ if (val==null) {
+ // Not set on builder, if the leaf node has a default value, try the private setter that takes String
+ try {
+ if (node.getDefaultValue()!=null) {
+ Method setter = builder.getClass().getDeclaredMethod(node.getName(), String.class);
+ setter.setAccessible(true);
+ setter.invoke(builder, node.getDefaultValue().getValue());
+ }
+ } catch (Exception e) {
+ log.severe("For config '"+targetDef.getFullName()+"': Unable to apply the default value for field '"+node.getName()+
+ "' to config Builder (where it wasn't set): "+
+ Exceptions.toMessageString(e));
+ }
+ }
+ }
+ }
+
+ /**
+ * Create a ConfigBuilder given a definition key and a payload
+ * @param key The key to use to create the correct builder.
+ * @param payload The payload to populate the builder with.
+ * @return A ConfigBuilder initialized with payload.
+ */
+ static ConfigBuilder createBuilderFromPayload(ConfigDefinitionKey key, VespaModel model, ConfigPayload payload, ConfigDefinition targetDef) {
+ ConfigBuilder builderInstance = model.createBuilder(key, targetDef);
+ if (builderInstance == null || builderInstance instanceof GenericConfig.GenericConfigBuilder) {
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "Creating generic builder for key=" + key);
+ }
+ return new GenericConfig.GenericConfigBuilder(key, new ConfigPayloadBuilder(payload));
+ }
+ ConfigTransformer transformer = new ConfigTransformer(builderInstance.getClass().getDeclaringClass());
+ return transformer.toConfigBuilder(payload);
+ }
+
+ /**
+ * Returns a {@link ConfigInstance} of right type for given key using reflection
+ *
+ * @param cKey a ConfigKey
+ * @return a {@link ConfigInstance} or null if not available in classpath
+ */
+ static ConfigInstance getInstance(ConfigDefinitionKey cKey, ClassLoader instanceLoader) {
+ String className = ConfigGenerator.createClassName(cKey.getName());
+ Class<?> clazz;
+ String fullClassName = packageName(cKey) + "." + className;
+ try {
+ clazz = instanceLoader != null ? instanceLoader.loadClass(fullClassName) : Class.forName(fullClassName);
+ } catch (ClassNotFoundException e) {
+ return null;
+ }
+ Object i;
+ try {
+ i = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new ConfigurationRuntimeException(e);
+ }
+ if (!(i instanceof ConfigInstance)) {
+ throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance, can not produce config for the name '" + cKey.getName() + "'.");
+ }
+ return (ConfigInstance) i;
+ }
+
+ static String packageName(ConfigDefinitionKey cKey) {
+ String prefix = "com.yahoo.";
+ return prefix + (cKey.getNamespace().isEmpty() ? CNode.DEFAULT_NAMESPACE : cKey.getNamespace());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Logd.java b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java
new file mode 100644
index 00000000000..41127f6423b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+/**
+ * There is one logd running on each Vespa host, and one instance of
+ * this class is therefore created by each instance of class {@link
+ * Host}.
+ *
+ * @author gjoranv
+ */
+public class Logd extends AbstractService {
+
+ /**
+ * Creates a new Logd instance.
+ */
+ public Logd(Host host) {
+ super(host, "logd");
+ setProp("clustertype", "hosts");
+ setProp("clustername", "admin");
+ }
+
+ /**
+ * Logd does not need any ports.
+ *
+ * @return The number of ports reserved by the logd
+ */
+ public int getPortCount() { return 0; }
+
+ /**
+ * @return The command used to start logd
+ */
+ public String getStartupCommand() { return "exec sbin/logd"; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java b/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java
new file mode 100644
index 00000000000..d424f4fa31b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import java.util.logging.Formatter;
+import java.util.logging.LogRecord;
+
+/**
+ * A log formatter that returns a plain log message only with level, not
+ * including timestamp and method (as java.util.logging.SimpleFormatter).
+ * See bug #1789867.
+ *
+ * @author gjoranv
+ */
+public class PlainFormatter extends Formatter {
+
+ public PlainFormatter() {
+ super();
+ }
+
+ public String format(LogRecord record) {
+ StringBuffer sb = new StringBuffer();
+
+ sb.append(record.getLevel().getName()).append(": ");
+ sb.append(formatMessage(record)).append("\n");
+
+ return sb.toString();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java
new file mode 100644
index 00000000000..ea2151f9976
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java
@@ -0,0 +1,168 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Track metainformation about the ports of a service.
+ *
+ * @author Vidar Larsen
+ */
+public class PortsMeta implements Serializable {
+ /** A list of all ports. The list elements are lists of strings. */
+ private List<LinkedList<String>> ports;
+
+ /** Remember the rpc admin port offset. */
+ private Integer rpcAdminOffset = null;
+ /** Remember the rpc status port offset. */
+ private Integer rpcStatusOffset = null;
+ /** Remember the https admin port offset. */
+ private Integer httpAdminOffset = null;
+ /** Remember the http status port offset. */
+ private Integer httpStatusOffset = null;
+
+ /** Keep track of the offset to register for, when chaining */
+ private Integer currentOffset;
+
+ /** Create a new PortsMeta object. */
+ public PortsMeta() {
+ ports = new ArrayList<>();
+ }
+
+ /**
+ * Set up the port to tag, for chained usage.
+ * @param offset The relative port to tag.
+ * @return this portsmeta, to allow .tag calls.
+ */
+ public PortsMeta on(int offset) {
+ this.currentOffset = offset;
+ return this;
+ }
+
+ /**
+ * Tag a previously setup port (using 'on') with the specified tag.
+ * @param meta The tag to apply to the current port.
+ * @return this portsmeta, to allow further .tag calls.
+ */
+ public PortsMeta tag(String meta) {
+ if (currentOffset == null) {
+ throw new NullPointerException("No current port offset to tag, use 'on#1'");
+ }
+ return register(currentOffset, meta);
+ }
+
+ /**
+ * Register a given metainfo string to the port at offset.
+ * @param offset 0-based index to identify the port
+ * @param meta a String to be added to the given ports meta set.
+ * @return this for convenient chaining.
+ */
+ private PortsMeta register(int offset, String meta) {
+ // Allocate new LinkedLists on each element up-to-and-including offset
+ for (int i = ports.size(); i <= offset; i++) {
+ ports.add(i, new LinkedList<String>());
+ }
+ ports.get(offset).addFirst(meta);
+
+ return this;
+ }
+
+ /**
+ * Check if the port at a specific offset contains a particular meta attribute.
+ * @param offset The relative port offset
+ * @param meta The meta info we want to check for
+ * @return boolean true if the specific port has registered the meta
+ */
+ public boolean contains(int offset, String meta) {
+ return offset < ports.size() && ports.get(offset).contains(meta);
+ }
+
+ /**
+ * Get the number of ports with registered meta.
+ * @return the number of ports that have been registered.
+ */
+ public int getNumPorts() {
+ return ports.size();
+ }
+ /**
+ * Get an iterator of the Strings registered at the specific point.
+ * @param offset The relative offset to inquire about tags.
+ * @return List of tags.
+ */
+ public List<String> getTagsAt(int offset) {
+ try {
+ return ports.get(offset);
+ } catch (IndexOutOfBoundsException e) {
+ throw new RuntimeException("Trying to get ports meta with offset " + offset +
+ ", which is outside the range 0 to " + ports.size(), e);
+ }
+ }
+
+ /**
+ * Get the offset to the rpc port used for admin.
+ * @return Integer the offset, or null if none set.
+ */
+ public Integer getRpcAdminOffset() {
+ if (rpcAdminOffset == null) {
+ for (int p = 0; p < getNumPorts(); p++) {
+ if (contains(p, "rpc") && contains(p, "admin") && !contains(p, "nc")) {
+ rpcAdminOffset = p;
+ break;
+ }
+ }
+ }
+ return rpcAdminOffset;
+ }
+
+ /**
+ * Get the offset to the rpc port used for status.
+ * @return Integer the offset, or null if none set.
+ */
+ public Integer getRpcStatusOffset() {
+ if (rpcStatusOffset == null) {
+ for (int p = 0; p < getNumPorts(); p++) {
+ if (contains(p, "rpc") && contains(p, "status") && !contains(p, "nc")) {
+ rpcStatusOffset = p;
+ break;
+ }
+ }
+ }
+ return rpcStatusOffset;
+ }
+
+ /**
+ * Get the offset to the http port used for admin.
+ * @return Integer the offset, or null if none set.
+ */
+ public Integer getHttpAdminOffset() {
+ if (httpAdminOffset == null) {
+ for (int p = 0; p < getNumPorts(); p++) {
+ if (contains(p, "http") && contains(p, "admin")) {
+ httpAdminOffset = p;
+ break;
+ }
+ }
+ }
+ return httpAdminOffset;
+ }
+
+ /**
+ * Get the offset to the http port used for status.
+ * @return Integer the offset, or null if none set.
+ */
+ public Integer getHttpStatusOffset() {
+ if (httpStatusOffset == null) {
+ for (int p = 0; p < getNumPorts(); p++) {
+ if (contains(p, "http") && contains(p, "status")) {
+ httpStatusOffset = p;
+ break;
+ }
+ }
+ }
+ return httpStatusOffset;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java
new file mode 100644
index 00000000000..84f33b637fb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import java.util.LinkedList;
+import java.util.logging.Filter;
+import java.util.logging.LogRecord;
+
+/**
+ * A filter for log messages that prevents the most recently published messages
+ * to be published again. See bug #461290.
+ *
+ * @author gjoranv
+ */
+public class RecentLogFilter implements Filter {
+
+ LinkedList<String> recent = new LinkedList<>();
+ static final int maxMessages = 6;
+
+ public boolean isLoggable(LogRecord record) {
+ String msg = record.getMessage();
+ if (msg != null && recent.contains(msg)) {
+ return false; // duplicate
+ } else {
+ recent.addLast(msg);
+ if (recent.size() > maxMessages) {
+ recent.removeFirst();
+ }
+ return true; // new message
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java
new file mode 100644
index 00000000000..ccb6397dd46
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java
@@ -0,0 +1,142 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.model.api.ServiceInfo;
+
+import java.util.HashMap;
+import java.util.Optional;
+
+
+/**
+ * Representation of a process which runs a service
+ *
+ * @author gjoranv
+ */
+public interface Service extends ConfigProducer {
+
+ /**
+ * Services that should be started by config-sentinel must return
+ * non-null. The returned value will be used in config-sentinel
+ * configuration.
+ * TODO: Should change this to Optional of String
+ */
+ public String getStartupCommand();
+
+ /**
+ * Services that wish that a command should be run before shutdown
+ * should return the command here. The command will be executed
+ * by the config sentinel before sending SIGTERM to the service.
+ * The command is executed without a timeout.
+ */
+ public Optional<String> getPreShutdownCommand();
+
+ /**
+ * Tells if this service should be autostarted by
+ * config-sentinel. Returned value will be used to configure the
+ * config-sentinel.
+ */
+ public boolean getAutostartFlag();
+
+ /**
+ * Tells if this service should be autorestarted by
+ * config-sentinel. Returned value will be used to configure the
+ * config-sentinel.
+ */
+ public boolean getAutorestartFlag();
+
+ /**
+ * Returns the type of service. E.g. the class-name without the
+ * package prefix.
+ */
+ public String getServiceType();
+
+ /**
+ * Returns the name that identifies this service for the config-sentinel.
+ */
+ public String getServiceName();
+
+ /**
+ * Returns the desired base port for this service, or '0' if this
+ * service should use the default port allocation mechanism.
+ *
+ * @return The desired base port for this service.
+ */
+ public int getWantedPort();
+
+ /**
+ * Returns true if the desired base port (returned by
+ * getWantedPort()) for this service is the only allowed base
+ * port.
+ *
+ * @return true if this Service requires the wanted base port.
+ */
+ public boolean requiresWantedPort();
+
+ /**
+ * Returns the number of ports needed by this service.
+ */
+ public int getPortCount();
+
+ /**
+ * Returns a PortsMeta object, giving access to more information
+ * about the different ports of this service.
+ */
+ public PortsMeta getPortsMeta();
+
+ /**
+ * @return the physical host on which this service runs.
+ */
+ public Host getHost();
+
+ /**
+ * Get meta information about service.
+ * @return an instance of {@link com.yahoo.config.model.api.ServiceInfo}
+ */
+ public ServiceInfo getServiceInfo();
+
+ /**
+ * @return The hostname on which this service runs.
+ */
+ public String getHostName();
+
+ /** Optional JVM execution args for this service */
+ public String getJvmArgs();
+
+ /**
+ * Computes and returns the i'th port for this service, based on
+ * this Service's baseport.
+ *
+ * @param i The offset from 'basePort' of the port to return
+ * @return the i'th port relative to the base port.
+ * @throws IllegalStateException if i is out of range.
+ */
+ public int getRelativePort(int i);
+
+ /**
+ * Gets a service property value mapped to the given key
+ * as a String, or the value in <code>defStr</code> if no such key exists.
+ *
+ * @param key a key used for lookup in the service properties
+ * @param defStr default String value returned if no value for key found
+ * @return the associated String value for the given key, or
+ */
+ public String getServicePropertyString(String key, String defStr);
+
+ /**
+ * @return the service health port, to report status to yamas
+ */
+ public int getHealthPort();
+
+ /**
+ *
+ * @return HashMap of default dimensions for metrics.
+ */
+ public HashMap<String,String> getDefaultMetricDimensions();
+
+ /**
+ * Return the Affinity of this service if it has.
+ *
+ * @return The {@link com.yahoo.vespa.model.Affinity} for this service.
+ */
+ public Optional<Affinity> getAffinity();
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java
new file mode 100644
index 00000000000..29508eda1eb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+/**
+ * A provider of services (an entity which has its own top-level tag in the services.xml file).
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public interface ServiceProvider {
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java
new file mode 100644
index 00000000000..3e5f816f583
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * Some configuration level with no special handling of its own.
+ *
+ * @author arnej27959
+ */
+public final class SimpleConfigProducer<T extends AbstractConfigProducer<?>> extends AbstractConfigProducer<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Creates a new instance
+ *
+ * @param parent parent ConfigProducer.
+ * @param configId name of this instance
+ */
+ public SimpleConfigProducer(AbstractConfigProducer parent, String configId) {
+ super(parent, configId);
+ }
+
+ //Ease access restriction
+ @Override
+ public void addChild(T child) {
+ super.addChild(child);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java
new file mode 100644
index 00000000000..6b9aa3fe8c5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.model.ConfigModelRegistry;
+import com.yahoo.config.model.admin.AdminModel;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.vespa.model.builder.xml.dom.*;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking;
+import com.yahoo.vespa.model.generic.GenericServicesBuilder;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Resolves the Vespa config model types
+ *
+ * @author bratseth
+ */
+public class VespaConfigModelRegistry extends ConfigModelRegistry {
+
+ private final List<ConfigModelBuilder> builderList = new ArrayList<>();
+
+ /** Creates a bundled model class registry which forwards unresolved requests to the argument instance */
+ public VespaConfigModelRegistry(ConfigModelRegistry chained) {
+ super(chained);
+ builderList.add(new AdminModel.BuilderV2());
+ builderList.add(new AdminModel.BuilderV4());
+ builderList.add(new DomRoutingBuilder());
+ builderList.add(new DomClientsBuilder());
+ builderList.add(new DomContentBuilder());
+ builderList.add(new ContainerModelBuilder(false, Networking.enable));
+ builderList.add(new GenericServicesBuilder());
+ }
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ Set<ConfigModelBuilder> builders = new HashSet<>(chained().resolve(id));
+ for (ConfigModelBuilder builder : builderList) {
+ if (builder.handlesElements().contains(id))
+ builders.add(builder);
+ }
+ return builders;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
new file mode 100644
index 00000000000..9a23be1f5c5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -0,0 +1,557 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.yahoo.config.ConfigBuilder;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.ConfigInstance.Builder;
+import com.yahoo.config.ConfigurationRuntimeException;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.codegen.ConfigGenerator;
+import com.yahoo.config.codegen.InnerCNode;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.ConfigModelRegistry;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.config.model.producer.UserConfigRepo;
+import com.yahoo.config.provision.ProvisionInfo;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.config.ConfigPayload;
+import com.yahoo.vespa.config.ConfigPayloadBuilder;
+import com.yahoo.vespa.config.GenericConfig;
+import com.yahoo.vespa.config.buildergen.ConfigDefinition;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.application.validation.ValidationId;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.builder.VespaModelBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.clients.Clients;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.filedistribution.FileDistributor;
+import com.yahoo.vespa.model.generic.service.ServiceCluster;
+import com.yahoo.vespa.model.routing.Routing;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.utils.internal.ReflectionUtil;
+import com.yahoo.yolean.Exceptions;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static com.yahoo.text.StringUtilities.quote;
+
+/**
+ * <p>
+ * The root ConfigProducer node for all Vespa systems (there is currently only one).
+ * The main class for building the Vespa model.
+ * </p>
+ * The vespa model starts in an unfrozen state, where children can be added freely,
+ * but no structure dependent information can be used.
+ * When frozen, structure dependent information(such as config id and controller) are
+ * made available, but no additional config producers can be added.
+ *
+ * @author gjoranv
+ */
+public final class VespaModel extends AbstractConfigProducerRoot implements Serializable, Model {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString());
+ private ConfigModelRepo configModelRepo = new ConfigModelRepo();
+ private final Optional<ProvisionInfo> info;
+
+ /**
+ * The config id for the root config producer
+ */
+ public static final String ROOT_CONFIGID = "";
+
+ private ApplicationConfigProducerRoot root = null;
+
+ /**
+ * Generic service instances - service clusters which have no specific model
+ */
+ private List<ServiceCluster> serviceClusters = new ArrayList<>();
+
+ private DeployState deployState;
+
+ /** The validation overrides of this. This is never null. */
+ private final ValidationOverrides validationOverrides;
+
+ /** Creates a Vespa Model from internal model types only */
+ public VespaModel(ApplicationPackage app) throws IOException, SAXException {
+ this(app, new NullConfigModelRegistry());
+ }
+
+ /** Creates a Vespa Model from internal model types only */
+ public VespaModel(DeployState deployState) throws IOException, SAXException {
+ this(new NullConfigModelRegistry(), deployState);
+ }
+
+ /**
+ * Constructs vespa model using config given in app
+ *
+ * @param app the application to create a model from
+ * @param configModelRegistry a registry of config model "main" classes which may be used
+ * to instantiate config models
+ */
+ public VespaModel(ApplicationPackage app, ConfigModelRegistry configModelRegistry) throws IOException, SAXException {
+ this(configModelRegistry, new DeployState.Builder().applicationPackage(app).build());
+ }
+
+ /**
+ * Constructs vespa model using config given in app
+ *
+ * @param configModelRegistry a registry of config model "main" classes which may be used
+ * to instantiate config models
+ * @param deployState the global deploy state to use for this model.
+ */
+ public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException {
+ super("vespamodel");
+ this.deployState = deployState;
+ this.validationOverrides = deployState.validationOverrides();
+ configModelRegistry = new VespaConfigModelRegistry(configModelRegistry);
+ VespaModelBuilder builder = new VespaDomBuilder();
+ root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this);
+ configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry);
+ addServiceClusters(deployState.getApplicationPackage(), builder);
+ setupRouting();
+ log.log(LogLevel.DEBUG, "hostsystem=" + getHostSystem());
+ this.info = Optional.of(createProvisionInfo());
+ getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState.getProperties());
+ freezeModelTopology();
+ root.prepare(configModelRepo);
+ configModelRepo.prepareConfigModels();
+ validateWrapExceptions();
+ this.deployState = null;
+ }
+
+ private ProvisionInfo createProvisionInfo() {
+ HostSystem hostSystem = root.getHostSystem();
+ return ProvisionInfo.withHosts(hostSystem.getHostToServiceSpecMapping().keySet());
+ }
+
+ private void validateWrapExceptions() {
+ try {
+ validate();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Error while validating model:", e);
+ }
+ }
+
+ /** Adds generic application specific clusters of services */
+ private void addServiceClusters(ApplicationPackage app, VespaModelBuilder builder) {
+ for (ServiceCluster sc : builder.getClusters(app, this))
+ serviceClusters.add(sc);
+ }
+
+ private void setupRouting() {
+ root.setupRouting(configModelRepo);
+ }
+
+ /** Returns the one and only HostSystem of this VespaModel */
+ public HostSystem getHostSystem() {
+ return root.getHostSystem();
+ }
+
+ /** Return a collection of all hostnames used in this application */
+ @Override
+ public Set<HostInfo> getHosts() {
+ Set<HostInfo> hosts = new LinkedHashSet<>();
+ for (HostResource host : root.getHostSystem().getHosts()) {
+ if (!host.getHost().isMultitenant()) {
+ hosts.add(host.getHostInfo());
+ }
+ }
+ return hosts;
+ }
+
+ public FileDistributor getFileDistributor() {
+ return root.getFileDistributionConfigProducer().getFileDistributor();
+ }
+
+ /** Returns this models Vespa instance */
+ public ApplicationConfigProducerRoot getVespa() { return root; }
+
+ @Override
+ public boolean allowModelVersionMismatch() {
+ return validationOverrides.allows(ValidationId.configModelVersionMismatch) ||
+ validationOverrides.allows(ValidationId.skipOldConfigModels); // implies this
+ }
+
+ @Override
+ public boolean skipOldConfigModels() {
+ return validationOverrides.allows(ValidationId.skipOldConfigModels);
+ }
+
+ /**
+ * Resolves config of the given type and config id, by first instantiating the correct {@link com.yahoo.config.ConfigInstance.Builder},
+ * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config
+ * types in the model.
+ *
+ * @param clazz The type of config
+ * @param configId The config id
+ * @return A config instance of the given type
+ */
+ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) {
+ try {
+ ConfigInstance.Builder builder = newBuilder(clazz);
+ getConfig(builder, configId);
+ return newConfigInstance(clazz, builder);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Populates an instance of configClass with config produced by configProducer.
+ */
+ public static <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ConfigProducer configProducer) {
+ try {
+ Builder builder = newBuilder(configClass);
+ populateConfigBuilder(builder, configProducer);
+ return newConfigInstance(configClass, builder);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed getting config for class " + configClass.getName(), e);
+ }
+ }
+
+ private static <CONFIGTYPE extends ConfigInstance> CONFIGTYPE newConfigInstance(Class<CONFIGTYPE> configClass, Builder builder)
+ throws NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException {
+
+ Constructor<CONFIGTYPE> constructor = configClass.getConstructor(builder.getClass());
+ return constructor.newInstance(builder);
+ }
+
+ private static Builder newBuilder(Class<? extends ConfigInstance> configClass)
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException {
+
+ Class builderClazz = configClass.getClassLoader().loadClass(configClass.getName() + "$Builder");
+ return (Builder)builderClazz.newInstance();
+ }
+
+ /**
+ * Throw if the config id does not exist in the model.
+ *
+ * @param configId a config id
+ */
+ protected void checkId(String configId) {
+ if ( ! id2producer.containsKey(configId)) {
+ log.log(LogLevel.DEBUG, "Invalid config id: " + configId);
+ }
+ }
+
+ /**
+ * Resolves config for a given config id and populates the given builder with the config.
+ *
+ * @param builder a configinstance builder
+ * @param configId the config id for the config client
+ * @return the builder if a producer was found, and it did apply config, null otherwise
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) {
+ checkId(configId);
+ Optional<ConfigProducer> configProducer = getConfigProducer(configId);
+ if ( ! configProducer.isPresent()) return null;
+
+ populateConfigBuilder(builder, configProducer.get());
+ return builder;
+ }
+
+ private static void populateConfigBuilder(Builder builder, ConfigProducer configProducer) {
+ boolean found = configProducer.cascadeConfig(builder);
+ boolean foundOverride = configProducer.addUserConfig(builder);
+ if (logDebug()) {
+ log.log(LogLevel.DEBUG, "Trying to get config for " + builder.getClass().getDeclaringClass().getName() +
+ " for config id " + quote(configProducer.getConfigId()) +
+ ", found=" + found + ", foundOverride=" + foundOverride);
+ }
+ }
+
+ /** TODO: Remove once configserver has switched to using {@link VespaModel#getConfig(ConfigKey, ConfigDefinition, ConfigPayload)} instead. **/
+ public ConfigPayload getConfig(ConfigKey configKey, InnerCNode targetDef, ConfigPayload userOverride) throws IOException {
+ return getConfig(configKey, targetDef == null ? null : new ConfigDefinition(targetDef), userOverride);
+ }
+
+ /**
+ * Resolve config for a given key and a def. Apply an override if given.
+ *
+ * @param configKey The key to resolve.
+ * @param targetDef The def file to use for the schema.
+ * @param userOverride A user override that should be applied to the config
+ * @return The payload as a list of strings
+ */
+ @Override
+ public ConfigPayload getConfig(ConfigKey configKey, com.yahoo.vespa.config.buildergen.ConfigDefinition targetDef, ConfigPayload userOverride) throws IOException {
+ ConfigBuilder builder = InstanceResolver.resolveToBuilder(configKey, this, targetDef);
+ if (builder != null) {
+ log.log(LogLevel.DEBUG, () -> "Found builder for " + configKey);
+ // Support deprecated configs/ user override
+ applyConfigsOverride(configKey, builder, userOverride, targetDef);
+ ConfigPayload payload;
+ InnerCNode innerCNode = targetDef != null ? targetDef.getCNode() : null;
+ if (builder instanceof GenericConfig.GenericConfigBuilder) {
+ payload = getConfigFromGenericBuilder(builder);
+ } else {
+ payload = getConfigFromBuilder(configKey, builder, innerCNode);
+ }
+ return (innerCNode != null) ? payload.applyDefaultsFromDef(innerCNode) : payload;
+ }
+ return null;
+ }
+
+ private ConfigPayload getConfigFromBuilder(ConfigKey configKey, ConfigBuilder builder, InnerCNode targetDef) {
+ try {
+ ConfigInstance instance = InstanceResolver.resolveToInstance(configKey, builder, targetDef);
+ log.log(LogLevel.DEBUG, () -> "getConfigFromBuilder for " + configKey + ",instance=" + instance);
+ return ConfigPayload.fromInstance(instance);
+ } catch (ConfigurationRuntimeException e) {
+ // This can happen in cases where services ask for config that no longer exist before they have been able
+ // to reconfigure themselves. This happens for instance whenever jdisc reconfigures itself until
+ // ticket 6599572 is fixed. When that happens, consider propagating a full error rather than empty payload
+ // back to the client.
+ log.log(LogLevel.INFO, "Error resolving instance for key '" + configKey + "', returning empty config: " + Exceptions.toMessageString(e));
+ return ConfigPayload.fromBuilder(new ConfigPayloadBuilder());
+ }
+ }
+
+ private ConfigPayload getConfigFromGenericBuilder(ConfigBuilder builder) throws IOException {
+ return ((GenericConfig.GenericConfigBuilder) builder).getPayload();
+ }
+
+ private void applyConfigsOverride(ConfigKey configKey, ConfigBuilder builder, ConfigPayload userOverride, ConfigDefinition targetDef) {
+ if (userOverride != null) {
+ ConfigBuilder override = InstanceResolver.createBuilderFromPayload(new ConfigDefinitionKey(configKey), this, userOverride, targetDef);
+ ConfigInstanceUtil.setValues(builder, override);
+ }
+ }
+
+ @Override
+ public Set<ConfigKey<?>> allConfigsProduced() {
+ Set<ConfigKey<?>> keySet = new LinkedHashSet<>();
+ for (ConfigProducer producer : id2producer().values()) {
+ keySet.addAll(configsProduced(producer));
+ }
+ return keySet;
+ }
+
+ public ConfigInstance.Builder createBuilder(ConfigDefinitionKey key, ConfigDefinition targetDef) {
+ String className = ConfigGenerator.createClassName(key.getName());
+ Class<?> clazz;
+
+ final String fullClassName = InstanceResolver.packageName(key) + "." + className;
+ final String builderName = fullClassName + "$Builder";
+ final String producerName = fullClassName + "$Producer";
+ ClassLoader classLoader = getConfigClassLoader(producerName);
+ if (classLoader == null) {
+ classLoader = getClass().getClassLoader();
+ if (logDebug()) {
+ log.log(LogLevel.DEBUG, "No producer found to get classloader from for " + fullClassName + ". Using default");
+ }
+ }
+ try {
+ clazz = classLoader.loadClass(builderName);
+ } catch (ClassNotFoundException e) {
+ if (logDebug()) {
+ log.log(LogLevel.DEBUG, "Tried to load " + builderName + ", not found, trying with generic builder");
+ }
+ // TODO: Enable config compiler when configserver is using new API.
+ // ConfigCompiler compiler = new LazyConfigCompiler(Files.createTempDir());
+ // return compiler.compile(targetDef.generateClass()).newInstance();
+ return new GenericConfig.GenericConfigBuilder(key, new ConfigPayloadBuilder());
+ }
+ Object i;
+ try {
+ i = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new ConfigurationRuntimeException(e);
+ }
+ if (!(i instanceof ConfigInstance.Builder)) {
+ throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance.Builder, can not produce config for the name '" + key.getName() + "'.");
+ }
+ return (ConfigInstance.Builder) i;
+ }
+
+ private static boolean logDebug() {
+ return log.isLoggable(LogLevel.DEBUG);
+ }
+
+ /**
+ * The set of all config ids present
+ * @return set of config ids
+ */
+ public Set<String> allConfigIds() {
+ return id2producer.keySet();
+ }
+
+ @Override
+ public void distributeFiles(FileDistribution fileDistribution) {
+ getFileDistributor().sendDeployedFiles(fileDistribution);
+ }
+
+ @Override
+ public void reloadDeployFileDistributor(FileDistribution fileDistribution) {
+ getFileDistributor().reloadDeployFileDistributor(fileDistribution);
+ }
+
+ @Override
+ public Optional<ProvisionInfo> getProvisionInfo() {
+ return info;
+ }
+
+ private static Set<ConfigKey<?>> configsProduced(ConfigProducer cp) {
+ Set<ConfigKey<?>> ret = ReflectionUtil.configsProducedByInterface(cp.getClass(), cp.getConfigId());
+ UserConfigRepo userConfigs = cp.getUserConfigs();
+ for (ConfigDefinitionKey userKey : userConfigs.configsProduced()) {
+ ret.add(new ConfigKey<>(userKey.getName(), cp.getConfigId(), userKey.getNamespace()));
+ }
+ return ret;
+ }
+
+ @Override
+ public DeployState getDeployState() {
+ if (deployState == null) {
+ throw new IllegalStateException("Cannot call getDeployState() once model has been built");
+ }
+ return deployState;
+ }
+
+ /**
+ * @return an unmodifiable copy of the set of configIds in this VespaModel.
+ */
+ public Set<String> getConfigIds() {
+ return Collections.unmodifiableSet(id2producer.keySet());
+ }
+
+ /**
+ * Returns the admin component of the vespamodel.
+ *
+ * @return Admin
+ */
+ public Admin getAdmin() {
+ return root.getAdmin();
+ }
+
+ /**
+ * Adds the descendant (at any depth level), so it can be looked up
+ * on configId in the Map.
+ *
+ * @param configId the id to register with, not necessarily equal to descendant.getConfigId().
+ * @param descendant The configProducer descendant to add
+ */
+ public void addDescendant(String configId, AbstractConfigProducer descendant) {
+ if (id2producer.containsKey(configId)) {
+ throw new RuntimeException
+ ("Config ID '" + configId + "' cannot be reserved by an instance of class '" +
+ descendant.getClass().getName() +
+ "' since it is already used by an instance of class '" +
+ id2producer.get(configId).getClass().getName() +
+ "'. (This is commonly caused by service/node index " +
+ "collisions in the config.)");
+ }
+ id2producer.put(configId, descendant);
+ }
+
+ /**
+ * Writes MODEL.cfg files for all config producers.
+ *
+ * @param baseDirectory dir to write files to
+ */
+ public void writeFiles(File baseDirectory) throws IOException {
+ super.writeFiles(baseDirectory);
+ for (ConfigProducer cp : id2producer.values()) {
+ try {
+ File destination = new File(baseDirectory, cp.getConfigId().replace("/", File.separator));
+ cp.writeFiles(destination);
+ } catch (IOException e) {
+ throw new IOException(cp.getConfigId() + ": " + e.getMessage());
+ }
+ }
+ }
+
+ public Clients getClients() {
+ return configModelRepo.getClients();
+ }
+
+ /** Returns all search clusters, both in Search and Content */
+ public List<AbstractSearchCluster> getSearchClusters() {
+ return Content.getSearchClusters(configModelRepo());
+ }
+
+ /** Returns a map of content clusters by ID */
+ public Map<String, ContentCluster> getContentClusters() {
+ Map<String, ContentCluster> clusters = new LinkedHashMap<>();
+ for (Content model : configModelRepo.getModels(Content.class)) {
+ clusters.put(model.getId(), model.getCluster());
+ }
+ return Collections.unmodifiableMap(clusters);
+ }
+
+ /** Returns a map of container clusters by ID */
+ public Map<String, ContainerCluster> getContainerClusters() {
+ Map<String, ContainerCluster> clusters = new LinkedHashMap<>();
+ for (ContainerModel model : configModelRepo.getModels(ContainerModel.class)) {
+ clusters.put(model.getId(), model.getCluster());
+ }
+ return Collections.unmodifiableMap(clusters);
+ }
+
+ /** Returns the routing config model. This might be null. */
+ public Routing getRouting() {
+ return configModelRepo.getRouting();
+ }
+
+ public FileDistributionConfigProducer getFileDistributionConfigProducer() {
+ return root.getFileDistributionConfigProducer();
+ }
+
+ /** The clusters of application specific generic services */
+ public List<ServiceCluster> serviceClusters() {
+ return serviceClusters;
+ }
+
+ /** Returns an unmodifiable view of the mapping of config id to {@link ConfigProducer} */
+ public Map<String, ConfigProducer> id2producer() {
+ return Collections.unmodifiableMap(id2producer);
+ }
+
+ /**
+ * @return this root's model repository
+ */
+ public ConfigModelRepo configModelRepo() {
+ return configModelRepo;
+ }
+
+ @Override
+ public DeployLogger deployLogger() {
+ return getDeployState().getDeployLogger();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
new file mode 100644
index 00000000000..0dccc2c9750
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
@@ -0,0 +1,196 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+import com.google.inject.Inject;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.ConfigModelRegistry;
+import com.yahoo.config.model.MapConfigModelRegistry;
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.api.ConfigModelPlugin;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.api.ModelCreateResult;
+import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.deploy.DeployProperties;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.provision.HostsXmlProvisioner;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Version;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.config.VespaVersion;
+import com.yahoo.vespa.model.application.validation.Validation;
+
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Factory for creating {@link VespaModel} instances.
+ *
+ * @author lulf
+ */
+public class VespaModelFactory implements ModelFactory {
+
+ private static final Logger log = Logger.getLogger(VespaModelFactory.class.getName());
+ private final ConfigModelRegistry configModelRegistry;
+ private final Zone zone;
+
+ @Inject
+ public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) {
+ List<ConfigModelBuilder> modelBuilders = new ArrayList<>();
+ for (ConfigModelPlugin plugin : pluginRegistry.allComponents()) {
+ if (plugin instanceof ConfigModelBuilder) {
+ modelBuilders.add((ConfigModelBuilder) plugin);
+ }
+ }
+ this.configModelRegistry = new MapConfigModelRegistry(modelBuilders);
+ this.zone = zone;
+ }
+
+ public VespaModelFactory(ConfigModelRegistry configModelRegistry) {
+ if (configModelRegistry == null) {
+ this.configModelRegistry = new NullConfigModelRegistry();
+ log.info("Will not load config models from plugins, as no registry is available");
+ } else {
+ this.configModelRegistry = configModelRegistry;
+ }
+ this.zone = Zone.defaultZone();
+ }
+
+ @Override
+ public Version getVersion() {
+ return Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro);
+ }
+
+ @Override
+ public Model createModel(ModelContext modelContext) {
+ return buildModel(createDeployState(modelContext));
+ }
+
+ private VespaModel buildModel(DeployState deployState) {
+ try {
+ return new VespaModel(configModelRegistry, deployState);
+ } catch (IOException | SAXException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private DeployState createDeployState(ModelContext modelContext) {
+ DeployState.Builder builder = new DeployState.Builder()
+ .applicationPackage(modelContext.applicationPackage())
+ .deployLogger(modelContext.deployLogger())
+ .configDefinitionRepo(modelContext.configDefinitionRepo())
+ .fileRegistry(modelContext.getFileRegistry())
+ .permanentApplicationPackage(modelContext.permanentApplicationPackage())
+ .properties(createDeployProperties(modelContext.properties()))
+ .modelHostProvisioner(createHostProvisioner(modelContext))
+ .rotations(modelContext.properties().rotations())
+ .zone(zone);
+ modelContext.previousModel().ifPresent(builder::previousModel);
+ return builder.build();
+ }
+
+ private DeployProperties createDeployProperties(ModelContext.Properties properties) {
+ return new DeployProperties.Builder()
+ .applicationId(properties.applicationId())
+ .configServerSpecs(properties.configServerSpecs())
+ .multitenant(properties.multitenant())
+ .hostedVespa(properties.hostedVespa())
+ .vespaVersion(getVersion())
+ .zone(properties.zone())
+ .build();
+ }
+
+
+ private static HostProvisioner createHostProvisioner(ModelContext modelContext) {
+ if (isHostedVespaRoutingApplication(modelContext)) {
+ //TODO: This belongs in HostedVespaProvisioner.
+ //Added here for now since com.yahoo.config.model.api.HostProvisioner is not created per application,
+ //and allocation is independent of ApplicationPackage.
+ return new HostsXmlProvisioner(hostsXml(modelContext));
+ } else {
+ return modelContext.hostProvisioner().orElse(
+ DeployState.getDefaultModelHostProvisioner(modelContext.applicationPackage()));
+ }
+ }
+
+ private static Reader hostsXml(ModelContext modelContext) {
+ Reader hosts = modelContext.applicationPackage().getHosts();
+ if (hosts == null) {
+ //TODO: throw InvalidApplicationException directly. Not possible now, as it resides in the configserver module.
+ //SessionPreparer maps IllegalArgumentException -> InvalidApplicationException
+ throw new IllegalArgumentException("Hosted vespa routing application must use " + ApplicationPackage.HOSTS
+ + " to allocate hosts.");
+ }
+ return hosts;
+ }
+
+ private static boolean isHostedVespaRoutingApplication(ModelContext modelContext) {
+ ApplicationId id = modelContext.properties().applicationId();
+ return modelContext.properties().hostedVespa() && id.isHostedVespaRoutingApplication();
+ }
+
+ @Override
+ public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) {
+ if (modelContext.appDir().isPresent()) {
+ ApplicationPackageXmlFilesValidator validator =
+ ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(modelContext.appDir().get(),
+ modelContext.deployLogger(),
+ modelContext.vespaVersion());
+ try {
+ validator.checkApplication();
+ ApplicationPackageXmlFilesValidator.checkIncludedDirs(modelContext.applicationPackage());
+ } catch (IllegalArgumentException e) {
+ rethrowUnlessIgnoreErrors(e, ignoreValidationErrors);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ } else {
+ validateXML(modelContext.applicationPackage(), modelContext.deployLogger(), ignoreValidationErrors);
+ }
+ DeployState deployState = createDeployState(modelContext);
+ VespaModel model = buildModel(deployState);
+ List<ConfigChangeAction> changeActions = validateModel(model, deployState, ignoreValidationErrors);
+ return new ModelCreateResult(model, changeActions);
+ }
+
+ private void validateXML(ApplicationPackage applicationPackage, DeployLogger deployLogger, boolean ignoreValidationErrors) {
+ try {
+ applicationPackage.validateXML(deployLogger);
+ } catch (IllegalArgumentException e) {
+ rethrowUnlessIgnoreErrors(e, ignoreValidationErrors);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, boolean ignoreValidationErrors) {
+ try {
+ deployState.getApplicationPackage().validateXML(deployState.getDeployLogger());
+ return Validation.validate(model, ignoreValidationErrors, deployState);
+ } catch (IllegalArgumentException e) {
+ rethrowUnlessIgnoreErrors(e, ignoreValidationErrors);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return new ArrayList<>();
+ }
+
+ private static void rethrowUnlessIgnoreErrors(IllegalArgumentException e, boolean ignoreValidationErrors) {
+ if (!ignoreValidationErrors) {
+ throw e;
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java
new file mode 100644
index 00000000000..155dcca03d9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import java.util.Objects;
+
+/**
+ *
+ * Represents an abstract monitoring service
+ *
+ * @author musum
+ * @since 5.1.20
+ *
+*/
+class AbstractMonitoringSystem implements MonitoringSystem {
+
+ private final Integer interval;
+ private final String clustername;
+
+ public AbstractMonitoringSystem(String clustername, Integer interval) {
+ Objects.requireNonNull(clustername);
+ Objects.requireNonNull(interval);
+ this.clustername = clustername;
+ this.interval = interval;
+ }
+
+ @Override
+ public Integer getInterval() {
+ return interval;
+ }
+
+ @Override
+ public Integer getIntervalSeconds() {
+ return interval * 60;
+ }
+
+ @Override
+ public String getClustername() {
+ return clustername;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
new file mode 100644
index 00000000000..38a1e59433f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -0,0 +1,248 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.cloud.config.ZookeepersConfig;
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.model.deploy.DeployProperties;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.cloud.config.SlobroksConfig;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.filedistribution.FileDistributor;
+import com.yahoo.vespa.model.filedistribution.FileDistributorService;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.vespa.model.HostResource;
+
+/**
+ * This is the admin pseudo-plugin of the Vespa model, responsible for
+ * creating all admin services.
+ *
+ * @author gjoranv
+ */
+public class Admin extends AbstractConfigProducer implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Yamas yamas;
+ private final Map<String,MetricsConsumer> metricsConsumers;
+ private final List<Configserver> configservers = new ArrayList<>();
+
+ private final List<Slobrok> slobroks = new ArrayList<>();
+ private Configserver defaultConfigserver;
+ private Logserver logserver;
+
+ /**
+ * The single cluster controller cluster shared by all content clusters by default when not multitenant.
+ * If multitenant, this is null.
+ */
+ private ContainerCluster clusterControllers;
+
+ private ZooKeepersConfigProvider zooKeepersConfigProvider;
+ private FileDistributionConfigProducer fileDistribution;
+ private final boolean multitenant;
+
+ public Admin(AbstractConfigProducer parent, Yamas yamas, Map<String, MetricsConsumer> metricsConsumers, boolean multitenant) {
+ super(parent, "admin");
+ this.yamas = yamas;
+ this.metricsConsumers = metricsConsumers;
+ this.multitenant = multitenant;
+ }
+
+ public Configserver getConfigserver() {
+ return defaultConfigserver;
+ }
+
+ /** Returns the configured yamas end point. Is null if yamas is not configured */
+ public Yamas getYamas() {
+ return yamas;
+ }
+
+ /** Returns the configured userMetricConsumers. Null if not configured */
+ public Map<String, MetricsConsumer> getUserMetricsConsumers(){
+ return metricsConsumers;
+ }
+
+ /** Returns a list of all config servers */
+ public List<Configserver> getConfigservers() {
+ return configservers;
+ }
+
+ public List<ConfigServerSpec> getConfigServerSpecs() {
+ List<ConfigServerSpec> serverSpecs = new ArrayList<>();
+ for (Configserver server : getConfigservers()) {
+ serverSpecs.add(server.getConfigServerSpec());
+ }
+ return serverSpecs;
+ }
+
+ public void removeSlobroks() { slobroks.clear(); }
+
+ /** Returns an immutable list of the slobroks in this */
+ public List<Slobrok> getSlobroks() { return Collections.unmodifiableList(slobroks); }
+
+ public void setLogserver(Logserver logserver) { this.logserver = logserver; }
+
+ public Logserver getLogserver() { return logserver; }
+
+ public void addConfigservers(List<Configserver> configservers) {
+ this.configservers.addAll(configservers);
+ if (this.configservers.size() > 0) {
+ this.defaultConfigserver = configservers.get(0);
+ }
+ this.zooKeepersConfigProvider = new ZooKeepersConfigProvider(configservers);
+ }
+
+ public void addSlobroks(List<Slobrok> slobroks) {
+ this.slobroks.addAll(slobroks);
+ }
+
+ public ContainerCluster getClusterControllers() { return clusterControllers; }
+
+ public void setClusterControllers(ContainerCluster clusterControllers) {
+ if (multitenant) throw new RuntimeException("Should not use admin cluster controller in a multitenant environment");
+ this.clusterControllers = clusterControllers;
+ }
+
+ public ZooKeepersConfigProvider getZooKeepersConfigProvider() {
+ return zooKeepersConfigProvider;
+ }
+
+ public void getConfig(LogdConfig.Builder builder) {
+ builder.
+ logserver(new LogdConfig.Logserver.Builder().
+ host(logserver.getHostName()).
+ port(logserver.getRelativePort(1)));
+ }
+
+ public void getConfig(SlobroksConfig.Builder builder) {
+ for (Slobrok slob : slobroks) {
+ builder.
+ slobrok(new SlobroksConfig.Slobrok.Builder().
+ connectionspec(slob.getConnectionSpec()));
+ }
+ }
+
+ public void getConfig(ZookeepersConfig.Builder builder) {
+ zooKeepersConfigProvider.getConfig(builder);
+ }
+
+ public void setFileDistribution(FileDistributionConfigProducer fileDistribution) {
+ this.fileDistribution = fileDistribution;
+ }
+
+ public FileDistributionConfigProducer getFileDistributionConfigProducer() {
+ return fileDistribution;
+ }
+
+ public List<HostResource> getClusterControllerHosts() {
+ List<HostResource> hosts = new ArrayList<>();
+ if (multitenant) {
+ if (logserver != null)
+ hosts.add(logserver.getHostResource());
+ } else {
+ for (Configserver configserver : getConfigservers()) {
+ hosts.add(configserver.getHostResource());
+ }
+ }
+ return hosts;
+ }
+
+ /**
+ * Adds services to all hosts in the system.
+ */
+ public void addPerHostServices(List<HostResource> hosts, DeployProperties properties) {
+ if (slobroks.isEmpty()) // TODO: Move to caller
+ slobroks.addAll(createDefaultSlobrokSetup());
+ for (HostResource host : hosts) {
+ if (!host.getHost().isMultitenant()) {
+ addCommonServices(host, properties);
+ }
+ }
+ }
+ private void addCommonServices(HostResource host, DeployProperties properties) {
+ addConfigSentinel(host, properties.applicationId(), properties.zone());
+ addLogd(host);
+ addConfigProxy(host);
+ addFileDistribution(host);
+ }
+
+ private void addConfigSentinel(HostResource host, ApplicationId applicationId, Zone zone) {
+ ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone);
+ addAndInitializeService(host, configSentinel);
+ host.getHost().setConfigSentinel(configSentinel);
+ }
+
+ private void addLogd(HostResource host) {
+ addAndInitializeService(host, new Logd(host.getHost()));
+ }
+
+ private void addConfigProxy(HostResource host) {
+ addAndInitializeService(host, new ConfigProxy(host.getHost()));
+ }
+
+ public void addAndInitializeService(HostResource host, AbstractService service) {
+ service.setHostResource(host);
+ service.initService();
+ }
+
+ private void addFileDistribution(HostResource host) {
+ FileDistributor fileDistributor = fileDistribution.getFileDistributor();
+ HostResource deployHost = getHostSystem().getHostByHostname(fileDistributor.fileSourceHost());
+ if (deployHostIsMissing(deployHost)) {
+ throw new RuntimeException("Could not find host in the application's host system: '" +
+ fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem());
+ }
+
+ FileDistributorService fds = new FileDistributorService(fileDistribution, host.getHost().getHostName(),
+ fileDistribution.getFileDistributor(), fileDistribution.getOptions(), host == deployHost);
+ fds.setHostResource(host);
+ fds.initService();
+ fileDistribution.addFileDistributionService(host.getHost(), fds);
+ }
+
+ private boolean deployHostIsMissing(HostResource deployHost) {
+ return !multitenant && deployHost == null;
+ }
+
+ // If not configured by user: Use default setup: max 3 slobroks, 1 on the default configserver host
+ private List<Slobrok> createDefaultSlobrokSetup() {
+ List<HostResource> hosts = getHostSystem().getHosts();
+ List<Slobrok> slobs = new ArrayList<>();
+ if (logserver != null) {
+ Slobrok slobrok = new Slobrok(this, 0);
+ addAndInitializeService(logserver.getHostResource(), slobrok);
+ slobs.add(slobrok);
+ }
+
+ int n = 0;
+ while ((n < hosts.size()) && (slobs.size() < 3)) {
+ HostResource host = hosts.get(n);
+ if ((logserver== null || host != logserver.getHostResource()) && ! host.getHost().isMultitenant()) {
+ Slobrok newSlobrok = new Slobrok(this, slobs.size());
+ addAndInitializeService(host, newSlobrok);
+ slobs.add(newSlobrok);
+ }
+ n++;
+ }
+ int j = 0;
+ for (Slobrok s : slobs) {
+ s.setProp("index", j);
+ j++;
+ }
+ return slobs;
+ }
+
+ public boolean multitenant() {
+ return multitenant;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
new file mode 100644
index 00000000000..d12ab40835d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
@@ -0,0 +1,145 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import java.util.logging.Logger;
+
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.log.LogLevel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.AbstractService;
+
+/**
+ * Represents a Configserver. There may be one or more Configservers in a
+ * Vespa system.
+ *
+ * NOTE: The Configserver is not started by the config system, and
+ * does not receive any config. It's included here so we know what host
+ * it runs on, and to give an error message if another service tries
+ * to reserve the ports it is using.
+ *
+ * @author gjoranv
+ */
+public class Configserver extends AbstractService {
+ private static final long serialVersionUID = 1L;
+ public static final int defaultPort = 19070;
+ private static final Logger log = Logger.getLogger(Configserver.class.getName());
+
+ public Configserver(AbstractConfigProducer parent, String name) {
+ super(parent, name);
+ portsMeta.on(0).tag("rpc").tag("config");
+ portsMeta.on(1).tag("http").tag("config").tag("state");
+ setProp("clustertype", "admin");
+ setProp("clustername", "admin");
+ monitorService();
+ }
+
+ /**
+ * Returns the desired base port for this service.
+ */
+ public int getWantedPort() {
+ try {
+ // TODO: Provide configserver port as argument when creating this service instead
+ Process process = new ProcessBuilder(Defaults.getDefaults().vespaHome() + "libexec/vespa/vespa-config.pl", "-configserverport").start();
+ InputStream in = process.getInputStream();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ return Integer.parseInt(reader.readLine().trim());
+ } catch (Exception exception) {
+ log.log(LogLevel.DEBUG, "Error reading port from script, using " + defaultPort);
+ return defaultPort;
+ }
+ }
+
+ /**
+ * The desired base port is the only allowed base port.
+ * @return 'true' always
+ */
+ public boolean requiresWantedPort() {
+ return getId() < 2;
+ }
+
+ /**
+ * @return the number of ports needed by the configserver.
+ */
+ public int getPortCount() { return 2; }
+
+ /**
+ * The configserver is not started by the config system!
+ */
+ public boolean getAutostartFlag() { return false; }
+
+ /**
+ * The configserver is not started by the config system!
+ */
+ public boolean getAutorestartFlag() { return false; }
+
+ private int getConfigServerRpcPort() {
+ return getRelativePort(0);
+ }
+
+ private int getConfigServerHttpPort() {
+ return getRelativePort(1);
+ }
+
+ public ConfigServerSpec getConfigServerSpec() {
+ return new Spec(getHostName(), getConfigServerRpcPort(), getConfigServerHttpPort(), ZooKeepersConfigProvider.zkPort);
+ }
+
+ @Override
+ public int getHealthPort() {
+ return getRelativePort(1);
+ }
+
+ // TODO: Remove this implementation when we are on Hosted Vespa.
+ public static class Spec implements ConfigServerSpec {
+ private final String hostName;
+ private final int configServerPort;
+ private final int httpPort;
+ private final int zooKeeperPort;
+ public String getHostName() {
+ return hostName;
+ }
+
+ public int getConfigServerPort() {
+ return configServerPort;
+ }
+
+ public int getHttpPort() {
+ return httpPort;
+ }
+
+ public int getZooKeeperPort() {
+ return zooKeeperPort;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ConfigServerSpec) {
+ ConfigServerSpec other = (ConfigServerSpec)o;
+
+ return hostName.equals(other.getHostName()) &&
+ configServerPort == other.getConfigServerPort() &&
+ httpPort == other.getHttpPort() &&
+ zooKeeperPort == other.getZooKeeperPort();
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return hostName.hashCode();
+ }
+
+ public Spec(String hostName, int configServerPort, int httpPort, int zooKeeperPort) {
+ this.hostName = hostName;
+ this.configServerPort = configServerPort;
+ this.httpPort = httpPort;
+ this.zooKeeperPort = zooKeeperPort;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java
new file mode 100644
index 00000000000..6082ca9f72d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java
@@ -0,0 +1,294 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class to set up the default metrics for all services to be forwarded to Yamas
+ *
+ * @author <a href="mailto:trygve@yahoo-inc.com">Trygve Bolsø Berdal</a>
+ */
+public class DefaultMetricConsumers {
+
+ /**
+ * Populates a map of with consumer as key and metrics for that consumer as value. The metrics
+ * are to be forwarded to consumers (ymon and yamas are the options at the moment).
+ *
+ * @return A map of default metric consumers and default metrics for that consumer.
+ */
+ public Map<String, MetricsConsumer> getDefaultMetricsConsumers() {
+ Map<String, MetricsConsumer> metricsConsumers = new LinkedHashMap<>();
+ metricsConsumers.put("yamas", getDefaultYamasConsumer());
+ metricsConsumers.put("ymon", getDefaultYmonConsumer());
+ return metricsConsumers;
+ }
+
+ private MetricsConsumer getDefaultYmonConsumer(){
+ Map<String, Metric> metricMap = new LinkedHashMap<>();
+ for (Metric metric : commonMetrics()) {
+ metricMap.put(metric.getName(), metric);
+ }
+
+ return new MetricsConsumer("ymon", metricMap);
+ }
+
+ private MetricsConsumer getDefaultYamasConsumer(){
+ // include common metrics
+ List<Metric> metrics = commonMetrics();
+
+ //Search node
+ // jobs
+ metrics.add(new Metric("content.proton.documentdb.job.total.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.attribute_flush.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.memory_index_flush.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.disk_index_fusion.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.document_store_flush.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.document_store_compact.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.bucket_move.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.lid_space_compact.average"));
+ metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average"));
+
+ // lid space
+ metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average"));
+ metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_bloat_factor.average"));
+ metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_bloat_factor.average"));
+ metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_fragmentation_factor.average"));
+ metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_fragmentation_factor.average"));
+ metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_fragmentation_factor.average"));
+
+ // resource usage
+ metrics.add(new Metric("content.proton.resource_usage.disk.average"));
+ metrics.add(new Metric("content.proton.resource_usage.memory.average"));
+ metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.last"));
+ metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average"));
+ metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average"));
+ metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last"));
+
+ // transaction log
+ metrics.add(new Metric("content.proton.transactionlog.entries.average"));
+
+ // document store
+ metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_usage.average"));
+ metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_bloat.average"));
+ metrics.add(new Metric("content.proton.documentdb.ready.document_store.max_bucket_spread.average"));
+ metrics.add(new Metric("content.proton.documentdb.notready.document_store.disk_usage.average"));
+ metrics.add(new Metric("content.proton.documentdb.notready.document_store.disk_bloat.average"));
+ metrics.add(new Metric("content.proton.documentdb.notready.document_store.max_bucket_spread.average"));
+ metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_usage.average"));
+ metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_bloat.average"));
+ metrics.add(new Metric("content.proton.documentdb.removed.document_store.max_bucket_spread.average"));
+
+
+ //Storage
+ metrics.add(new Metric("vds.memfilepersistence.cache.files.average"));
+ metrics.add(new Metric("vds.memfilepersistence.cache.body.average"));
+ metrics.add(new Metric("vds.memfilepersistence.cache.header.average"));
+ metrics.add(new Metric("vds.memfilepersistence.cache.meta.average"));
+ metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average"));
+ metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average"));
+ metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate","visit"));
+
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.average"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate"));
+
+ metrics.add(new Metric("vds.filestor.spi.put.success.average"));
+ metrics.add(new Metric("vds.filestor.spi.remove.success.average"));
+ metrics.add(new Metric("vds.filestor.spi.update.success.average"));
+ metrics.add(new Metric("vds.filestor.spi.get.success.average"));
+ metrics.add(new Metric("vds.filestor.spi.iterate.success.average"));
+ metrics.add(new Metric("vds.filestor.spi.put.success.rate"));
+ metrics.add(new Metric("vds.filestor.spi.remove.success.rate"));
+ metrics.add(new Metric("vds.filestor.spi.update.success.rate"));
+ metrics.add(new Metric("vds.filestor.spi.get.success.rate"));
+ metrics.add(new Metric("vds.filestor.spi.iterate.success.rate"));
+
+
+ //Distributor
+ metrics.add(new Metric("vds.visitor.sum.latency.average"));
+ metrics.add(new Metric("vds.visitor.sum.failed.rate"));
+ metrics.add(new Metric("vds.idealstate.buckets_rechecking.average"));
+ metrics.add(new Metric("vds.idealstate.idealstate_diff.average"));
+ metrics.add(new Metric("vds.idealstate.buckets_toofewcopies.average"));
+ metrics.add(new Metric("vds.idealstate.buckets_toomanycopies.average"));
+ metrics.add(new Metric("vds.idealstate.buckets.average"));
+ metrics.add(new Metric("vds.idealstate.buckets_notrusted.average"));
+
+
+ metrics.add(new Metric("vds.distributor.puts.sum.latency.average"));
+ metrics.add(new Metric("vds.distributor.puts.sum.ok.rate"));
+ metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.removes.sum.latency.average"));
+ metrics.add(new Metric("vds.distributor.removes.sum.ok.rate"));
+ metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.updates.sum.latency.average"));
+ metrics.add(new Metric("vds.distributor.updates.sum.ok.rate"));
+ metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.removelocations.sum.latency.average"));
+ metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate"));
+ metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.gets.sum.latency.average"));
+ metrics.add(new Metric("vds.distributor.gets.sum.ok.rate"));
+ metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate"));
+ metrics.add(new Metric("vds.distributor.docsstored.average"));
+ metrics.add(new Metric("vds.distributor.bytesstored.average"));
+ metrics.add(new Metric("vds.visitor.sum.latency.average"));
+ metrics.add(new Metric("vds.visitor.sum.failed.rate"));
+
+ // Cluster Controller
+
+ metrics.add(new Metric("cluster-controller.down.count.last"));
+ metrics.add(new Metric("cluster-controller.initializing.count.last"));
+ metrics.add(new Metric("cluster-controller.maintenance.count.last"));
+ metrics.add(new Metric("cluster-controller.retired.count.last"));
+ metrics.add(new Metric("cluster-controller.stopping.count.last"));
+ metrics.add(new Metric("cluster-controller.up.count.last"));
+ metrics.add(new Metric("cluster-controller.cluster-state-change.count", "content.cluster-controller.cluster-state-change.count"));
+
+ metrics.add(new Metric("cluster-controller.is-master.last"));
+ // TODO(hakon): Update this name once persistent "count" metrics has been implemented.
+ // DO NOT RELY ON THIS METRIC YET.
+ metrics.add(new Metric("cluster-controller.node-event.count"));
+
+ //Errors from qrserver
+ metrics.add(new Metric("error.timeout.rate","error.timeout"));
+ metrics.add(new Metric("error.backends_oos.rate","error.backends_oos"));
+ metrics.add(new Metric("error.plugin_failure.rate","error.plugin_failure"));
+ metrics.add(new Metric("error.backend_communication_error.rate","error.backend_communication_error"));
+ metrics.add(new Metric("error.empty_document_summaries.rate","error.empty_document_summaries"));
+ metrics.add(new Metric("error.invalid_query_parameter.rate","error.invalid_query_parameter"));
+ metrics.add(new Metric("error.internal_server_error.rate", "error.internal_server_error"));
+ metrics.add(new Metric("error.misconfigured_server.rate","error.misconfigured_server"));
+ metrics.add(new Metric("error.invalid_query_transformation.rate","error.invalid_query_transformation"));
+ metrics.add(new Metric("error.result_with_errors.rate","error.result_with_errors"));
+ metrics.add(new Metric("error.unspecified.rate","error.unspecified"));
+ metrics.add(new Metric("error.unhandled_exception.rate","error.unhandled_exception"));
+ metrics.add(new Metric("http.status.1xx.rate"));
+ metrics.add(new Metric("http.status.2xx.rate"));
+ metrics.add(new Metric("http.status.3xx.rate"));
+ metrics.add(new Metric("http.status.4xx.rate"));
+ metrics.add(new Metric("http.status.5xx.rate"));
+
+
+ // container
+ metrics.add(new Metric("serverRejectedRequests.rate"));
+ metrics.add(new Metric("serverRejectedRequests.count"));
+
+ metrics.add(new Metric("serverThreadPoolSize.average"));
+ metrics.add(new Metric("serverThreadPoolSize.min"));
+ metrics.add(new Metric("serverThreadPoolSize.max"));
+ metrics.add(new Metric("serverThreadPoolSize.rate"));
+ metrics.add(new Metric("serverThreadPoolSize.count"));
+ metrics.add(new Metric("serverThreadPoolSize.last"));
+
+ metrics.add(new Metric("serverActiveThreads.average"));
+ metrics.add(new Metric("serverActiveThreads.min"));
+ metrics.add(new Metric("serverActiveThreads.max"));
+ metrics.add(new Metric("serverActiveThreads.rate"));
+ metrics.add(new Metric("serverActiveThreads.count"));
+ metrics.add(new Metric("serverActiveThreads.last"));
+
+ metrics.add(new Metric("httpapi_latency.average"));
+ metrics.add(new Metric("httpapi_pending.average"));
+ metrics.add(new Metric("httpapi_num_operations.rate"));
+ metrics.add(new Metric("httpapi_num_updates.rate"));
+ metrics.add(new Metric("httpapi_num_removes.rate"));
+ metrics.add(new Metric("httpapi_num_puts.rate"));
+ metrics.add(new Metric("httpapi_succeeded.rate"));
+ metrics.add(new Metric("httpapi_failed.rate"));
+
+
+ // Config server
+ metrics.add(new Metric("configserver.requests.count", "configserver.requests"));
+ metrics.add(new Metric("configserver.failedRequests.count", "configserver.failedRequests"));
+ metrics.add(new Metric("configserver.latency.average", "configserver.latency"));
+ metrics.add(new Metric("configserver.cacheConfigElems.last", "configserver.cacheConfigElems"));
+ metrics.add(new Metric("configserver.cacheChecksumElems.last", "configserver.cacheChecksumElems"));
+ metrics.add(new Metric("configserver.hosts.last", "configserver.hosts"));
+ metrics.add(new Metric("configserver.delayedResponses.count", "configserver.delayedResponses"));
+ metrics.add(new Metric("configserver.sessionChangeErrors.count", "configserver.sessionChangeErrors"));
+
+
+ Map<String, Metric> metricMap = new LinkedHashMap<>();
+ for (Metric metric : metrics) {
+ metricMap.put(metric.getName(), metric);
+ }
+
+ return new MetricsConsumer("yamas", metricMap);
+ }
+
+ // Common metrics for ymon and yamas. For ymon metric names needs to be less then 19 characters long
+ private List<Metric> commonMetrics(){
+ List<Metric> metrics = new ArrayList<>();
+
+ //Searchnode
+ metrics.add(new Metric("proton.numstoreddocs.last", "documents_total"));
+ metrics.add(new Metric("proton.numindexeddocs.last", "documents_ready"));
+ metrics.add(new Metric("proton.numactivedocs.last", "documents_active"));
+ metrics.add(new Metric("proton.numremoveddocs.last", "documents_removed"));
+
+ metrics.add(new Metric("proton.docsinmemory.last", "documents_inmemory"));
+ metrics.add(new Metric("proton.diskusage.last", "diskusage"));
+ metrics.add(new Metric("proton.memoryusage.max", "content.proton.memoryusage.max"));
+ metrics.add(new Metric("proton.transport.query.count.rate", "query_requests"));
+ metrics.add(new Metric("proton.transport.docsum.docs.rate", "document_requests"));
+ metrics.add(new Metric("proton.transport.docsum.latency.average", "content.proton.transport.docsum.latency.average"));
+ metrics.add(new Metric("proton.transport.query.latency.average", "query_latency"));
+
+ //Docproc - per chain
+ metrics.add(new Metric("documents_processed.rate", "documents_processed"));
+
+ //Qrserver
+ metrics.add(new Metric("peak_qps.average", "peak_qps"));
+ metrics.add(new Metric("search_connections.average", "search_connections"));
+ metrics.add(new Metric("active_queries.average", "active_queries"));
+ metrics.add(new Metric("queries.rate", "queries"));
+ metrics.add(new Metric("query_latency.average", "mean_query_latency"));
+ metrics.add(new Metric("query_latency.max", "max_query_latency"));
+ metrics.add(new Metric("query_latency.95percentile", "95p_query_latency"));
+ metrics.add(new Metric("query_latency.99percentile", "99p_query_latency"));
+ metrics.add(new Metric("failed_queries.rate", "failed_queries"));
+ metrics.add(new Metric("hits_per_query.average", "hits_per_query"));
+ metrics.add(new Metric("empty_results.rate", "empty_results"));
+ metrics.add(new Metric("requestsOverQuota.rate"));
+ metrics.add(new Metric("requestsOverQuota.count"));
+
+ //Storage
+ metrics.add(new Metric("vds.datastored.alldisks.docs.average","docs"));
+ metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes"));
+ metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average","visitorlifetime"));
+ metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average","visitorqueuewait"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate","put"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate","remove"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate","get"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate","update"));
+ metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize"));
+ metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait"));
+
+
+ //Distributor
+ metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate","deleteok"));
+ metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate","deletefailed"));
+ metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average","deletepending"));
+ metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate","mergeok"));
+ metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate","mergefailed"));
+ metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average","mergepending"));
+ metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate","splitok"));
+ metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate","splitfailed"));
+ metrics.add(new Metric("vds.idealstate.split_bucket.pending.average","splitpending"));
+ metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate","joinok"));
+ metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate","joinfailed"));
+ metrics.add(new Metric("vds.idealstate.join_bucket.pending.average","joinpending"));
+
+ return metrics;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java
new file mode 100644
index 00000000000..a5115d6c358
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.binaryprefix.BinaryPrefix;
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+import com.yahoo.cloud.config.filedistribution.FiledistributorConfig;
+
+/**
+ * Options for controlling the behavior of the file distribution services.
+ * @author tonytv
+ */
+public class FileDistributionOptions implements FiledistributorConfig.Producer {
+ public static FileDistributionOptions defaultOptions() {
+ return new FileDistributionOptions();
+ }
+
+ private FileDistributionOptions() {}
+
+ private BinaryScaledAmount uploadbitrate = new BinaryScaledAmount();
+ private BinaryScaledAmount downloadbitrate = new BinaryScaledAmount();
+
+ //Called through reflection
+ public void downloadbitrate(BinaryScaledAmount amount) {
+ ensureNonNegative(amount);
+ downloadbitrate = amount;
+ }
+
+ //Called through reflection
+ public void uploadbitrate(BinaryScaledAmount amount) {
+ ensureNonNegative(amount);
+ uploadbitrate = amount;
+ }
+
+ private void ensureNonNegative(BinaryScaledAmount amount) {
+ if (amount.amount < 0)
+ throw new IllegalArgumentException("Expected non-negative number, got " + amount.amount);
+ }
+
+ private int byteRate(BinaryScaledAmount bitRate) {
+ BinaryScaledAmount byteRate = bitRate.divide(8);
+ return (int)byteRate.as(BinaryPrefix.unit);
+ }
+
+ @Override
+ public void getConfig(FiledistributorConfig.Builder builder) {
+ builder.maxuploadspeed((double)byteRate(uploadbitrate));
+ builder.maxdownloadspeed((double)byteRate(downloadbitrate));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java
new file mode 100644
index 00000000000..12341297ebb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+
+/**
+ * Represents the Logserver. There is exactly one logserver in a Vespa
+ * system.
+ *
+ * @author gjoranv
+ */
+public class Logserver extends AbstractService {
+ private static final long serialVersionUID = 1L;
+ private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive";
+
+ public Logserver(AbstractConfigProducer parent) {
+ super(parent, "logserver");
+ portsMeta.on(0).tag("unused");
+ portsMeta.on(1).tag("logtp");
+ portsMeta.on(2).tag("logtp").tag("telnet").tag("last-errors-holder");
+ portsMeta.on(3).tag("logtp").tag("telnet").tag("replicator");
+ setProp("clustertype", "admin");
+ setProp("clustername", "admin");
+ monitorService();
+ }
+
+ /**
+ * @return the startup command for the logserver
+ */
+ public String getStartupCommand() {
+ return "exec $ROOT/bin/logserver-start " + getMyJVMArgs() + " " + getJvmArgs();
+ }
+
+ /**
+ * @return the jvm args to be used by the logserver.
+ */
+ private String getMyJVMArgs() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("-Dlogserver.http.port=").append(getRelativePort(0));
+ sb.append(" ");
+ sb.append("-Dlogserver.listenport=").append(getRelativePort(1));
+ sb.append(" ");
+ sb.append("-Dlogserver.last-errors-holder.port=").append(getRelativePort(2));
+ sb.append(" ");
+ sb.append("-Dlogserver.replicator.port=").append(getRelativePort(3));
+ sb.append(" ");
+ sb.append("-Dlogserver.logarchive.dir=" + logArchiveDir);
+ return sb.toString();
+ }
+
+ /**
+ * Returns the desired base port for this service.
+ */
+ public int getWantedPort() {
+ return 19080;
+ }
+
+ /**
+ * The desired base port is the only allowed base port.
+ *
+ * @return 'true' always
+ */
+ public boolean requiresWantedPort() {
+ return true;
+ }
+
+ /**
+ * @return the number of ports needed by the logserver.
+ */
+ public int getPortCount() {
+ return 4;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java
new file mode 100644
index 00000000000..e1056f954dc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Helper class to model a metric.
+ *
+ * @author trygve
+ */
+public class Metric {
+
+ private final String name;
+ private String outputName;
+ private final Map<String, String> dimensions = new HashMap<>();
+ private final String description;
+
+ /**
+ * @param name The metric name
+ * @param outputName The name of the metric in yamas
+ * @param description The description of this metric
+ */
+ public Metric(String name, String outputName, String description) {
+ this.name = name;
+ this.outputName = outputName;
+ this.description = description;
+ }
+
+ /**
+ * Creates a metric with empty dimensions and consumers containing the default consumer
+ *
+ * @param name the metric name
+ * @param outputName name tp be used in yamas
+ */
+ public Metric(String name, String outputName) {
+ this(name, outputName, "");
+ }
+
+ /**
+ * Creates a metric with same outputname as metricname and empty dimensions and consumers containing the default consumer and
+ *
+ * @param name The name of the metric, same name used for outputname
+ */
+ public Metric(String name) {
+ this(name, name);
+ }
+
+ public String getDescription() {
+ return this.description;
+ }
+
+
+ public String getOutputName() {
+ return outputName;
+ }
+
+ public void setOutputName(String outputName) {
+ this.outputName = outputName;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Map<String, String> getDimensions() {
+ return dimensions;
+ }
+
+ @Override
+ public String toString() {
+ return "Metric{" +
+ "name='" + name + '\'' +
+ ", outputName='" + outputName + '\'' +
+ ", dimensions=" + dimensions +
+ '}';
+ }
+
+
+ /**
+ * Two metrics are considered equal if they have the same name.
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Metric metric = (Metric) o;
+
+ return name.equals(metric.name);
+
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java
new file mode 100644
index 00000000000..0bed49f38ac
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import java.util.Map;
+
+/**
+ * Represents an arbitrary metric consumer
+ *
+ * @author trygve
+ */
+public class MetricsConsumer {
+ private final String consumer;
+ private final Map<String, Metric> metrics;
+
+ /**
+ * @param consumer The consumer
+ * @param metrics The metrics for the the consumer
+ */
+ public MetricsConsumer(String consumer, Map<String, Metric> metrics) {
+ this.consumer = consumer;
+ this.metrics = metrics;
+ }
+
+ public String getConsumer() {
+ return consumer;
+ }
+
+ /**
+ * @return Map of metric with metric name as key
+ */
+ public Map<String, Metric> getMetrics() {
+ return metrics;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java
new file mode 100644
index 00000000000..1e9b485a939
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * A config provider for the model config. The ModelConfig is a common config and produced by {@link ApplicationConfigProducerRoot} , this config
+ * producer exists to make the admin/model config id exist for legacy reasons.
+ * @author <a href="mailto:musum@yahoo-inc.com">musum</a>
+ * @author gjoranv
+ * @since 5.0.8
+ */
+public class ModelConfigProvider extends AbstractConfigProducer {
+
+ public ModelConfigProvider(AbstractConfigProducer<?> parent) {
+ super(parent, "model");
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java
new file mode 100644
index 00000000000..de95c9a8d9a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+/**
+ * Interface for different monitoring services
+ *
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ */
+public interface MonitoringSystem {
+ /**
+ * @return Snapshot interval in minutes
+ */
+ public Integer getInterval();
+
+ /**
+ * @return Snapshot interval in seconds.
+ */
+ public Integer getIntervalSeconds();
+
+ /**
+ * @return the monitoring cluster name
+ */
+ public String getClustername();
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
new file mode 100644
index 00000000000..a2b7c7a5532
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+
+/**
+ * Represents a Slobrok service.
+ *
+ * @author gjoranv
+ */
+public class Slobrok extends AbstractService {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param parent The parent ConfigProducer.
+ * @param index unique index for all slobroks
+ */
+ public Slobrok(AbstractConfigProducer parent, int index) {
+ super(parent, "slobrok." + index);
+ portsMeta.on(0).tag("rpc").tag("admin").tag("status");
+ portsMeta.on(1).tag("http").tag("state");
+ setProp("index", index);
+ setProp("clustertype", "slobrok");
+ setProp("clustername", "admin");
+ monitorService();
+ }
+
+ @Override
+ public boolean requiresConsecutivePorts() {
+ return false;
+ }
+
+ @Override
+ public int getWantedPort() {
+ if (getId() == 1) {
+ return 19099;
+ } else {
+ return 0;
+ }
+ }
+
+ public String getStartupCommand() {
+ return "exec $ROOT/bin/slobrok -p " + getPort() +
+ " -s " + getStatePort() +
+ " -c " + getConfigId();
+ }
+
+ /**
+ * @return The number of ports needed by the slobrok.
+ */
+ public int getPortCount() {
+ return 2;
+ }
+
+ /**
+ * @return The port on which this slobrok should respond, as a String.
+ */
+ public String getPort() {
+ return String.valueOf(getRelativePort(0));
+ }
+
+ /**
+ * @return The port on which the state server should respond
+ */
+ public String getStatePort() {
+ return String.valueOf(getRelativePort(1));
+ }
+
+ /**
+ * @return The connection spec to this Slobrok
+ */
+ public String getConnectionSpec() {
+ return "tcp/" + getHostName() + ":" + getPort();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java
new file mode 100644
index 00000000000..43775fe0dec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Properties for yamas monitoring service
+ *
+ * @author musum
+ * @since 5.1.20
+ */
+public class Yamas extends AbstractMonitoringSystem implements Serializable {
+ public Yamas(String clustername, Integer interval) {
+ super(clustername, interval);
+ }
+}
+
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java
new file mode 100644
index 00000000000..6c8b486c8a6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.collections.CollectionUtil;
+import com.yahoo.cloud.config.ZookeepersConfig;
+import com.yahoo.config.model.api.ConfigServerSpec;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author tonytv
+ */
+public class ZooKeepersConfigProvider implements ZookeepersConfig.Producer {
+
+ public static final int zkPort = Integer.getInteger("zk_port", 2181);
+
+ private final List<Configserver> configServers;
+
+ public ZooKeepersConfigProvider(List<Configserver> configServers) {
+ if (configServers == null) {
+ configServers = new ArrayList<>();
+ }
+ this.configServers = configServers;
+ }
+
+ // format for each element: hostname:port
+ public List<String> getZooKeepers() {
+ List<String> servers = new ArrayList<>();
+ for (Configserver server : configServers) {
+ ConfigServerSpec serverSpec = server.getConfigServerSpec();
+ servers.add(serverSpec.getHostName() + ":" + serverSpec.getZooKeeperPort());
+
+ }
+ return servers;
+ }
+
+ // format: hostname1:port2,hostname2:port2,...
+ public String getZooKeepersConnectionSpec() {
+ return CollectionUtil.mkString(getZooKeepers(), ",");
+ }
+
+ @Override
+ public void getConfig(ZookeepersConfig.Builder builder) {
+ builder.zookeeperserverlist(getZooKeepersConnectionSpec());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java
new file mode 100644
index 00000000000..b82f58fbb40
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin.clustercontroller;
+
+import com.google.common.base.Joiner;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.cloud.config.ZookeepersConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.admin.Configserver;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * Used if clustercontroller is run standalone (not as part of the config server ZooKeeper cluster)
+ * to provide common configs to container components.
+ *
+ * @author lulf
+ * @since 5.6
+ */
+public class ClusterControllerCluster extends AbstractConfigProducer<ContainerCluster> implements ZookeeperServerConfig.Producer, ZookeepersConfig.Producer {
+
+ private static final int ZK_CLIENT_PORT = 2181;
+ private ContainerCluster containerCluster = null;
+
+ public ClusterControllerCluster(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ }
+
+ @Override
+ public void getConfig(ZookeeperServerConfig.Builder builder) {
+ builder.clientPort(ZK_CLIENT_PORT);
+ for (Container c : containerCluster.getContainers()) {
+ ClusterControllerContainer container = (ClusterControllerContainer) c;
+ ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder();
+ serverBuilder.hostname(container.getHostName());
+ serverBuilder.id(container.getIndex());
+ builder.server(serverBuilder);
+ }
+ }
+
+ @Override
+ public void getConfig(ZookeepersConfig.Builder builder) {
+ final Collection<String> controllerHosts = new ArrayList<>();
+ for (Container container : containerCluster.getContainers()) {
+ controllerHosts.add(container.getHostName() + ":" + ZK_CLIENT_PORT);
+ }
+ builder.zookeeperserverlist(Joiner.on(",").join(controllerHosts));
+ }
+
+ @Override
+ protected void addChild(ContainerCluster cluster) {
+ super.addChild(cluster);
+ this.containerCluster = cluster;
+ }
+
+ @Override
+ public void validate() {
+ assert(containerCluster != null);
+ for (Container c1 : containerCluster.getContainers()) {
+ assert(c1 instanceof ClusterControllerContainer);
+ for (Service service : c1.getHostResource().getServices()) {
+ if (service instanceof Configserver) {
+ throw new IllegalArgumentException("Error validating cluster controller cluster: cluster controller '" + c1.getConfigId() + "' is set to run on the same host as a configserver");
+ }
+ }
+ for (Container c2 : containerCluster.getContainers()) {
+ if (c1 != c2 && c1.getHostName().equals(c2.getHostName())) {
+ throw new IllegalArgumentException("Error validating cluster controller cluster: cluster controllers '" + c1.getConfigId() + "' and '" + c2.getConfigId() + "' share the same host");
+ }
+ }
+ }
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java
new file mode 100644
index 00000000000..8ec1cecf47a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin.clustercontroller;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+
+/**
+ * Sets up a simple component to keep the state of the cluster controller, even when configuration changes.
+ */
+public class ClusterControllerComponent extends SimpleComponent
+{
+ public ClusterControllerComponent() {
+ super(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller"),
+ new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.ClusterController"),
+ new ComponentSpecification("clustercontroller-apps"))));
+ }
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java
new file mode 100644
index 00000000000..e3e79a57261
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin.clustercontroller;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.vespa.config.content.FleetcontrollerConfig;
+import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+/**
+ * Model serving class. Wraps fleet controller config serving.
+ */
+public class ClusterControllerConfigurer extends SimpleComponent implements StorDistributionConfig.Producer,
+ FleetcontrollerConfig.Producer
+{
+ private final ContentCluster cluster;
+ private final int index;
+ private final int nodeCount;
+
+ public ClusterControllerConfigurer(ContentCluster cluster, int index, int nodeCount) {
+ super(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller" + "-" + cluster.getName() + "-configurer"),
+ new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.ClusterControllerClusterConfigurer"),
+ new ComponentSpecification("clustercontroller-apps"))));
+ this.cluster = cluster;
+ this.index = index;
+ this.nodeCount = nodeCount;
+ }
+
+ @Override
+ public void getConfig(StorDistributionConfig.Builder builder) {
+ cluster.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(FleetcontrollerConfig.Builder builder) {
+ cluster.getConfig(builder);
+ cluster.getClusterControllerConfig().getConfig(builder);
+ builder.index(index);
+ builder.fleet_controller_count(nodeCount);
+ builder.http_port(0);
+ builder.rpc_port(0);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
new file mode 100644
index 00000000000..1290b0b22d6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.admin.clustercontroller;
+
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.BundlesConfig;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.log.LogLevel;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.config.content.FleetcontrollerConfig;
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.component.AccessLogComponent;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.Handler;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Extends the container producer to allow us to override ports.
+ */
+@RestartConfigs({FleetcontrollerConfig.class, ZookeeperServerConfig.class})
+public class ClusterControllerContainer extends Container implements BundlesConfig.Producer, ZookeeperServerConfig.Producer {
+ private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps");
+ private static final ComponentSpecification ZKFACADE_BUNDLE = new ComponentSpecification("zkfacade");
+ private final int index;
+
+ private final Set<String> bundles = new TreeSet<>();
+
+ public ClusterControllerContainer(AbstractConfigProducer parent, int index, boolean runStandaloneZooKeeper) {
+ super(parent, "" + index);
+ this.index = index;
+ addHandler(
+ new Handler(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller-status"),
+ new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler"),
+ CLUSTERCONTROLLER_BUNDLE))), "clustercontroller-status/*"
+ );
+ addHandler(
+ new Handler(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller-state-restapi-v2"),
+ new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler"),
+ CLUSTERCONTROLLER_BUNDLE))), "cluster/v2/*"
+ );
+ if (runStandaloneZooKeeper) {
+ addComponent(new Component<>(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller-zkrunner"),
+ new ComponentSpecification("com.yahoo.vespa.zookeeper.ZooKeeperServer"), ZKFACADE_BUNDLE))));
+ addComponent(new Component<>(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller-zkprovider"),
+ new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.StandaloneZooKeeperProvider"), CLUSTERCONTROLLER_BUNDLE))));
+ } else {
+ addComponent(new Component<>(new ComponentModel(new BundleInstantiationSpecification(
+ new ComponentSpecification("clustercontroller-zkprovider"),
+ new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.DummyZooKeeperProvider"), CLUSTERCONTROLLER_BUNDLE))));
+ }
+ addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-apps-jar-with-dependencies.jar"));
+ addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-apputil-jar-with-dependencies.jar"));
+ addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-core-jar-with-dependencies.jar"));
+ addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-utils-jar-with-dependencies.jar"));
+ addBundle("file:" + getDefaults().underVespaHome("lib/jars/zkfacade-jar-with-dependencies.jar"));
+
+ log.log(LogLevel.DEBUG, "Adding access log for cluster controller ...");
+ addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.queryAccessLog, "controller"));
+ }
+
+ @Override
+ public int getWantedPort() {
+ return 19050;
+ }
+
+ @Override
+ public boolean requiresWantedPort() {
+ return index == 0;
+ }
+
+ @Override
+ public String getServiceType() {
+ return "container-clustercontroller";
+ }
+
+ private void addHandler(Handler h, String binding) {
+ h.addServerBindings("http://*/" + binding,
+ "https://*/" + binding);
+ super.addHandler(h);
+ }
+
+ public void addBundle(String bundlePath) {
+ bundles.add(bundlePath);
+ }
+
+ @Override
+ public void getConfig(BundlesConfig.Builder builder) {
+ for (String bundle : bundles) {
+ builder.bundle(bundle);
+ }
+ }
+
+ @Override
+ public void getConfig(ZookeeperServerConfig.Builder builder) {
+ builder.myid(index);
+ }
+
+ int getIndex() {
+ return index;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java
new file mode 100644
index 00000000000..bd3556cc605
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.admin.clustercontroller;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java
new file mode 100644
index 00000000000..e3c7221f7b7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Provides the classes for the admin components of the Vespa config
+ * model.
+ *
+ * The {@link com.yahoo.vespa.model.admin.Admin Admin} class is
+ * the natural starting point. It takes the admin part of the
+ * user-defined application specification as input argument to its
+ * constructor. The services given in the specification are
+ * instantiated, in addition to some mandatory services (e.g. {@link
+ * com.yahoo.vespa.model.admin.Slobrok Slobrok}) that will be created
+ * even if the user does not specify them.
+ */
+@ExportPackage
+package com.yahoo.vespa.model.admin;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java
new file mode 100644
index 00000000000..00553f681fe
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.config.application.api.ComponentInfo;
+import com.yahoo.config.application.api.DeployLogger;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+import java.util.logging.Level;
+import java.util.zip.ZipException;
+
+/**
+ * A validator for bundles. Uses BND library for some of the validation (not active yet)
+ *
+ * @author musum
+ * @since 2010-11-11
+ */
+public class ComponentValidator extends Validator {
+ private JarFile jarFile;
+
+ public ComponentValidator() {
+ }
+
+ public ComponentValidator(JarFile jarFile) {
+ this.jarFile = jarFile;
+ }
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ ApplicationPackage app = deployState.getApplicationPackage();
+ for (ComponentInfo info : app.getComponentsInfo(deployState.getProperties().vespaVersion())) {
+ try {
+ this.jarFile = new JarFile(app.getFileReference(Path.fromString(info.getPathRelativeToAppDir())));
+ } catch (ZipException e) {
+ throw new IllegalArgumentException("Error opening jar file '" + info.getPathRelativeToAppDir() +
+ "'. Please check that this is a valid jar file");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ validateAll(deployState.getDeployLogger());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ public void validateAll(DeployLogger deployLogger) throws IOException {
+ validateOSGIHeaders(deployLogger);
+ }
+
+ public void validateOSGIHeaders(DeployLogger deployLogger) throws IOException {
+ Manifest mf = jarFile.getManifest();
+ if (mf == null) {
+ throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarFile.getName());
+ }
+
+ // Check for required OSGI headers
+ Attributes attributes = mf.getMainAttributes();
+ HashSet<String> mfAttributes = new HashSet<>();
+ for (Object attributeSet : attributes.entrySet()) {
+ Map.Entry<Object, Object> e = (Map.Entry<Object, Object>) attributeSet;
+ mfAttributes.add(e.getKey().toString());
+ }
+ List<String> requiredOSGIHeaders = Arrays.asList(
+ "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version");
+ for (String header : requiredOSGIHeaders) {
+ if (!mfAttributes.contains(header)) {
+ throw new IllegalArgumentException("Required OSGI header '" + header +
+ "' was not found in manifest in '" + jarFile.getName() + "'");
+ }
+ }
+
+ if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) {
+ deployLogger.log(Level.WARNING, "Deploying snapshot bundle " + jarFile.getName() +
+ ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml.");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java
new file mode 100644
index 00000000000..9b62d66bc07
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+
+import java.io.Reader;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Validates that deployment file (deployment.xml) has valid values (for now
+ * only global-service-id is validated)
+ *
+ * @author musum
+ */
+public class DeploymentFileValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment();
+
+ if (deployment.isPresent()) {
+ Reader deploymentReader = deployment.get();
+ DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader);
+ final Optional<String> globalServiceId = deploymentSpec.globalServiceId();
+ if (globalServiceId.isPresent()) {
+ Set<ContainerCluster> containerClusters = model.getRoot().configModelRepo().getModels(ContainerModel.class).stream().
+ map(ContainerModel::getCluster).filter(cc -> cc.getName().equals(globalServiceId.get())).collect(Collectors.toSet());
+ if (containerClusters.size() != 1) {
+ throw new IllegalArgumentException("global-service-id '" + globalServiceId.get() + "' specified in deployment.xml does not match any container cluster id");
+ }
+ }
+ }
+ }
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
new file mode 100644
index 00000000000..d9aa600a840
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.DocumentDatabase;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+
+import java.util.Map;
+
+/**
+ * match:prefix for indexed fields not supported
+ * @author vegardh
+ *
+ */
+public class NoPrefixForIndexes extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ for (AbstractSearchCluster cluster : model.getSearchClusters()) {
+ if (cluster instanceof IndexedSearchCluster) {
+ IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
+ for (DocumentDatabase docDb : sc.getDocumentDbs()) {
+ DerivedConfiguration sdConfig = docDb.getDerivedConfiguration();
+ Search search = sdConfig.getSearch();
+ for (SDField field : search.allFieldsList()) {
+ if (field.doesIndexing()) {
+ //if (!field.getIndexTo().isEmpty() && !field.getIndexTo().contains(field.getName())) continue;
+ if (field.getMatching().getAlgorithm().equals(Matching.Algorithm.PREFIX)) {
+ failField(search, field);
+ }
+ for (Map.Entry<String, Index> e : field.getIndices().entrySet()) {
+ if (e.getValue().isPrefix()) {
+ failField(search, field);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void failField(Search search, SDField field) {
+ throw new IllegalArgumentException("For search '" + search.getName() + "', field '" + field.getName() +
+ "': match/index:prefix is not supported for indexes.");
+ }
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
new file mode 100644
index 00000000000..89a9d245de7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.io.IOUtils;
+import com.yahoo.log.InvalidLogFormatException;
+import com.yahoo.log.LogMessage;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.system.ProcessExecuter;
+import com.yahoo.text.StringUtilities;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.collections.Pair;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.vespa.config.search.IndexschemaConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.DocumentDatabase;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+import com.yahoo.vespa.model.search.SearchCluster;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+
+/**
+ * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done
+ * by running through the binary 'verify_ranksetup'
+ *
+ * @author vegardh
+ *
+ */
+public class RankSetupValidator extends Validator {
+ private final boolean force;
+
+ public RankSetupValidator(boolean force) {
+ this.force = force;
+ }
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ File cfgDir = makeTempConfigDir(deployState.getDeployLogger());
+ if (cfgDir == null) return;
+
+ for (AbstractSearchCluster cluster : model.getSearchClusters()) {
+ // Skipping rank expression checking for streaming clusters, not implemented yet
+ if (cluster.isRealtime()) {
+ IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
+ String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/";
+ for (DocumentDatabase docDb : sc.getDocumentDbs()) {
+ String searchDir = clusterDir + docDb.getDerivedConfiguration().getSearch().getName() + "/";
+ writeConfigs(searchDir, docDb);
+ if (!validate("dir:" + searchDir, sc, docDb.getDerivedConfiguration().getSearch().getName(), deployState.getDeployLogger(), cfgDir)) {
+ return;
+ }
+ }
+ }
+ }
+ deleteTempDir(cfgDir);
+ }
+
+ private boolean validate(String configId, SearchCluster sc, String sdName, DeployLogger logger, File tempDir) {
+ try {
+ boolean ret = execValidate(configId, sc, sdName, logger);
+ if (!ret) {
+ // Give up, don't say same error msg repeatedly
+ deleteTempDir(tempDir);
+ }
+ return ret;
+ } catch (IllegalArgumentException e) {
+ deleteTempDir(tempDir);
+ throw e;
+ }
+ }
+
+ private void deleteTempDir(File dir) {
+ if (!IOUtils.recursiveDeleteDir(dir)) {
+ throw new RuntimeException("Failed deleting " + dir);
+ }
+ }
+
+ private void writeConfigs(String dir, AbstractConfigProducer producer) {
+ try {
+ RankProfilesConfig.Builder rpb = new RankProfilesConfig.Builder();
+ RankProfilesConfig.Producer rpProd = (RankProfilesConfig.Producer) producer;
+ rpProd.getConfig(rpb);
+ writeConfig(dir, "rank-profiles.cfg", new RankProfilesConfig(rpb));
+
+ IndexschemaConfig.Builder isB = new IndexschemaConfig.Builder();
+ IndexschemaConfig.Producer isProd = (IndexschemaConfig.Producer) producer;
+ isProd.getConfig(isB);
+ writeConfig(dir, "indexschema.cfg", new IndexschemaConfig(isB));
+
+ AttributesConfig.Builder acb = new AttributesConfig.Builder();
+ AttributesConfig.Producer acProd = (AttributesConfig.Producer) producer;
+ acProd.getConfig(acb);
+ writeConfig(dir, "attributes.cfg", new AttributesConfig(acb));
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ private static void writeConfig(String dir, String configName, ConfigInstance config) throws IOException {
+ IOUtils.writeFile(dir + configName, StringUtilities.implodeMultiline(ConfigInstance.serialize(config)), false);
+ }
+
+ private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger logger) {
+ String job = "verify_ranksetup-bin " + configId;
+ ProcessExecuter executer = new ProcessExecuter();
+ try {
+ Pair<Integer, String> ret = executer.exec(job);
+ if (ret.getFirst() != 0) {
+ validateFail(ret.getSecond(), sc, sdName, logger);
+ }
+ } catch (IOException e) {
+ validateWarn(executer, e, logger);
+ return false;
+ }
+ return true;
+ }
+
+ @SuppressWarnings({"UnusedDeclaration"})
+ private void validateWarn(ProcessExecuter executer, Exception e, DeployLogger logger) {
+ String msg = "Unable to execute 'verify_ranksetup', validation of rank expressions will only take place when you start Vespa: " +
+ Exceptions.toMessageString(e);
+ logger.log(Level.WARNING, msg);
+ }
+
+ private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger logger) {
+ String errMsg = "For search cluster '" + sc.getClusterName() + "', search definition '" + sdName + "': error in rank setup. Details:\n";
+ for (String line : output.split("\n")) {
+ // Remove debug lines from start script
+ if (line.startsWith("debug\t")) continue;
+ try {
+ LogMessage logmsg = LogMessage.parseNativeFormat(line);
+ errMsg = errMsg + logmsg.getLevel() + ": " + logmsg.getPayload() + "\n";
+ } catch (InvalidLogFormatException e) {
+ errMsg = errMsg + line + "\n";
+ }
+ }
+ if (force) {
+ logger.log(Level.WARNING, errMsg + "(Continuing because of force.)");
+ } else {
+ throw new IllegalArgumentException(errMsg);
+ }
+ }
+
+ private File makeTempConfigDir(DeployLogger deployLogger) {
+ String name = "/tmp/deploy_ranksetup_" + System.currentTimeMillis() + "/";
+ File tempDir = new File(name);
+ if (!tempDir.mkdir()) {
+ deployLogger.log(Level.WARNING, "Not able to create '" + name + "' when validating rank setup");
+ return null;
+ }
+ return tempDir;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java
new file mode 100644
index 00000000000..1bf15d6dd4f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.vespa.model.application.validation.change.ConfigValueChangeValidator;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Services should use this annotation to list all consumed configurations which contain definitions flagged with restart.
+ * This annotation is required for the {@link ConfigValueChangeValidator}
+ * to detect config changes that will require restart of some services. The {@link com.yahoo.config.ConfigInstance}
+ * values are inherited; any configs annotated on a service will be inherited to all sub-classes of that service.
+ * These sub-classes can supplement with more ConfigInstances (in addition to the inherited one) with the annotation.
+ * This is different the inheritance that {@link java.lang.annotation.Inherited} provides, where sub-classes can either
+ * inherit or override the annotation from the super class.
+ *
+ * NOTE: This annotation is will only have effect on subclasses of {@link com.yahoo.vespa.model.Service}.
+ * Do not use this annotation on other types config producers.
+ *
+ * @author bjorncs
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface RestartConfigs {
+ Class<? extends ConfigInstance>[] value() default {};
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java
new file mode 100644
index 00000000000..ad79b1eed45
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.document.select.DocumentSelector;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+
+
+/**
+ * Validates routing selector for search and content clusters
+ *
+ */
+public class RoutingSelectorValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ for (AbstractSearchCluster cluster : model.getSearchClusters()) {
+ if (cluster instanceof IndexedSearchCluster) {
+ IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
+ String routingSelector = sc.getRoutingSelector();
+ if (routingSelector == null) continue;
+ try {
+ new DocumentSelector(routingSelector);
+ } catch (com.yahoo.document.select.parser.ParseException e) {
+ throw new IllegalArgumentException("Failed to parse routing selector for search cluster '" +
+ sc.getClusterName() + "'", e);
+ }
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java
new file mode 100644
index 00000000000..6e94c39cc51
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.util.List;
+
+/**
+ * Validates routing
+ *
+ */
+public class RoutingValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ List<String> errors = model.getRouting().getErrors();
+ if (!errors.isEmpty()) {
+ StringBuilder msg = new StringBuilder();
+ msg.append("The routing specification contains ").append(errors.size()).append(" error(s):\n");
+ for (int i = 0, len = errors.size(); i < len; ++i) {
+ msg.append(i + 1).append(". ").append(errors.get(i)).append("\n");
+ }
+ throw new IllegalArgumentException(msg.toString());
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
new file mode 100644
index 00000000000..c03fb0617b8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.SearchDefinition;
+
+import java.util.List;
+
+/**
+ * This Validator iterates through all search cluster in the given VespaModel to make sure that there are no custom
+ * structs defined in any of its search definitions.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SearchDataTypeValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ List<AbstractSearchCluster> clusters = model.getSearchClusters();
+ for (AbstractSearchCluster cluster : clusters) {
+ if (cluster.isStreaming()) {
+ continue;
+ }
+ for (AbstractSearchCluster.SearchDefinitionSpec spec : cluster.getLocalSDS()) {
+ SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument();
+ if (docType == null) {
+ continue;
+ }
+ validateDocument(cluster, spec.getSearchDefinition(), docType);
+ }
+ }
+ }
+
+ private void validateDocument(AbstractSearchCluster cluster, SearchDefinition def, SDDocumentType doc) {
+ for (SDDocumentType child : doc.getTypes()) {
+ validateDocument(cluster, def, child);
+ }
+ for (Field field : doc.fieldSet()) {
+ DataType fieldType = field.getDataType();
+ disallowIndexingOfMaps(cluster, def, field);
+ if (!validateDataType(fieldType)) {
+ throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " +
+ "clusters (field '" + field.getName() + "' in definition '" +
+ def.getName() + "' for cluster '" + cluster.getClusterName() + "').");
+ }
+ }
+ }
+
+ private boolean validateDataType(DataType dataType) {
+ if (dataType instanceof ArrayDataType ||
+ dataType instanceof WeightedSetDataType)
+ {
+ return validateDataType(((CollectionDataType)dataType).getNestedType());
+ }
+ if (dataType instanceof StructDataType) {
+ return true; // Struct will work for summary TODO maybe check individual fields
+ }
+ if (dataType instanceof MapDataType) {
+ return true; // Maps will work for summary, see disallowIndexingOfMaps()
+ }
+ return dataType.equals(DataType.INT) ||
+ dataType.equals(DataType.FLOAT) ||
+ dataType.equals(DataType.STRING) ||
+ dataType.equals(DataType.RAW) ||
+ dataType.equals(DataType.LONG) ||
+ dataType.equals(DataType.DOUBLE) ||
+ dataType.equals(DataType.URI) ||
+ dataType.equals(DataType.BYTE) ||
+ dataType.equals(DataType.PREDICATE) ||
+ dataType.equals(DataType.TENSOR);
+ }
+
+ private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) {
+ DataType fieldType = field.getDataType();
+ if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) {
+ throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " +
+ "clusters (field '" + field.getName() + "' in definition '" +
+ def.getName() + "' for cluster '" + cluster.getClusterName() + "').");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
new file mode 100644
index 00000000000..9d6d10f2861
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import java.util.List;
+import java.util.logging.Level;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.SearchCluster;
+
+
+/**
+ * Validates streaming mode
+ */
+public class StreamingValidator extends Validator {
+
+ @Override
+ public void validate(VespaModel model, DeployState deployState) {
+ List<AbstractSearchCluster> searchClusters = model.getSearchClusters();
+ for (AbstractSearchCluster cluster : searchClusters) {
+ if (!cluster.isStreaming()) {
+ continue;
+ }
+ SearchCluster sc = (SearchCluster) cluster;
+ warnStreamingAttributes(sc, deployState.getDeployLogger());
+ warnStreamingGramMatching(sc, deployState.getDeployLogger());
+ }
+ }
+
+ private void warnStreamingGramMatching(SearchCluster sc, DeployLogger logger) {
+ if (sc.getSdConfig() != null) {
+ for (SDField sd : sc.getSdConfig().getSearch().allFieldsList()) {
+ if (sd.getMatching().getType().equals(Matching.Type.GRAM)) {
+ logger.log(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() +
+ "', SD field '" + sd.getName() + "': n-gram matching is not supported for streaming search.");
+ }
+ }
+ }
+ }
+
+ /**
+ * Warn if one or more attributes are defined in a streaming search cluster SD.
+ *
+ * @param sc a search cluster to be checked for attributes in streaming search
+ * @param logger a DeployLogger
+ */
+ private void warnStreamingAttributes(SearchCluster sc, DeployLogger logger) {
+ if (sc.getSdConfig() != null) {
+ for (SDField sd : sc.getSdConfig().getSearch().allFieldsList()) {
+ if (sd.doesAttributing()) {
+ warnStreamingAttribute(sc, sd, logger);
+ }
+ }
+ }
+ }
+
+ private void warnStreamingAttribute(SearchCluster sc, SDField sd, DeployLogger logger) {
+ // If the field is numeric, we can't print this, because we may have converted the field to
+ // attribute indexing ourselves (IntegerIndex2Attribute)
+ if (sd.getDataType() instanceof NumericDataType) return;
+ logger.log(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() +
+ "', SD field '" + sd.getName() + "': 'attribute' has same match semantics as 'index'.");
+ }
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
new file mode 100644
index 00000000000..0c74fddcc51
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.change.ChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator;
+import com.yahoo.vespa.model.application.validation.change.ConfigValueChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.ContainerRestartValidator;
+import com.yahoo.vespa.model.application.validation.change.ContentClusterRemovalValidator;
+import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Executor of validators. This defines the right order of validator execution.
+ * Validators that must be run after search cluster search definition deriving are
+ * defined in PostSdValidation.
+ *
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ * @since 2010-01-29
+ */
+public class Validation {
+
+ /** Validate everything */
+ public static List<ConfigChangeAction> validate(VespaModel model, boolean force, DeployState deployState) {
+ return validate(model, true, force, deployState);
+ }
+
+ /**
+ * Validate with optional checking of routing, which cannot always be valid in unit tests
+ *
+ * @return a list of required changes needed to make this configuration live
+ */
+ public static List<ConfigChangeAction> validate(VespaModel model, boolean checkRouting, boolean force, DeployState deployState) {
+ if (checkRouting) {
+ new RoutingValidator().validate(model, deployState);
+ new RoutingSelectorValidator().validate(model, deployState);
+ }
+ new ComponentValidator().validate(model, deployState);
+ new SearchDataTypeValidator().validate(model, deployState);
+ new StreamingValidator().validate(model, deployState);
+ new RankSetupValidator(force).validate(model, deployState);
+ new NoPrefixForIndexes().validate(model, deployState);
+ new DeploymentFileValidator().validate(model, deployState);
+
+ Optional<Model> currentActiveModel = deployState.getPreviousModel();
+ if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel))
+ return validateChanges((VespaModel)currentActiveModel.get(), model,
+ deployState.validationOverrides(), deployState.getDeployLogger());
+ else
+ return new ArrayList<>();
+ }
+
+ private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, VespaModel nextModel,
+ ValidationOverrides overrides, DeployLogger logger) {
+ ChangeValidator[] validators = new ChangeValidator[] {
+ new IndexingModeChangeValidator(),
+ new IndexedSearchClusterChangeValidator(),
+ new ConfigValueChangeValidator(logger),
+ new StartupCommandChangeValidator(),
+ new ContentClusterRemovalValidator(),
+ new ClusterSizeReductionValidator(),
+ new ContainerRestartValidator(),
+ };
+ return Arrays.stream(validators)
+ .flatMap(v -> v.validate(currentModel, nextModel, overrides).stream())
+ .collect(toList());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java
new file mode 100644
index 00000000000..cb60e6fc77f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import java.util.Optional;
+
+/**
+ * Ids of validations that can be overridden
+ *
+ * @author bratseth
+ */
+public enum ValidationId {
+
+ indexingChange("indexing-change"),
+ indexModeChange("indexing-mode-change"),
+ fieldTypeChange("field-type-change"),
+ clusterSizeReduction("cluster-size-reduction"),
+ contentClusterRemoval("content-cluster-removal"),
+ configModelVersionMismatch("config-model-version-mismatch"),
+ skipOldConfigModels("skip-old-config-models");
+
+ private final String id;
+
+ ValidationId(String id) { this.id = id; }
+
+ public String value() { return id; }
+
+ @Override
+ public String toString() { return id; }
+
+ /**
+ * Returns the validation id from this string.
+ * Use this instead of valueOf to match string on the (canonical) dash-separated form.
+ *
+ * @return the matching validation id or empty if none
+ */
+ public static Optional<ValidationId> from(String id) {
+ for (ValidationId candidate : ValidationId.values())
+ if (id.equals(candidate.toString())) return Optional.of(candidate);
+ return Optional.empty();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java
new file mode 100644
index 00000000000..6a12d206189
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.google.common.collect.ImmutableList;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A set of allows which suppresses specific validations in limited time periods.
+ * This is useful to be able to complete a deployment in cases where the application
+ * owner believes that the changes to be deployed have acceptable consequences.
+ * Immutable.
+ *
+ * @author bratseth
+ */
+public class ValidationOverrides {
+
+ private final List<Allow> overrides;
+
+ /** Instant to use as "now". This is a field to allow unit testing. */
+ private final Instant now;
+
+ /** Creates validation overrides for the current instant */
+ public ValidationOverrides(List<Allow> overrides) {
+ this(overrides, Instant.now());
+ }
+
+ public ValidationOverrides(List<Allow> overrides, Instant now) {
+ this.overrides = ImmutableList.copyOf(overrides);
+ this.now = now;
+ for (Allow override : overrides)
+ if (now.plus(Duration.ofDays(30)).isBefore(override.until))
+ throw new IllegalArgumentException(override + " is too far in the future: Max 30 days is allowed");
+ }
+
+ /** Throws a ValidationException unless this validation is overridden at this time */
+ public void invalid(ValidationId validationId, String message) {
+ if ( ! allows(validationId))
+ throw new ValidationException(validationId, message);
+ }
+
+ public boolean allows(String validationIdString) {
+ Optional<ValidationId> validationId = ValidationId.from(validationIdString);
+ if ( ! validationId.isPresent()) return false; // unknown id -> not allowed
+ return allows(validationId.get());
+ }
+
+ /** Returns whether the given (assumed invalid) change is allowed by this at the moment */
+ public boolean allows(ValidationId validationId) {
+ for (Allow override : overrides)
+ if (override.allows(validationId, now))
+ return true;
+ return false;
+ }
+
+ public static ValidationOverrides empty() { return new ValidationOverrides(ImmutableList.of()); }
+
+ /** A validation override which allows a particular change. Immutable. */
+ public static class Allow {
+
+ private final ValidationId validationId;
+ private final Instant until;
+
+ public Allow(ValidationId validationId, Instant until) {
+ this.validationId = validationId;
+ this.until = until;
+ }
+
+ public boolean allows(ValidationId validationId, Instant now) {
+ return this.validationId.equals(validationId) && now.isBefore(until);
+ }
+
+ @Override
+ public String toString() { return "allow '" + validationId + "' until " + until; }
+
+ }
+
+ /**
+ * A deployment validation exception.
+ * Deployment validations can be {@link ValidationOverrides overridden} based on their id.
+ * The purpose of this exception is to model that id as a separate field.
+ */
+ public static class ValidationException extends IllegalArgumentException {
+
+ private final ValidationId validationId;
+
+ private ValidationException(ValidationId validationId, String message) {
+ super(message);
+ this.validationId = validationId;
+ }
+
+ /** Returns the unique id of this validation, which can be used to {@link ValidationOverrides override} it */
+ public ValidationId validationId() { return validationId; }
+
+ /** Returns "validationId: message" */
+ @Override
+ public String getMessage() { return validationId + ": " + super.getMessage(); }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java
new file mode 100644
index 00000000000..d92e15fea66
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+
+/**
+ * Abstract superclass of all application package validators.
+ *
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ * @since 2010-01-29
+ */
+public abstract class Validator {
+
+ /**
+ * Validates the input vespamodel
+ *
+ * @param model a VespaModel object
+ * @param deployState The {@link DeployState} built from building the model
+ */
+ public abstract void validate(VespaModel model, DeployState deployState);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java
new file mode 100644
index 00000000000..f60b1871e59
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+
+import java.util.List;
+
+/**
+ * Interface for validating changes between a current active and next config model.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-11-18
+ */
+public interface ChangeValidator {
+
+ /**
+ * Validates the current active vespa model with the next model.
+ * Both current and next should be non-null.
+ *
+ * @param current the current active model
+ * @param next the next model we would like to activate
+ * @param overrides validation overrides
+ * @return a list of actions specifying what needs to be done in order to activate the new model.
+ * Return an empty list if nothing needs to be done
+ */
+ List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
new file mode 100644
index 00000000000..279f74db46d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationId;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Checks that no cluster sizes are reduced too much in one go.
+ *
+ * @author bratseth
+ */
+public class ClusterSizeReductionValidator implements ChangeValidator {
+
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) {
+ for (ContainerCluster currentCluster : current.getContainerClusters().values()) {
+ ContainerCluster nextCluster = next.getContainerClusters().get(currentCluster.getName());
+ if (nextCluster == null) continue;
+ validate(currentCluster.getContainers().size(),
+ nextCluster.getContainers().size(),
+ currentCluster.getName(),
+ overrides);
+ }
+
+ for (ContentCluster currentCluster : current.getContentClusters().values()) {
+ ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getName());
+ if (nextCluster == null) continue;
+ validate(currentCluster.getSearch().getSearchNodes().size(),
+ nextCluster.getSearch().getSearchNodes().size(),
+ currentCluster.getName(),
+ overrides);
+ }
+
+ return Collections.emptyList();
+ }
+
+ private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides) {
+ // don't allow more than 50% reduction, but always allow to reduce size with 1
+ if ( nextSize < ((double)currentSize) * 0.5 && nextSize != currentSize - 1)
+ overrides.invalid(ValidationId.clusterSizeReduction,
+ "Size reduction in '" + clusterName + "' is too large. Current size: " + currentSize +
+ ", new size: " + nextSize + ". New size must be at least 50% of the current size");
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java
new file mode 100644
index 00000000000..06d7c358678
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java
@@ -0,0 +1,162 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.ChangesRequiringRestart;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.ChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.VespaRestartAction;
+import com.yahoo.vespa.model.utils.internal.ReflectionUtil;
+import org.apache.commons.lang3.ClassUtils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.joining;
+
+/**
+ * Compares the config instances in the current and next Vespa model to determine if some services will require restart.
+ * The configs used by a given service is deduced from the
+ * {@link com.yahoo.vespa.model.application.validation.RestartConfigs} annotation.
+ *
+ * @author bjorncs
+ */
+public class ConfigValueChangeValidator implements ChangeValidator {
+
+ private final DeployLogger logger;
+
+ public ConfigValueChangeValidator(DeployLogger logger) {
+ this.logger = logger;
+ }
+
+ /** Inspects the configuration in the new and old Vespa model to determine which services that require restart */
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel,
+ ValidationOverrides overrides) {
+ return findConfigChangesFromModels(currentModel, nextModel).collect(Collectors.toList());
+ }
+
+ public Stream<ConfigChangeAction> findConfigChangesFromModels(
+ AbstractConfigProducerRoot currentModel,
+ AbstractConfigProducerRoot nextModel) {
+ return nextModel.getDescendantServices().stream()
+ .map(service -> findConfigChangeActionForService(service, currentModel, nextModel))
+ .filter(Optional::isPresent)
+ .map(Optional::get);
+ }
+
+ private Optional<ConfigChangeAction> findConfigChangeActionForService(
+ Service service,
+ AbstractConfigProducerRoot currentModel,
+ AbstractConfigProducerRoot nextModel) {
+ List<ChangesRequiringRestart> changes = findConfigChangesForService(service, currentModel, nextModel)
+ .collect(Collectors.toList());
+ if (changes.isEmpty()) {
+ return Optional.empty();
+ }
+ String description = createDescriptionOfConfigChanges(changes.stream());
+ return Optional.of(new VespaRestartAction(description, service.getServiceInfo()));
+ }
+
+ private Stream<ChangesRequiringRestart> findConfigChangesForService(
+ Service service,
+ AbstractConfigProducerRoot currentModel,
+ AbstractConfigProducerRoot nextModel) {
+ Class<? extends Service> serviceClass = service.getClass();
+ if (!currentModel.getService(service.getConfigId()).isPresent()) {
+ // Service does not exist in the current model.
+ return Stream.empty();
+ }
+ return getConfigInstancesFromServiceAnnotations(serviceClass)
+ .map(configClass -> compareConfigFromCurrentAndNextModel(service, configClass, currentModel, nextModel))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .filter(ChangesRequiringRestart::needsRestart);
+ }
+
+ private static String createDescriptionOfConfigChanges(Stream<ChangesRequiringRestart> changesStream) {
+ return changesStream
+ .map(changes -> changes.toString(""))
+ .collect(joining("\n"));
+ }
+
+ /**
+ * Returns the ConfigInstances classes from the annotation on the given Service class ,
+ * including the ConfigInstances annotated on any of its super classes.
+ * NOTE: Only the super classes that are subclass of Service are inspected.
+ */
+ private static Stream<Class<? extends ConfigInstance>> getConfigInstancesFromServiceAnnotations(Class<? extends Service> serviceClass) {
+ List<Class<?>> classHierarchy = ClassUtils.getAllSuperclasses(serviceClass);
+ classHierarchy.add(serviceClass);
+ return classHierarchy.stream()
+ .filter(Service.class::isAssignableFrom)
+ .filter(clazz -> clazz.isAnnotationPresent(RestartConfigs.class))
+ .map(clazz -> {
+ RestartConfigs annotation = clazz.getDeclaredAnnotation(RestartConfigs.class);
+ if (annotation.value().length == 0) {
+ throw new IllegalStateException(String.format(
+ "%s has a %s annotation with no ConfigInstances given as argument.",
+ clazz.getSimpleName(), RestartConfigs.class.getSimpleName()));
+ }
+ return annotation;
+ })
+ .map(RestartConfigs::value)
+ .flatMap(Arrays::stream)
+ .distinct();
+ }
+
+ private Optional<ChangesRequiringRestart> compareConfigFromCurrentAndNextModel(
+ Service service, Class<? extends ConfigInstance> configClass,
+ AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) {
+
+ if (!hasConfigFieldsFlaggedWithRestart(configClass, service.getClass())) {
+ logger.log(Level.FINE, String.format("%s is listed in the annotation for %s, " +
+ "but does not have any restart flags in its config definition.",
+ configClass.getSimpleName(), service.getClass().getSimpleName()));
+ return Optional.empty();
+ }
+
+ Optional<ConfigInstance> nextConfig = getConfigFromModel(nextModel, configClass, service.getConfigId());
+ if (!nextConfig.isPresent()) {
+ logger.log(Level.FINE, String.format(
+ "%s is listed as restart config for %s, but the config does not exist in the new model.",
+ configClass.getSimpleName(), service.getClass().getSimpleName()));
+ return Optional.empty();
+ }
+
+ Optional<ConfigInstance> currentConfig = getConfigFromModel(currentModel, configClass, service.getConfigId());
+ if (!currentConfig.isPresent()) {
+ return Optional.empty();
+ }
+ return Optional.of(ReflectionUtil.getChangesRequiringRestart(currentConfig.get(), nextConfig.get()));
+ }
+
+ private static boolean hasConfigFieldsFlaggedWithRestart(
+ Class<? extends ConfigInstance> configClass, Class<? extends Service> serviceClass) {
+ if (!ReflectionUtil.hasRestartMethods(configClass)) {
+ throw new IllegalStateException(String.format(
+ "%s is listed as restart config for %s but does not contain any restart inspection methods.",
+ configClass.getSimpleName(), serviceClass.getSimpleName()));
+ }
+ return ReflectionUtil.containsFieldsFlaggedWithRestart(configClass);
+ }
+
+ private static Optional<ConfigInstance> getConfigFromModel(
+ AbstractConfigProducerRoot configModel, Class<? extends ConfigInstance> configClass, String configKey) {
+ try {
+ return Optional.ofNullable(configModel.getConfig(configClass, configKey));
+ } catch (Exception e) {
+ return Optional.empty();
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java
new file mode 100644
index 00000000000..3d56fe200d8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.container.QrConfig;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.container.Container;
+
+import java.util.List;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Returns a restart action for each container that has turned on {@link QrConfig#restartOnDeploy}.
+ *
+ * @author bjorncs
+ */
+public class ContainerRestartValidator implements ChangeValidator {
+
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides ignored) {
+ return nextModel.getContainerClusters().values().stream()
+ .flatMap(cluster -> cluster.getContainers().stream())
+ .filter(container -> isExistingContainer(container, currentModel))
+ .filter(container -> shouldContainerRestartOnDeploy(container, nextModel))
+ .map(ContainerRestartValidator::createConfigChangeAction)
+ .collect(toList());
+ }
+
+ private static ConfigChangeAction createConfigChangeAction(Container container) {
+ return new VespaRestartAction(createMessage(container), container.getServiceInfo());
+ }
+
+ private static String createMessage(Container container) {
+ return String.format("Container '%s' is configured to always restart on deploy.", container.getConfigId());
+ }
+
+ private static boolean shouldContainerRestartOnDeploy(Container container, VespaModel nextModel) {
+ QrConfig config = nextModel.getConfig(QrConfig.class, container.getConfigId());
+ return config.restartOnDeploy();
+ }
+
+ private static boolean isExistingContainer(Container container, VespaModel currentModel) {
+ return currentModel.getService(container.getConfigId()).isPresent();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java
new file mode 100644
index 00000000000..57e9d124c77
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationId;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Checks that this does not remove a content cluster (or changes its id)
+ * as that means losing all data of that cluster.
+ *
+ * @author bratseth
+ */
+public class ContentClusterRemovalValidator implements ChangeValidator {
+
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) {
+ for (String currentClusterId : current.getContentClusters().keySet()) {
+ ContentCluster nextCluster = next.getContentClusters().get(currentClusterId);
+ if (nextCluster == null)
+ overrides.invalid(ValidationId.contentClusterRemoval,
+ "Content cluster '" + currentClusterId + "' is removed. " +
+ "This will cause loss of all data in this cluster");
+ }
+
+ return Collections.emptyList();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java
new file mode 100644
index 00000000000..ab6bc5a1cd7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.search.DocumentDatabaseChangeValidator;
+import com.yahoo.vespa.model.content.ContentSearchCluster;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.search.DocumentDatabase;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Validates the changes between all current and next indexed search clusters in a vespa model.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-11-18
+ */
+public class IndexedSearchClusterChangeValidator implements ChangeValidator {
+
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) {
+ List<ConfigChangeAction> result = new ArrayList<>();
+ for (Map.Entry<String, ContentCluster> currentEntry : current.getContentClusters().entrySet()) {
+ ContentCluster nextCluster = next.getContentClusters().get(currentEntry.getKey());
+ if (nextCluster != null && nextCluster.getSearch().hasIndexedCluster()) {
+ result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, overrides));
+ }
+ }
+ return result;
+ }
+
+ private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster,
+ ContentCluster nextCluster,
+ ValidationOverrides overrides) {
+ List<ConfigChangeAction> result = new ArrayList<>();
+ result.addAll(validateDocumentDatabases(currentCluster, nextCluster, overrides));
+ return result;
+ }
+
+ private static List<ConfigChangeAction> validateDocumentDatabases(ContentCluster currentCluster,
+ ContentCluster nextCluster,
+ ValidationOverrides overrides) {
+ List<ConfigChangeAction> result = new ArrayList<>();
+ for (DocumentDatabase currentDb : getDocumentDbs(currentCluster.getSearch())) {
+ String docTypeName = currentDb.getName();
+ Optional<DocumentDatabase> nextDb = nextCluster.getSearch().getIndexed().getDocumentDbs().stream().
+ filter(db -> db.getName().equals(docTypeName)).findFirst();
+ if (nextDb.isPresent()) {
+ result.addAll(validateDocumentDatabase(currentCluster, nextCluster, docTypeName,
+ currentDb, nextDb.get(), overrides));
+ }
+ }
+ return result;
+ }
+
+ private static List<ConfigChangeAction> validateDocumentDatabase(ContentCluster currentCluster,
+ ContentCluster nextCluster,
+ String docTypeName,
+ DocumentDatabase currentDb,
+ DocumentDatabase nextDb,
+ ValidationOverrides overrides) {
+ NewDocumentType currentDocType = currentCluster.getDocumentDefinitions().get(docTypeName);
+ NewDocumentType nextDocType = nextCluster.getDocumentDefinitions().get(docTypeName);
+ List<VespaConfigChangeAction> result =
+ new DocumentDatabaseChangeValidator(currentDb, currentDocType, nextDb, nextDocType).validate(overrides);
+
+ return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getIndexed()), docTypeName);
+ }
+
+ private static List<DocumentDatabase> getDocumentDbs(ContentSearchCluster cluster) {
+ if (cluster.getIndexed() != null) {
+ return cluster.getIndexed().getDocumentDbs();
+ }
+ return new ArrayList<>();
+ }
+
+ private static List<ServiceInfo> getSearchNodeServices(IndexedSearchCluster cluster) {
+ return cluster.getSearchNodes().stream().
+ map(node -> node.getServiceInfo()).
+ collect(Collectors.toList());
+ }
+
+ private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result,
+ List<ServiceInfo> services,
+ String docTypeName) {
+ return result.stream().
+ map(action -> action.modifyAction("Document type '" + docTypeName + "': " + action.getMessage(),
+ services, docTypeName)).
+ collect(Collectors.toList());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java
new file mode 100644
index 00000000000..18683ce2411
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.ChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Returns any change to the indexing mode of a cluster.
+ *
+ * @author musum
+ */
+public class IndexingModeChangeValidator implements ChangeValidator {
+
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides overrides) {
+ List<ConfigChangeAction> actions = new ArrayList<>();
+ for (Map.Entry<String, ContentCluster> currentEntry : currentModel.getContentClusters().entrySet()) {
+ ContentCluster nextCluster = nextModel.getContentClusters().get(currentEntry.getKey());
+ if (nextCluster == null) continue;
+
+ Optional<ConfigChangeAction> change = validateContentCluster(currentEntry.getValue(), nextCluster, overrides);
+ if (change.isPresent())
+ actions.add(change.get());
+ }
+ return actions;
+ }
+
+ private Optional<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster,
+ ValidationOverrides overrides) {
+ final boolean currentClusterIsIndexed = currentCluster.getSearch().hasIndexedCluster();
+ final boolean nextClusterIsIndexed = nextCluster.getSearch().hasIndexedCluster();
+
+ if (currentClusterIsIndexed == nextClusterIsIndexed) return Optional.empty();
+
+ return Optional.of(VespaRefeedAction.of("indexing-mode-change",
+ overrides,
+ "Cluster '" + currentCluster.getName() + "' changed indexing mode from '" +
+ indexingMode(currentClusterIsIndexed) + "' to '" + indexingMode(nextClusterIsIndexed) + "'"));
+ }
+
+ private String indexingMode(boolean isIndexed) {
+ return isIndexed ? "indexed" : "streaming";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java
new file mode 100644
index 00000000000..fb90de60a2f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.ChangeValidator;
+import com.yahoo.vespa.model.application.validation.change.VespaRestartAction;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Compares the startup command for the services in the next model with the ones in the old model.
+ * If the startup command has changes, a change entry is created and reported back.
+ *
+ * @author bjorncs
+ */
+public class StartupCommandChangeValidator implements ChangeValidator {
+
+ @Override
+ public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel,
+ ValidationOverrides overrides) {
+ return findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList());
+ }
+
+ public Stream<ConfigChangeAction> findServicesWithChangedStartupCommmand(
+ AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) {
+ return nextModel.getDescendantServices().stream()
+ .map(nextService -> currentModel.getService(nextService.getConfigId())
+ .flatMap(currentService -> compareStartupCommand(currentService, nextService)))
+ .filter(Optional::isPresent)
+ .map(Optional::get);
+ }
+
+ private Optional<ConfigChangeAction> compareStartupCommand(Service currentService, Service nextService) {
+ String currentCommand = currentService.getStartupCommand();
+ String nextCommand = nextService.getStartupCommand();
+
+ // Objects.equals is null-aware
+ if (Objects.equals(currentCommand, nextCommand)) {
+ return Optional.empty();
+ }
+ String message = String.format("Startup command for '%s' has changed.\nNew command: %s.\nOld command: %s.",
+ currentService.getServiceName(), nextCommand, currentCommand);
+ return Optional.of(new VespaRestartAction(message, currentService.getServiceInfo()));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java
new file mode 100644
index 00000000000..a8a334448ff
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.model.api.ServiceInfo;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Class containing the action to be performed on the given services to handle a config change
+ * between the current active vespa model and the next vespa model to prepare.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-11-18
+ */
+public abstract class VespaConfigChangeAction implements ConfigChangeAction {
+
+ private final String message;
+ private final List<ServiceInfo> services;
+
+ protected VespaConfigChangeAction(String message, List<ServiceInfo> services) {
+ this.message = message;
+ this.services = services;
+ }
+
+ public abstract VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType);
+
+ @Override
+ public String getMessage() {
+ return message;
+ }
+
+ @Override
+ public List<ServiceInfo> getServices() {
+ return services;
+ }
+
+ @Override
+ public String toString() {
+ return "type='" + getType() + "', message='" + message + "', services=[" +
+ services.stream().
+ map(service -> service.getServiceName() + " '" + service.getConfigId() + "'").
+ collect(Collectors.joining(", ")) + "]";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof VespaConfigChangeAction)) {
+ return false;
+ }
+ VespaConfigChangeAction rhs = (VespaConfigChangeAction)o;
+ if (!getType().equals(rhs.getType())) return false;
+ if (!message.equals(rhs.message)) return false;
+ if (!services.equals(rhs.services)) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getType().hashCode();
+ result = 31 * result + message.hashCode();
+ result = 31 * result + services.hashCode();
+ return result;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java
new file mode 100644
index 00000000000..6b8d18fadea
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeRefeedAction;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents an action to re-feed a document type in order to handle a config change.
+ *
+ * @author geirst
+ * @author bratseth
+ * @since 5.43
+ */
+public class VespaRefeedAction extends VespaConfigChangeAction implements ConfigChangeRefeedAction {
+
+ /**
+ * The name of this action, which must be a valid ValidationId. This is a string here because
+ * the validation ids belong to the Vespa model while these names are exposed to the config server,
+ * which is model version independent.
+ */
+ private final String name;
+
+ private final String documentType;
+ private final boolean allowed;
+
+ private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) {
+ super(message, services);
+ this.name = name;
+ this.documentType = documentType;
+ this.allowed = allowed;
+ }
+
+ /** Creates a refeed action with some missing information */
+ // TODO: We should require document type or model its absence properly
+ public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message) {
+ return new VespaRefeedAction(name, message, Collections.emptyList(), "", overrides.allows(name));
+ }
+
+ /** Creates a refeed action */
+ public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message,
+ List<ServiceInfo> services, String documentType) {
+ return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name));
+ }
+
+ @Override
+ public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) {
+ return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed);
+ }
+
+ @Override
+ public String name() { return name; }
+
+ @Override
+ public String getDocumentType() { return documentType; }
+
+ @Override
+ public boolean allowed() { return allowed; }
+
+ @Override
+ public String toString() {
+ return super.toString() + ", documentType='" + documentType + "'";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if ( ! super.equals(o)) return false;
+ if ( ! (o instanceof VespaRefeedAction)) return false;
+ VespaRefeedAction other = (VespaRefeedAction)o;
+ if ( ! this.documentType.equals(other.documentType)) return false;
+ if ( ! this.name.equals(other.name)) return false;
+ if ( ! this.allowed == other.allowed) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * super.hashCode() + 11 * name.hashCode() + documentType.hashCode();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java
new file mode 100644
index 00000000000..1974847a83c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change;
+
+import com.yahoo.config.model.api.ConfigChangeRestartAction;
+import com.yahoo.config.model.api.ServiceInfo;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents an action to restart services in order to handle a config change.
+ *
+ * @author geirst
+ * @since 5.43
+ */
+public class VespaRestartAction extends VespaConfigChangeAction implements ConfigChangeRestartAction {
+
+ public VespaRestartAction(String message) {
+ super(message, new ArrayList<>());
+ }
+
+ public VespaRestartAction(String message, ServiceInfo service) {
+ super(message, Collections.singletonList(service));
+ }
+
+ public VespaRestartAction(String message, List<ServiceInfo> services) {
+ super(message, services);
+ }
+
+ @Override
+ public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) {
+ return new VespaRestartAction(newMessage, newServices);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java
new file mode 100644
index 00000000000..9e057c6cd28
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change.search;
+
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.searchdefinition.derived.AttributeFields;
+import com.yahoo.searchdefinition.derived.IndexSchema;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction;
+import com.yahoo.vespa.model.application.validation.change.VespaRestartAction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * Validates the changes between the current and next set of attribute fields in a document database.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-12-04
+ */
+public class AttributeChangeValidator {
+
+ private final AttributeFields currentFields;
+ private final IndexSchema currentIndexSchema;
+ private final NewDocumentType currentDocType;
+ private final AttributeFields nextFields;
+ private final IndexSchema nextIndexSchema;
+ private final NewDocumentType nextDocType;
+
+ public AttributeChangeValidator(AttributeFields currentFields,
+ IndexSchema currentIndexSchema,
+ NewDocumentType currentDocType,
+ AttributeFields nextFields,
+ IndexSchema nextIndexSchema,
+ NewDocumentType nextDocType) {
+ this.currentFields = currentFields;
+ this.currentIndexSchema = currentIndexSchema;
+ this.currentDocType = currentDocType;
+ this.nextFields = nextFields;
+ this.nextIndexSchema = nextIndexSchema;
+ this.nextDocType = nextDocType;
+ }
+
+ public List<VespaConfigChangeAction> validate() {
+ List<VespaConfigChangeAction> result = new ArrayList<>();
+ result.addAll(validateAddAttributeAspect());
+ result.addAll(validateRemoveAttributeAspect());
+ result.addAll(validateAttributeSettings());
+ return result;
+ }
+
+ private List<VespaConfigChangeAction> validateAddAttributeAspect() {
+ return nextFields.attributes().stream().
+ map(attr -> attr.getName()).
+ filter(attrName -> !currentFields.containsAttribute(attrName) &&
+ currentDocType.containsField(attrName)).
+ map(attrName -> new VespaRestartAction(new ChangeMessageBuilder(attrName).
+ addChange("add attribute aspect").build())).
+ collect(Collectors.toList());
+ }
+
+ private List<VespaConfigChangeAction> validateRemoveAttributeAspect() {
+ return currentFields.attributes().stream().
+ map(attr -> attr.getName()).
+ filter(attrName -> !nextFields.containsAttribute(attrName) &&
+ nextDocType.containsField(attrName) &&
+ !isIndexField(attrName)).
+ map(attrName -> new VespaRestartAction(new ChangeMessageBuilder(attrName).
+ addChange("remove attribute aspect").build())).
+ collect(Collectors.toList());
+ }
+
+ private boolean isIndexField(String fieldName) {
+ return currentIndexSchema.containsField(fieldName) && nextIndexSchema.containsField(fieldName);
+ }
+
+ private List<VespaConfigChangeAction> validateAttributeSettings() {
+ List<VespaConfigChangeAction> result = new ArrayList<>();
+ for (Attribute nextAttr : nextFields.attributes()) {
+ Attribute currAttr = currentFields.getAttribute(nextAttr.getName());
+ if (currAttr != null) {
+ validateAttributeSetting(currAttr, nextAttr, Attribute::isFastSearch, "fast-search", result);
+ validateAttributeSetting(currAttr, nextAttr, Attribute::isFastAccess, "fast-access", result);
+ validateAttributeSetting(currAttr, nextAttr, Attribute::isHuge, "huge", result);
+ validateAttributeSetting(currAttr, nextAttr, Attribute::densePostingListThreshold, "dense-posting-list-threshold", result);
+ }
+ }
+ return result;
+ }
+
+ private static void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr,
+ Predicate<Attribute> predicate, String setting,
+ List<VespaConfigChangeAction> result) {
+ final boolean nextValue = predicate.test(nextAttr);
+ if (predicate.test(currentAttr) != nextValue) {
+ String change = nextValue ? "add" : "remove";
+ result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()).
+ addChange(change + " attribute '" + setting + "'").build()));
+ }
+ }
+
+ private static <T> void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr,
+ Function<Attribute, T> settingValueProvider, String setting,
+ List<VespaConfigChangeAction> result) {
+ T currentValue = settingValueProvider.apply(currentAttr);
+ T nextValue = settingValueProvider.apply(nextAttr);
+ if (!Objects.equals(currentValue, nextValue)) {
+ String message = String.format("change property '%s' from '%s' to '%s'", setting, currentValue, nextValue);
+ result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()).addChange(message).build()));
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java
new file mode 100644
index 00000000000..28b04d54307
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change.search;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Class used to build a message describing the changes in a given field.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-12-09
+ */
+public class ChangeMessageBuilder {
+
+ private final String fieldName;
+ private final List<String> changes = new ArrayList<>();
+
+ public ChangeMessageBuilder(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public String build() {
+ StringBuilder retval = new StringBuilder();
+ retval.append("Field '" + fieldName + "' changed: ");
+ retval.append(String.join(", ", changes));
+ return retval.toString();
+ }
+
+ public ChangeMessageBuilder addChange(String component, String from, String to) {
+ changes.add(component + ": '" + from + "' -> '" + to + "'");
+ return this;
+ }
+
+ public ChangeMessageBuilder addChange(String message) {
+ changes.add(message);
+ return this;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java
new file mode 100644
index 00000000000..862b6caf0ca
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change.search;
+
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction;
+import com.yahoo.vespa.model.search.DocumentDatabase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Validates the changes between a current and next document database that is part of an indexed search cluster.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-11-18
+ */
+public class DocumentDatabaseChangeValidator {
+
+ private DocumentDatabase currentDatabase;
+ private NewDocumentType currentDocType;
+ private DocumentDatabase nextDatabase;
+ private NewDocumentType nextDocType;
+
+ public DocumentDatabaseChangeValidator(DocumentDatabase currentDatabase,
+ NewDocumentType currentDocType,
+ DocumentDatabase nextDatabase,
+ NewDocumentType nextDocType) {
+ this.currentDatabase = currentDatabase;
+ this.currentDocType = currentDocType;
+ this.nextDatabase = nextDatabase;
+ this.nextDocType = nextDocType;
+ }
+
+ public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) {
+ List<VespaConfigChangeAction> result = new ArrayList<>();
+ result.addAll(validateAttributeChanges());
+ result.addAll(validateIndexingScriptChanges(overrides));
+ result.addAll(validateDocumentTypeChanges(overrides));
+ return result;
+ }
+
+ private List<VespaConfigChangeAction> validateAttributeChanges() {
+ return new AttributeChangeValidator(
+ currentDatabase.getDerivedConfiguration().getAttributeFields(),
+ currentDatabase.getDerivedConfiguration().getIndexSchema(), currentDocType,
+ nextDatabase.getDerivedConfiguration().getAttributeFields(),
+ nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType).validate();
+ }
+
+ private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides) {
+ return new IndexingScriptChangeValidator(currentDatabase.getDerivedConfiguration().getSearch(),
+ nextDatabase.getDerivedConfiguration().getSearch()).validate(overrides);
+ }
+
+ private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides) {
+ return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java
new file mode 100644
index 00000000000..4b0e350718c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change.search;
+
+import com.yahoo.document.StructDataType;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction;
+import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Validates the changes between a current and next document type used in a document database.
+ *
+ * @author <a href="mailto:Tor.Egge@yahoo-inc.com">Tor Egge</a>
+ * @since 2014-11-25
+ */
+public class DocumentTypeChangeValidator {
+
+ private NewDocumentType currentDocType;
+ private NewDocumentType nextDocType;
+
+ private static abstract class FieldChange {
+
+ protected final Field currentField;
+ protected final Field nextField;
+
+ public FieldChange(Field currentField, Field nextField) {
+ this.currentField = currentField;
+ this.nextField = nextField;
+ }
+
+ public String fieldName() {
+ return currentField.getName();
+ }
+
+ public boolean valid() {
+ return nextField != null;
+ }
+
+ public abstract boolean changedType();
+ public abstract String currentTypeName();
+ public abstract String nextTypeName();
+ }
+
+ private static class SimpleFieldChange extends FieldChange {
+
+ public SimpleFieldChange(Field currentField, Field nextField) {
+ super(currentField, nextField);
+ }
+
+ public boolean changedType() {
+ return !currentField.getDataType().equals(nextField.getDataType());
+ }
+
+ public String currentTypeName() {
+ return currentField.getDataType().getName();
+ }
+
+ public String nextTypeName() {
+ return nextField.getDataType().getName();
+ }
+ }
+
+ private static class StructFieldChange extends FieldChange {
+
+ private final StructDataType currentType;
+ private final StructDataType nextType;
+
+ public StructFieldChange(Field currentField, Field nextField) {
+ super(currentField, nextField);
+ this.currentType = (StructDataType)currentField.getDataType();
+ this.nextType = (StructDataType)nextField.getDataType();
+ }
+
+ public boolean changedType() {
+ return changedType(currentType, nextType);
+ }
+
+ public String currentTypeName() {
+ return toString(currentType);
+ }
+
+ public String nextTypeName() {
+ return toString(nextType);
+ }
+
+ private static boolean changedType(StructDataType currentType, StructDataType nextType) {
+ for (Field currentField : currentType.getFields()) {
+ Field nextField = nextType.getField(currentField.getName());
+ if (nextField != null) {
+ if (areStructFields(currentField, nextField)) {
+ if (changedType((StructDataType) currentField.getDataType(),
+ (StructDataType) nextField.getDataType())) {
+ return true;
+ }
+ } else {
+ if (!currentField.getDataType().equals(nextField.getDataType())) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private static String toString(StructDataType dataType) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(dataType.getName()).append(":{");
+ boolean first = true;
+ for (Field field : dataType.getFields()) {
+ if (!first) {
+ builder.append(",");
+ }
+ if (field.getDataType() instanceof StructDataType) {
+ builder.append(toString((StructDataType) field.getDataType()));
+ } else {
+ builder.append(field.getName() + ":" + field.getDataType().getName());
+ }
+ first = false;
+ }
+ builder.append("}");
+ return builder.toString();
+ }
+ }
+
+ public DocumentTypeChangeValidator(NewDocumentType currentDocType,
+ NewDocumentType nextDocType) {
+ this.currentDocType = currentDocType;
+ this.nextDocType = nextDocType;
+ }
+
+ public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) {
+ return currentDocType.getAllFields().stream().
+ map(field -> createFieldChange(field, nextDocType)).
+ filter(fieldChange -> fieldChange.valid() && fieldChange.changedType()).
+ map(fieldChange -> VespaRefeedAction.of("field-type-change",
+ overrides,
+ new ChangeMessageBuilder(fieldChange.fieldName()).
+ addChange("data type", fieldChange.currentTypeName(),
+ fieldChange.nextTypeName()).build())).
+ collect(Collectors.toList());
+ }
+
+ private static FieldChange createFieldChange(Field currentField, NewDocumentType nextDocType) {
+ Field nextField = nextDocType.getField(currentField.getName());
+ if (nextField != null && areStructFields(currentField, nextField)) {
+ return new StructFieldChange(currentField, nextField);
+ }
+ return new SimpleFieldChange(currentField, nextField);
+ }
+
+ private static boolean areStructFields(Field currentField, Field nextField) {
+ return (currentField.getDataType() instanceof StructDataType) &&
+ (nextField.getDataType() instanceof StructDataType);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java
new file mode 100644
index 00000000000..5d8121b6ef8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change.search;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.NormalizeLevel;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+/**
+ * Class used to build a message describing the usual field changes causing changes in the indexing script.
+ * This message should be more descriptive for the end-user than just seeing the changed indexing script.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-12-09
+ */
+public class IndexingScriptChangeMessageBuilder {
+
+ private final Search currentSearch;
+ private final SDField currentField;
+ private final Search nextSearch;
+ private final SDField nextField;
+
+ public IndexingScriptChangeMessageBuilder(Search currentSearch, SDField currentField,
+ Search nextSearch, SDField nextField) {
+ this.currentSearch = currentSearch;
+ this.currentField = currentField;
+ this.nextSearch = nextSearch;
+ this.nextField = nextField;
+ }
+
+ public void populate(ChangeMessageBuilder builder) {
+ checkIndexing(builder);
+ checkMatching(builder);
+ checkStemming(builder);
+ checkNormalizing(builder);
+ checkSummaryTransform(builder);
+ }
+
+ private void checkIndexing(ChangeMessageBuilder builder) {
+ if (currentField.doesIndexing() != nextField.doesIndexing()) {
+ String change = nextField.doesIndexing() ? "add" : "remove";
+ builder.addChange(change + " index aspect");
+ }
+ }
+
+ private void checkMatching(ChangeMessageBuilder builder) {
+ Matching currentMatching = currentField.getMatching();
+ Matching nextMatching = nextField.getMatching();
+ if (!currentMatching.equals(nextMatching)) {
+ builder.addChange("matching", toString(currentMatching), toString(nextMatching));
+ }
+ }
+
+ private void checkStemming(ChangeMessageBuilder builder) {
+ Stemming currentStemming = currentField.getStemming(currentSearch);
+ Stemming nextStemming = nextField.getStemming(nextSearch);
+ if (!currentStemming.equals(nextStemming)) {
+ builder.addChange("stemming", currentStemming.getName(), nextStemming.getName());
+ }
+ }
+
+ private void checkNormalizing(ChangeMessageBuilder builder) {
+ NormalizeLevel.Level currentLevel = currentField.getNormalizing().getLevel();
+ NormalizeLevel.Level nextLevel = nextField.getNormalizing().getLevel();
+ if (!currentLevel.equals(nextLevel)) {
+ builder.addChange("normalizing", currentLevel.toString(), nextLevel.toString());
+ }
+ }
+
+ private void checkSummaryTransform(ChangeMessageBuilder builder) {
+ for (SummaryField nextSummaryField : nextField.getSummaryFields()) {
+ String fieldName = nextSummaryField.getName();
+ SummaryField currentSummaryField = currentField.getSummaryField(fieldName);
+ if (currentSummaryField != null) {
+ SummaryTransform currentTransform = currentSummaryField.getTransform();
+ SummaryTransform nextTransform = nextSummaryField.getTransform();
+ if (!currentSummaryField.getTransform().equals(nextSummaryField.getTransform())) {
+ builder.addChange("summary field '" + fieldName + "' transform",
+ currentTransform.getName(), nextTransform.getName());
+ }
+ }
+ }
+ }
+
+ private static String toString(Matching matching) {
+ Matching.Type type = matching.getType();
+ String retval = type.getName();
+ if (type.equals(Matching.Type.GRAM)) {
+ retval += " (size " + matching.getGramSize() + ")";
+ }
+ return retval;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java
new file mode 100644
index 00000000000..f1043f14fdc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.change.search;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction;
+import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Validates the indexing script changes in all fields in the current and next search model.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @since 2014-12-08
+ */
+public class IndexingScriptChangeValidator {
+
+ private final Search currentSearch;
+ private final Search nextSearch;
+
+ public IndexingScriptChangeValidator(Search currentSearch, Search nextSearch) {
+ this.currentSearch = currentSearch;
+ this.nextSearch = nextSearch;
+ }
+
+ public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) {
+ List<VespaConfigChangeAction> result = new ArrayList<>();
+ for (SDField nextField : nextSearch.allFieldsList()) {
+ String fieldName = nextField.getName();
+ SDField currentField = currentSearch.getField(fieldName);
+ if (currentField != null) {
+ validateScripts(currentField, nextField, overrides).ifPresent(r -> result.add(r));
+ }
+ }
+ return result;
+ }
+
+ private Optional<VespaConfigChangeAction> validateScripts(SDField currentField, SDField nextField,
+ ValidationOverrides overrides) {
+ ScriptExpression currentScript = currentField.getIndexingScript();
+ ScriptExpression nextScript = nextField.getIndexingScript();
+ if (!equalScripts(currentScript, nextScript)) {
+ ChangeMessageBuilder messageBuilder = new ChangeMessageBuilder(nextField.getName());
+ new IndexingScriptChangeMessageBuilder(currentSearch, currentField, nextSearch, nextField).populate(messageBuilder);
+ messageBuilder.addChange("indexing script", currentScript.toString(), nextScript.toString());
+ return Optional.of(VespaRefeedAction.of("indexing-change", overrides, messageBuilder.build()));
+ }
+ return Optional.empty();
+ }
+
+ static boolean equalScripts(ScriptExpression currentScript,
+ ScriptExpression nextScript) {
+ // Output expressions are specifying in which context a field value is used (attribute, index, summary),
+ // and do not affect how the field value is generated in the indexing doc proc.
+ // The output expressions are therefore removed before doing the comparison.
+ // Validating the addition / removal of attribute and index aspects are handled in other validators.
+ return removeOutputExpressions(currentScript).equals(removeOutputExpressions(nextScript));
+ }
+
+ private static ScriptExpression removeOutputExpressions(ScriptExpression script) {
+ ScriptExpression retval = (ScriptExpression) new OutputExpressionRemover().convert(script);
+ return retval;
+ }
+
+ private static class OutputExpressionRemover extends ExpressionConverter {
+
+ @Override
+ protected boolean shouldConvert(Expression exp) {
+ return exp instanceof OutputExpression;
+ }
+
+ @Override
+ protected Expression doConvert(Expression exp) {
+ if (exp instanceof OutputExpression) {
+ return null;
+ }
+ return exp;
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java
new file mode 100644
index 00000000000..1e2df635ab6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java
new file mode 100644
index 00000000000..0bbc4e32010
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation.xml;
+
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.application.validation.ValidationId;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import org.w3c.dom.Element;
+
+import java.io.Reader;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Reader of the validation-allows.xml file in application packages.
+ *
+ * @author bratseth
+ */
+public class ValidationOverridesXMLReader {
+
+ /**
+ * Returns a ValidationOverrides instance with the content of the given Reader.
+ * An empty ValidationOverrides is returned if the argument is empty.
+ *
+ * @param reader the reader which optionally contains a validation-overrides XML structure
+ * @param now the instant to use as "now", settable for unit testing
+ * @return a ValidationOverrides from the argument
+ * @throws IllegalArgumentException if the validation-allows.xml file exists but is invalid
+ */
+ public ValidationOverrides read(Optional<Reader> reader, Instant now) {
+ if ( ! reader.isPresent()) return ValidationOverrides.empty();
+
+ try {
+ // Assume valid structure is ensured by schema validation
+ Element root = XML.getDocument(reader.get()).getDocumentElement();
+ List<ValidationOverrides.Allow> overrides = new ArrayList<>();
+ for (Element allow : XML.getChildren(root, "allow")) {
+ Instant until = LocalDate.parse(allow.getAttribute("until"), DateTimeFormatter.ISO_DATE)
+ .atStartOfDay().atZone(ZoneOffset.UTC).toInstant()
+ .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date
+ Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow));
+ if (validationId.isPresent()) // skip unknonw ids as they may be valid for other model versions
+ overrides.add(new ValidationOverrides.Allow(validationId.get(), until));
+ }
+ return new ValidationOverrides(overrides, now);
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("validation-overrides is invalid", e);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java
new file mode 100644
index 00000000000..612df56cfc9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder;
+
+import com.yahoo.config.model.deploy.ConfigDefinitionStore;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.producer.UserConfigRepo;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.config.*;
+import com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder;
+import org.w3c.dom.Element;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class UserConfigBuilder {
+
+ public static final Logger log = Logger.getLogger(UserConfigBuilder.class.getPackage().toString());
+
+ public static UserConfigRepo build(Element producerSpec, ConfigDefinitionStore configDefinitionStore, DeployLogger deployLogger) {
+ final Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap = new LinkedHashMap<>();
+ if (producerSpec == null) {
+ log.log(LogLevel.SPAM, "In getUserConfigs. producerSpec is null");
+ }
+ log.log(LogLevel.DEBUG, "getUserConfigs for " + producerSpec);
+ for (Element configE : XML.getChildren(producerSpec, "config")) {
+ buildElement(configE, builderMap, configDefinitionStore, deployLogger);
+ }
+ return new UserConfigRepo(builderMap);
+ }
+
+
+ private static void buildElement(Element element, Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap, ConfigDefinitionStore configDefinitionStore, DeployLogger logger) {
+ ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(element);
+ log.log(LogLevel.SPAM, "Looking at " + key);
+
+ ConfigDefinition def = getConfigDef(key, configDefinitionStore);
+ // TODO: Fail here unless deploying with :force true
+ if (def == null) {
+ logger.log(LogLevel.WARNING, "Unable to find config definition for config '" + key.getNamespace() + "." + key.getName() +
+ "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle.");
+ }
+ List<String> issuedWarnings = new ArrayList<>();
+ for (String warning : issuedWarnings) {
+ logger.log(LogLevel.WARNING, warning);
+ }
+ ConfigPayloadBuilder payloadBuilder = new DomConfigPayloadBuilder(def).build(element, issuedWarnings);
+ log.log(LogLevel.SPAM, "configvalue=" + ConfigPayload.fromBuilder(payloadBuilder).toString());
+ log.log(LogLevel.DEBUG, "Looking up key: " + key.toString());
+ ConfigPayloadBuilder old = builderMap.get(key);
+ if (old != null) {
+ logger.log(LogLevel.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered");
+ log.log(LogLevel.DEBUG, "old configvalue=" + old);
+ old.override(payloadBuilder);
+ } else {
+ builderMap.put(key, payloadBuilder);
+ }
+ }
+
+ /**
+ * Returns the config definition matching the given name, or null if not found.
+ */
+ private static ConfigDefinition getConfigDef(ConfigDefinitionKey configDefinitionKey, ConfigDefinitionStore configDefinitionStore) {
+ try {
+ return configDefinitionStore.getConfigDefinition(configDefinitionKey);
+ } catch (IllegalArgumentException e) {
+ log.log(LogLevel.DEBUG, "Unable to retrieve config definition: " + e.getMessage());
+ return null;
+ }
+ }
+
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java
new file mode 100644
index 00000000000..75e9caefbd5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder;
+
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.generic.service.ServiceCluster;
+
+import java.util.List;
+
+/**
+ * Base class for classes capable of building vespa model.
+ *
+ * @author vegardh
+ */
+public abstract class VespaModelBuilder {
+
+
+ public abstract ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent);
+ public abstract List<ServiceCluster> getClusters(ApplicationPackage pkg, AbstractConfigProducer parent);
+
+ /**
+ * Processing that requires access across plugins
+ * @param producerRoot The root producer.
+ * @param configModelRepo a {@link com.yahoo.config.model.ConfigModelRepo instance}
+ */
+ public abstract void postProc(AbstractConfigProducer producerRoot, ConfigModelRepo configModelRepo);
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java
new file mode 100644
index 00000000000..9fbdee30661
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.binaryprefix.BinaryPrefix;
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author tonytv
+ */
+public class BinaryScaledAmountParser {
+ //The pattern must match the one given in the schema
+ private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?");
+
+ public static BinaryScaledAmount parse(String valueString) {
+ Matcher matcher = pattern.matcher(valueString);
+
+ if (!matcher.matches()) {
+ throw new RuntimeException("Pattern and schema is out of sync.");
+ }
+
+ double amount = Double.valueOf(matcher.group(1));
+ String binaryPrefixString = matcher.group(3);
+
+ return new BinaryScaledAmount(amount, asBinaryPrefix(binaryPrefixString));
+ }
+
+ private static BinaryPrefix asBinaryPrefix(String binaryPrefixString) {
+ if (binaryPrefixString == null) {
+ return BinaryPrefix.unit;
+ } else {
+ return BinaryPrefix.fromSymbol(binaryPrefixString.toUpperCase().charAt(0));
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java
new file mode 100644
index 00000000000..38d919f0f23
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * @author tonytv
+ */
+public class BinaryUnit {
+ //The pattern must match the one given in the schema
+ private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?");
+
+ public static double valueOf(String valueString) {
+ Matcher matcher = pattern.matcher(valueString);
+
+ matcher.matches();
+ double value = Double.valueOf(matcher.group(1));
+ String unit = matcher.group(3);
+ if (unit != null) {
+ value *= unitToValue(toLowerCase(unit).charAt(0));
+ }
+ return value;
+ }
+
+ private static double unitToValue(char unit) {
+ final char units[] = {'k', 'm', 'g'};
+ for (int i=0; i<units.length; ++i) {
+ if (units[i] == unit) {
+ return Math.pow(2, 10*(i+1));
+ }
+ }
+
+ throw new RuntimeException("No such unit: '" + unit + "'");
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
new file mode 100644
index 00000000000..179f6bad0f3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -0,0 +1,101 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.Host;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.HostSystem;
+import com.yahoo.vespa.model.admin.*;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.config.application.api.FileRegistry;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * A base class for admin model builders, to support common functionality across versions.
+ *
+ * @author lulf
+ * @author vegardh
+ * @since 5.12
+ */
+public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProducerBuilder<Admin> {
+
+ private static final int DEFAULT_INTERVAL = 1; // in minutes
+ private static final String DEFAULT_CLUSTER_NAME = "vespa";
+
+ private final List<ConfigServerSpec> configServerSpecs;
+ private final FileRegistry fileRegistry;
+ protected final boolean multitenant;
+
+ public DomAdminBuilderBase(FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) {
+ this.fileRegistry = fileRegistry;
+ this.multitenant = multitenant;
+ this.configServerSpecs = configServerSpecs;
+ }
+
+ protected List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) {
+ List<Configserver> configservers = new ArrayList<>();
+ for (ConfigServerSpec spec : configServerSpecs) {
+ HostSystem hostSystem = parent.getHostSystem();
+ HostResource host = new HostResource(Host.createMultitenantHost(hostSystem, spec.getHostName()));
+ hostSystem.addBoundHost(host);
+ Configserver configserver = new Configserver(parent, spec.getHostName());
+ configserver.setHostResource(host);
+ configserver.setBasePort(configserver.getWantedPort());
+ configserver.initService();
+ configservers.add(configserver);
+ }
+ return configservers;
+ }
+
+ @Override
+ protected Admin doBuild(AbstractConfigProducer parent, Element adminE) {
+ Yamas yamas = getYamas(XML.getChild(adminE, "yamas"));
+ Map<String, MetricsConsumer> metricsConsumers = DomMetricBuilderHelper.buildMetricsConsumers(XML.getChild(adminE, "metric-consumers"));
+
+ Admin admin = new Admin(parent, yamas, metricsConsumers, multitenant);
+
+ doBuildAdmin(admin, adminE);
+
+ new ModelConfigProvider(admin);
+
+ FileDistributionOptions fileDistributionOptions = new DomFileDistributionOptionsBuilder().build(XML.getChild(adminE, "filedistribution"));
+ admin.setFileDistribution(new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry));
+ return admin;
+ }
+
+ protected abstract void doBuildAdmin(Admin admin, Element adminE);
+
+ private Yamas getYamas(Element yamasE) {
+ Yamas yamas;
+ if (yamasE == null) {
+ yamas = new Yamas(DEFAULT_CLUSTER_NAME, DEFAULT_INTERVAL);
+ } else {
+ Integer minutes = getMonitoringInterval(yamasE);
+ if (minutes == null) {
+ minutes = DEFAULT_INTERVAL;
+ }
+ yamas = new Yamas(yamasE.getAttribute("systemname"), minutes);
+ }
+ return yamas;
+ }
+
+ private Integer getMonitoringInterval(Element monitoringE) {
+ Integer minutes = null;
+ String seconds = monitoringE.getAttribute("interval").trim();
+ if (!seconds.isEmpty()) {
+ minutes = Integer.parseInt(seconds);
+ minutes = minutes / 60;
+ if (!(minutes == 1 || minutes == 5)) {
+ throw new IllegalArgumentException("The only allowed values for 'interval' attribute in '" + monitoringE.getTagName() +
+ "' element is 60 or 300.");
+ }
+ }
+ return minutes;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
new file mode 100644
index 00000000000..c45cbf8f0d2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
@@ -0,0 +1,197 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.ConfigModelUtils;
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.admin.*;
+import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster;
+import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.config.application.api.FileRegistry;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Builds the admin model from a V2 admin XML tag.
+ *
+ * @author vegardh
+ */
+public class DomAdminV2Builder extends DomAdminBuilderBase {
+
+ private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper";
+
+ public DomAdminV2Builder(FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) {
+ super(fileRegistry, multitenant, configServerSpecs);
+ }
+
+ @Override
+ protected void doBuildAdmin(Admin admin, Element adminE) {
+ List<Configserver> configservers = parseConfigservers(admin, adminE);
+ admin.setLogserver(parseLogserver(admin, adminE));
+ admin.addConfigservers(configservers);
+ admin.addSlobroks(getSlobroks(admin, XML.getChild(adminE, "slobroks")));
+ if ( ! admin.multitenant())
+ admin.setClusterControllers(addConfiguredClusterControllers(admin, adminE));
+ }
+
+ private List<Configserver> parseConfigservers(Admin admin, Element adminE) {
+ List<Configserver> configservers;
+ if (multitenant) {
+ configservers = getConfigServersFromSpec(admin);
+ } else {
+ configservers = getConfigServers(admin, adminE);
+ }
+ int count = configservers.size();
+ if (count % 2 == 0) {
+ admin.deployLogger().log(Level.WARNING, "An even number (" + count + ") of config servers have been configured. " +
+ "This is discouraged, see " + "" +
+ ConfigModelUtils.createDocLink("cloudconfig/configuration-server.html#fault-tolerance"));
+ }
+ return configservers;
+ }
+
+ private Logserver parseLogserver(Admin admin, Element adminE) {
+ Element logserverE = XML.getChild(adminE, "logserver");
+ if (logserverE == null) {
+ logserverE = XML.getChild(adminE, "adminserver");
+ }
+ return new LogserverBuilder().build(admin, logserverE);
+ }
+
+ private ContainerCluster addConfiguredClusterControllers(AbstractConfigProducer parent, Element admin) {
+ Element controllersElements = XML.getChild(admin, "cluster-controllers");
+ if (controllersElements == null) return null;
+
+ List<Element> controllers = XML.getChildren(controllersElements, "cluster-controller");
+ if (controllers.isEmpty()) return null;
+
+ boolean standaloneZooKeeper = "true".equals(controllersElements.getAttribute(ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK)) || multitenant;
+ if (standaloneZooKeeper) {
+ parent = new ClusterControllerCluster(parent, "standalone");
+ }
+ ContainerCluster cluster = new ContainerCluster(parent, "cluster-controllers", "cluster-controllers");
+ ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster);
+
+ List<Container> containers = new ArrayList<>();
+
+ for (Element controller : controllers) {
+ ClusterControllerContainer clusterController = new ClusterControllerBuilder(containers.size(), standaloneZooKeeper).build(cluster, controller);
+ containers.add(clusterController);
+ }
+
+ cluster.addContainers(containers);
+ return cluster;
+ }
+
+ // Extra stupid because configservers tag is voluntary
+ private List<Configserver> getConfigServers(AbstractConfigProducer parent, Element adminE) {
+ SimpleConfigProducer configServers = new SimpleConfigProducer(parent, "configservers");
+ List<Configserver> cfgs = new ArrayList<>();
+ Element configserversE = XML.getChild(adminE, "configservers");
+ if (configserversE == null) {
+ Element configserverE = XML.getChild(adminE, "configserver");
+ if (configserverE == null) {
+ configserverE = XML.getChild(adminE, "adminserver");
+ } else {
+ parent.deployLogger().log(LogLevel.INFO, "Specifying configserver without parent element configservers in services.xml is deprecated");
+ }
+ Configserver cfgs0 = new ConfigserverBuilder(0).build(configServers, configserverE);
+ cfgs0.setProp("index", 0);
+ cfgs.add(cfgs0);
+ return cfgs;
+ }
+ // configservers tag in use
+ int i = 0;
+ for (Element configserverE : XML.getChildren(configserversE, "configserver")) {
+ Configserver cfgsrv = new ConfigserverBuilder(i).build(configServers, configserverE);
+ cfgsrv.setProp("index", i);
+ cfgs.add(cfgsrv);
+ i++;
+ }
+ return cfgs;
+ }
+
+ private List<Slobrok> getSlobroks(AbstractConfigProducer parent, Element slobroksE) {
+ List<Slobrok> slobs = new ArrayList<>();
+ if (slobroksE != null) {
+ slobs = getExplicitSlobrokSetup(parent, slobroksE);
+ }
+ return slobs;
+ }
+
+ private List<Slobrok> getExplicitSlobrokSetup(AbstractConfigProducer parent, Element slobroksE) {
+ List<Slobrok> slobs = new ArrayList<>();
+ List<Element> slobsE = XML.getChildren(slobroksE, "slobrok");
+ int i = 0;
+ for (Element e : slobsE) {
+ Slobrok slob = new SlobrokBuilder(i).build(parent, e);
+ slobs.add(slob);
+ i++;
+ }
+ return slobs;
+ }
+
+ private static class LogserverBuilder extends DomConfigProducerBuilder<Logserver> {
+ public LogserverBuilder() {
+ }
+
+ @Override
+ protected Logserver doBuild(AbstractConfigProducer parent, Element producerSpec) {
+ return new Logserver(parent);
+ }
+ }
+
+ private static class ConfigserverBuilder extends DomConfigProducerBuilder<Configserver> {
+ int i;
+
+ public ConfigserverBuilder(int i) {
+ this.i = i;
+ }
+
+ @Override
+ protected Configserver doBuild(AbstractConfigProducer parent,
+ Element spec) {
+ return new Configserver(parent, "configserver." + i);
+ }
+ }
+
+ private static class SlobrokBuilder extends DomConfigProducerBuilder<Slobrok> {
+ int i;
+
+ public SlobrokBuilder(int i) {
+ this.i = i;
+ }
+
+ @Override
+ protected Slobrok doBuild(AbstractConfigProducer parent,
+ Element spec) {
+ return new Slobrok(parent, i);
+ }
+ }
+
+ private static class ClusterControllerBuilder extends DomConfigProducerBuilder<ClusterControllerContainer> {
+ int i;
+ boolean runStandaloneZooKeeper;
+
+ public ClusterControllerBuilder(int i, boolean runStandaloneZooKeeper) {
+ this.i = i;
+ this.runStandaloneZooKeeper = runStandaloneZooKeeper;
+ }
+
+ @Override
+ protected ClusterControllerContainer doBuild(AbstractConfigProducer parent,
+ Element spec) {
+ return new ClusterControllerContainer(parent, i, runStandaloneZooKeeper);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
new file mode 100644
index 00000000000..1144ea775ab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.HostSystem;
+import com.yahoo.vespa.model.admin.*;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerModel;
+
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used.
+ *
+ * @author bratseth
+ */
+public class DomAdminV4Builder extends DomAdminBuilderBase {
+
+ private final Collection<ContainerModel> containerModels;
+ private final DeployLogger deployLogger;
+
+ public DomAdminV4Builder(ConfigModelContext modelContext, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) {
+ super(modelContext.getDeployState().getFileRegistry(), multitenant, configServerSpecs);
+ this.containerModels = containerModels;
+ this.deployLogger = modelContext.getDeployLogger();
+ }
+
+ @Override
+ protected void doBuildAdmin(Admin admin, Element w3cAdminElement) {
+ ModelElement adminElement = new ModelElement(w3cAdminElement);
+ admin.addConfigservers(getConfigServersFromSpec(admin));
+
+ // Note: These two elements only exists in admin version 4.0
+ // This build handles admin version 3.0 by ignoring its content (as the content is not useful)
+ Optional<NodesSpecification> requestedSlobroks = NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("slobroks"));
+ Optional<NodesSpecification> requestedLogservers = NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("logservers"));
+ assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3)), admin);
+ assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1)), admin);
+ }
+
+ private void assignSlobroks(NodesSpecification nodesSpecification, Admin admin) {
+ if (nodesSpecification.isDedicated()) {
+ createSlobroks(admin, allocateHosts(admin.getHostSystem(), "slobroks", nodesSpecification));
+ }
+ else {
+ createSlobroks(admin, pickContainerHosts(nodesSpecification.count()));
+ }
+ }
+
+ private void assignLogserver(NodesSpecification nodesSpecification, Admin admin) {
+ if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server");
+
+ if (nodesSpecification.isDedicated()) {
+ createLogserver(admin, allocateHosts(admin.getHostSystem(), "logserver", nodesSpecification));
+ }
+ else {
+ if (containerModels.iterator().hasNext())
+ createLogserver(admin, sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false));
+ }
+ }
+
+ private Collection<HostResource> allocateHosts(HostSystem hostSystem, String clusterId, NodesSpecification nodesSpecification) {
+ return nodesSpecification.provision(hostSystem, ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterId), Optional.empty(), deployLogger).keySet();
+ }
+
+ /**
+ * Returns a list of container hosts to use for an auxiliary cluster.
+ * The list returns the same nodes on each invocation given the same available nodes.
+ *
+ * @param count the desired number of nodes. More nodes may be returned to ensure a smooth transition
+ * on topology changes, and less nodes may be returned if fewer are available
+ */
+ private List<HostResource> pickContainerHosts(int count) {
+ // Pick from all container clusters to make sure we don't lose all nodes at once if some clusters are removed.
+ // This will overshoot the desired size (due to ceil and picking at least one node per cluster).
+ List<HostResource> picked = new ArrayList<>();
+ for (ContainerModel containerModel : containerModels)
+ picked.addAll(pickContainerHostsFrom(containerModel,
+ (int)Math.max(1, Math.ceil((double)count/containerModels.size()))));
+ return picked;
+ }
+
+ private List<HostResource> pickContainerHostsFrom(ContainerModel model, int count) {
+ boolean retired = true;
+ List<HostResource> picked = sortedContainerHostsFrom(model, count, !retired);
+
+ // if we can return multiple hosts, include retired nodes which would have been picked before
+ // (probably - assuming all previous nodes were retired, which is always true for a single cluster
+ // at the moment (Sept 2015)) // to ensure a smoother transition between the old and new topology
+ // by including both new and old nodes during the retirement period
+ picked.addAll(sortedContainerHostsFrom(model, count, retired));
+
+ return picked;
+ }
+
+ /** Returns the count first containers in the current model having isRetired set to the given value */
+ private List<HostResource> sortedContainerHostsFrom(ContainerModel model, int count, boolean retired) {
+ List<HostResource> hosts = new ArrayList<>();
+ for (Container container : model.getCluster().getContainers())
+ if (retired == container.isRetired())
+ hosts.add(container.getHostResource());
+ Collections.sort(hosts);
+ return hosts.subList(0, Math.min(count, hosts.size()));
+ }
+
+ private void createLogserver(Admin admin, Collection<HostResource> hosts) {
+ if (hosts.isEmpty()) return; // No log server can be created (and none is needed)
+ Logserver logserver = new Logserver(admin);
+ logserver.setHostResource(hosts.iterator().next());
+ admin.setLogserver(logserver);
+ logserver.initService();
+ }
+
+ private void createSlobroks(Admin admin, Collection<HostResource> hosts) {
+ if (hosts.isEmpty()) return; // No slobroks can be created (and none are needed)
+ List<Slobrok> slobroks = new ArrayList<>();
+ int index = 0;
+ for (HostResource host : hosts) {
+ Slobrok slobrok = new Slobrok(admin, index++);
+ slobrok.setHostResource(host);
+ slobroks.add(slobrok);
+ slobrok.initService();
+ }
+ admin.addSlobroks(slobroks);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
new file mode 100644
index 00000000000..56415c7232d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.Handler;
+import org.w3c.dom.Element;
+
+/**
+ * @author gjoranv
+ * @since 5.1.6
+ */
+public class DomClientProviderBuilder extends DomHandlerBuilder {
+
+ @Override
+ protected Handler doBuild(AbstractConfigProducer ancestor, Element clientElement) {
+ Handler<? super Component<?, ?>> client = getHandler(clientElement);
+
+ for (Element binding : XML.getChildren(clientElement, "binding"))
+ client.addClientBindings(XML.getValue(binding));
+
+ for (Element serverBinding : XML.getChildren(clientElement, "serverBinding"))
+ client.addServerBindings(XML.getValue(serverBinding));
+
+ DomComponentBuilder.addChildren(ancestor, clientElement, client);
+
+ return client;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java
new file mode 100644
index 00000000000..0e0c94abae1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.vespa.model.clients.*;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Builds the Clients plugin
+ *
+ * @author musum
+ */
+public class DomClientsBuilder extends LegacyConfigModelBuilder<Clients> {
+
+ public DomClientsBuilder() {
+ super(Clients.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() {
+ return Arrays.asList(ConfigModelId.fromNameAndVersion("clients", "2.0"));
+ }
+
+ @Override
+ public void doBuild(Clients clients, Element clientsE, ConfigModelContext modelContext) {
+ String version = clientsE.getAttribute("version");
+ if (version.startsWith("2.")) {
+ DomV20ClientsBuilder parser = new DomV20ClientsBuilder(clients, version);
+ parser.build(clientsE);
+ } else {
+ throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported.");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java
new file mode 100644
index 00000000000..17a59bd7590
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
+import org.w3c.dom.Element;
+
+/**
+ * @author gjoranv
+ * @author tonytv
+ */
+public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> {
+
+ public static final String elementName = "component" ;
+
+ private final ComponentId namespace;
+
+ public DomComponentBuilder() {
+ this(null);
+ }
+
+ public DomComponentBuilder(ComponentId namespace) {
+ this.namespace = namespace;
+ }
+
+ protected Component doBuild(AbstractConfigProducer ancestor, Element spec) {
+ Component component = buildComponent(spec);
+ addChildren(ancestor, spec, component);
+ return component;
+ }
+
+ private Component buildComponent(Element spec) {
+ BundleInstantiationSpecification bundleSpec =
+ BundleInstantiationSpecificationBuilder.build(spec, false).nestInNamespace(namespace);
+
+ return new Component<Component<?, ?>, ComponentModel>(new ComponentModel(bundleSpec));
+ }
+
+ public static void addChildren(AbstractConfigProducer ancestor, Element componentNode, Component<? super Component<?, ?>, ?> component) {
+ for (Element childNode : XML.getChildren(componentNode, elementName)) {
+ addAndInjectChild(ancestor, component, childNode);
+ }
+ }
+
+ private static void addAndInjectChild(AbstractConfigProducer ancestor, Component<? super Component<?, ?>, ?> component, Element childNode) {
+ Component<?, ?> child = new DomComponentBuilder(component.getComponentId()).build(ancestor, childNode);
+ component.addComponent(child);
+ component.inject(child);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java
new file mode 100644
index 00000000000..a2478f80397
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java
@@ -0,0 +1,243 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.config.ConfigurationRuntimeException;
+import com.yahoo.config.codegen.CNode;
+import com.yahoo.log.LogLevel;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.config.*;
+
+import com.yahoo.vespa.config.util.ConfigUtils;
+import org.w3c.dom.Element;
+
+import java.util.*;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/**
+ * Builder that transforms xml config to a slime tree representation of the config. The root element of the xml config
+ * must be named 'config' and have a 'name' attribute that matches the name of the {@link ConfigDefinition}. The values
+ * are not validated against their types. That task is moved to the builders.
+ *
+ * @author lulf
+ */
+public class DomConfigPayloadBuilder {
+
+ private static final Logger log = Logger.getLogger(DomConfigPayloadBuilder.class.getPackage().toString());
+
+ private static final Pattern namePattern = ConfigDefinition.namePattern;
+ private static final Pattern namespacePattern = ConfigDefinition.namespacePattern;
+
+ private final ConfigDefinition configDefinition;
+
+ public DomConfigPayloadBuilder(ConfigDefinition configDefinition) {
+ this.configDefinition = configDefinition;
+ }
+
+ /**
+ * Builds a {@link ConfigPayloadBuilder} representing the input 'config' xml element.
+ *
+ * @param configE The 'config' xml element
+ * @param issuedWarnings a list that will be populated with issued warnings when building the config payload
+ * @return a new payload builder built from xml.
+ */
+ public ConfigPayloadBuilder build(Element configE, List<String> issuedWarnings) {
+ parseConfigName(configE);
+
+ ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(configDefinition, issuedWarnings);
+ for (Element child : XML.getChildren(configE)) {
+ parseElement(child, payloadBuilder, null);
+ }
+ return payloadBuilder;
+ }
+
+ public static ConfigDefinitionKey parseConfigName(Element configE) {
+ if (!configE.getNodeName().equals("config")) {
+ throw new ConfigurationRuntimeException("The root element must be 'config', but was '"
+ + configE.getNodeName() + "'.");
+ }
+ if (!configE.hasAttribute("name")) {
+ throw new ConfigurationRuntimeException
+ ("The 'config' element must have a 'name' attribute that matches the name of the config definition.");
+ }
+
+ String xmlName = configE.getAttribute("name");
+ final boolean xmlNamespaceAttributeExists = configE.hasAttribute("namespace");
+
+ String xmlNamespace = null;
+ // If name contains dots, rewrite to name and namespace
+ if (xmlName.contains(".")) {
+ Tuple2<String, String> t = ConfigUtils.getNameAndNamespaceFromString(xmlName);
+ xmlName = t.first;
+ xmlNamespace = t.second;
+ } else {
+ if (!xmlNamespaceAttributeExists) {
+ log.log(LogLevel.WARNING, "No namespace in 'config name=" + xmlName + "', please specify one");
+ }
+ }
+
+ if (!validName(xmlName)) {
+ throw new ConfigurationRuntimeException("The config name '" + xmlName +
+ "' contains illegal characters. Only names with the pattern " + namePattern.toString() + " are legal.");
+ }
+
+ if (xmlNamespace == null) {
+ xmlNamespace = configE.getAttribute("namespace");
+ if (xmlNamespace == null || xmlNamespace.isEmpty()) {
+ xmlNamespace = CNode.DEFAULT_NAMESPACE;
+ }
+ }
+ if (!validNamespace(xmlNamespace)) {
+ throw new ConfigurationRuntimeException("The config namespace '" + xmlNamespace +
+ "' contains illegal characters. Only namespaces with the pattern " + namespacePattern.toString() + " are legal.");
+ }
+ return new ConfigDefinitionKey(xmlName, xmlNamespace);
+ }
+
+ private static boolean validName(String name) {
+ Matcher m = namePattern.matcher(name);
+ return m.matches();
+ }
+
+ private static boolean validNamespace(String namespace) {
+ Matcher m = namespacePattern.matcher(namespace);
+ return m.matches();
+ }
+
+ private String extractName(Element element) {
+ String initial = element.getNodeName();
+ if (initial.indexOf('-') < 0) {
+ return initial;
+ }
+ StringBuilder buf = new StringBuilder();
+ boolean upcase = false;
+ for (char ch : initial.toCharArray()) {
+ if (ch == '-') {
+ upcase = true;
+ } else if (upcase && ch >= 'a' && ch <= 'z') {
+ buf.append((char)('A' + ch - 'a'));
+ upcase = false;
+ } else {
+ buf.append(ch);
+ upcase = false;
+ }
+ }
+ return buf.toString();
+ }
+
+ /**
+ * Parse leaf value in an xml tree
+ */
+ private void parseLeaf(Element element, ConfigPayloadBuilder payloadBuilder, String parentName) {
+ String name = extractName(element);
+ String value = XML.getValue(element);
+ if (value == null) {
+ throw new ConfigurationRuntimeException("Element '" + name + "' must have either children or a value");
+ }
+
+
+ if (element.hasAttribute("index")) {
+ // Check for legacy (pre Vespa 6) usage
+ throw new IllegalArgumentException("The 'index' attribute on config elements is not supported - use <item>");
+ } else if (element.hasAttribute("operation")) {
+ // leaf array, currently the only supported operation is 'append'
+ verifyLegalOperation(element);
+ ConfigPayloadBuilder.Array a = payloadBuilder.getArray(name);
+ a.append(value);
+ } else if ("item".equals(name)) {
+ if (parentName == null)
+ throw new ConfigurationRuntimeException("<item> is a reserved keyword for array and map elements");
+ if (element.hasAttribute("key")) {
+ payloadBuilder.getMap(parentName).put(element.getAttribute("key"), value);
+ } else {
+ payloadBuilder.getArray(parentName).append(value);
+ }
+ } else {
+ // leaf scalar, e.g. <intVal>3</intVal>
+ payloadBuilder.setField(name, value);
+ }
+ }
+
+ private void parseComplex(Element element, List<Element> children, ConfigPayloadBuilder payloadBuilder, String parentName) {
+ String name = extractName(element);
+ // Inner value
+ if (element.hasAttribute("index")) {
+ // Check for legacy (pre Vespa 6) usage
+ throw new IllegalArgumentException("The 'index' attribute on config elements is not supported - use <item>");
+ } else if (element.hasAttribute("operation")) {
+ // inner array, currently the only supported operation is 'append'
+ verifyLegalOperation(element);
+ ConfigPayloadBuilder childPayloadBuilder = payloadBuilder.getArray(name).append();
+ //Cursor array = node.setArray(name);
+ for (Element child : children) {
+ //Cursor struct = array.addObject();
+ parseElement(child, childPayloadBuilder, name);
+ }
+ } else if ("item".equals(name)) {
+ // Reserved item means array/map element as struct
+ if (element.hasAttribute("key")) {
+ ConfigPayloadBuilder childPayloadBuilder = payloadBuilder.getMap(parentName).get(element.getAttribute("key"));
+ for (Element child : children) {
+ parseElement(child, childPayloadBuilder, parentName);
+ }
+ } else {
+ ConfigPayloadBuilder.Array array = payloadBuilder.getArray(parentName);
+ ConfigPayloadBuilder childPayloadBuilder = array.append();
+ for (Element child : children) {
+ parseElement(child, childPayloadBuilder, parentName);
+ }
+ }
+ } else {
+ int numMatching = 0;
+ for (Element child : children) {
+ numMatching += ("item".equals(child.getTagName())) ? 1 : 0;
+ }
+
+ if (numMatching == 0) {
+ // struct, e.g. <basicStruct>
+ ConfigPayloadBuilder p = payloadBuilder.getObject(name);
+ //Cursor struct = node.setObject(name);
+ for (Element child : children)
+ parseElement(child, p, name);
+ } else if (numMatching == children.size()) {
+ // Array with <item elements>
+ for (Element child : children) {
+ parseElement(child, payloadBuilder, name);
+ }
+ } else {
+ throw new ConfigurationRuntimeException("<item> is a reserved keyword for array and map elements");
+ }
+ }
+ }
+
+ /**
+ * Adds the values and children (recursively) in the given xml element to the given {@link ConfigPayloadBuilder}.
+ * @param currElem The element representing a config parameter.
+ * @param payloadBuilder The builder to use when adding elements.
+ */
+ private void parseElement(Element currElem, ConfigPayloadBuilder payloadBuilder, String parentName) {
+ List<Element> children = XML.getChildren(currElem);
+ try {
+ if (children.isEmpty()) {
+ parseLeaf(currElem, payloadBuilder, parentName);
+ } else {
+ parseComplex(currElem, children, payloadBuilder, parentName);
+ }
+ } catch (Exception exception) {
+ throw new ConfigurationRuntimeException("Error parsing element at " + XML.getNodePath(currElem, " > ") + ": " +
+ Exceptions.toMessageString(exception));
+ }
+ }
+
+ private void verifyLegalOperation(Element currElem) {
+ String operation = currElem.getAttribute("operation");
+ if (! operation.equalsIgnoreCase("append"))
+ throw new ConfigurationRuntimeException("The only supported array operation is 'append', got '"
+ + operation + "' at XML node '" + XML.getNodePath(currElem, " > ") + "'.");
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java
new file mode 100644
index 00000000000..fe339835c78
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author gjoranv
+ */
+public abstract class DomContainerClusterBuilder<CLUSTER extends ContainerCluster>
+ extends VespaDomBuilder.DomConfigProducerBuilder<CLUSTER> {
+
+ protected final Element outerChainsElem;
+
+ public DomContainerClusterBuilder(Element outerChainsElem) {
+ this.outerChainsElem = outerChainsElem;
+ }
+
+ protected void buildAndAddUserConfiguredComponents(ContainerCluster cluster, Element spec) {
+ buildAndAddConfiguredHandlers(cluster, spec);
+ buildAndAddClientProviders(cluster, spec);
+ buildAndAddServerProviders(cluster, spec);
+ buildAndAddGenericComponents(cluster, spec);
+ buildAndAddRenderers(cluster, spec);
+ buildAndAddFilters(cluster, spec);
+ }
+
+ public void addSpecialHandlers(ContainerCluster cluster) {
+ ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster);
+ }
+
+ private void buildAndAddConfiguredHandlers(ContainerCluster cluster, Element spec) {
+ List<Handler> handlers = buildConfiguredHandlers(new DomHandlerBuilder(true), cluster, spec, "handler");
+
+ for (Handler handler : handlers) {
+ // TODO: hack to avoid adding a simple Handler for an explicitly declared SearchHandler
+ if (handler.getClassId().getName().equals("com.yahoo.search.handler.SearchHandler")) {
+ final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(
+ cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler");
+ searchHandler.addServerBindings("http://*/search/*");
+ cluster.addComponent(searchHandler);
+ } else
+ cluster.addComponent(handler);
+ }
+ }
+
+ private void buildAndAddClientProviders(ContainerCluster cluster, Element spec) {
+ List<Handler> clients = buildConfiguredHandlers(new DomClientProviderBuilder(), cluster, spec, "client");
+
+ for (Handler client : clients) {
+ cluster.addComponent(client);
+ }
+ }
+
+ private void buildAndAddServerProviders(ContainerCluster cluster, Element spec) {
+ ContainerModelBuilder.addConfiguredComponents(cluster, spec, "server");
+ }
+
+ private void buildAndAddGenericComponents(ContainerCluster cluster, Element spec) {
+ ContainerModelBuilder.addConfiguredComponents(cluster, spec, DomComponentBuilder.elementName);
+ }
+
+ private void buildAndAddFilters(ContainerCluster cluster, Element spec) {
+ for (Component component : buildConfiguredFilters(cluster, spec, "filter")) {
+ cluster.addComponent(component);
+ }
+ }
+
+ private List<Component> buildConfiguredFilters(AbstractConfigProducer ancestor,
+ Element spec,
+ String componentName) {
+ List<Component> components = new ArrayList<>();
+
+ for (Element node : XML.getChildren(spec, componentName)) {
+ components.add(new DomFilterBuilder().build(ancestor, node));
+ }
+ return components;
+ }
+
+ private List<Handler> buildConfiguredHandlers(DomHandlerBuilder builder,
+ AbstractConfigProducer ancestor,
+ Element spec,
+ String componentName) {
+ List<Handler> handlers = new ArrayList<>();
+
+ for (Element node : XML.getChildren(spec, componentName)) {
+ handlers.add(builder.build(ancestor, node));
+ }
+ return handlers;
+ }
+
+ protected void buildAndAddRenderers(ContainerCluster cluster, Element spec) {
+ ContainerModelBuilder.addConfiguredComponents(cluster, spec, "renderer");
+ }
+
+ protected void buildAndAddProcessingRenderers(ContainerCluster cluster, Element spec) {
+ ContainerModelBuilder.addConfiguredComponents(cluster, spec, "renderer");
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java
new file mode 100644
index 00000000000..930e7105c2f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author balder
+ */
+public class DomContentBuilder extends ConfigModelBuilder<Content> {
+
+ public static final List<ConfigModelId> configModelIds = Collections.singletonList(ConfigModelId.fromName("content"));
+
+ public DomContentBuilder() {
+ super(Content.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() {
+ return configModelIds;
+ }
+
+ @Override
+ public void doBuild(Content content, Element xml, ConfigModelContext modelContext) {
+ Admin admin = content.adminModel() != null ? content.adminModel().getAdmin() : null; // This is null in tests only
+ ContentCluster cluster = new ContentCluster.Builder(admin, modelContext.getDeployLogger()).build(modelContext.getParentProducer(), xml);
+ content.setCluster(cluster, modelContext);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java
new file mode 100644
index 00000000000..434c57fdcb3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.admin.FileDistributionOptions;
+import org.w3c.dom.Element;
+
+/**
+ * Builds a file distribution options.
+ * @author tonytv
+ */
+public class DomFileDistributionOptionsBuilder {
+ private static void throwExceptionForElementInFileDistribution(String subElement, String reason) {
+ throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason);
+ }
+
+ private static void callSetter(FileDistributionOptions options, String name, BinaryScaledAmount amount) {
+ try {
+ options.getClass().getMethod(name, BinaryScaledAmountParser.class).invoke(options, amount);
+ } catch (IllegalArgumentException e) {
+ throwExceptionForElementInFileDistribution(name, e.getMessage());
+ }
+ catch (Exception e) {
+ if (e instanceof RuntimeException)
+ throw (RuntimeException)e;
+ else
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void setIfPresent(FileDistributionOptions options, String name, Element fileDistributionElement) {
+ try {
+ Element optionElement = XML.getChild(fileDistributionElement, name);
+ if (optionElement != null) {
+ String valueString = XML.getValue(optionElement);
+ BinaryScaledAmount amount = BinaryScaledAmountParser.parse(valueString);
+ callSetter(options, name, amount);
+ }
+ } catch (NumberFormatException e) {
+ throwExceptionForElementInFileDistribution(name, "Expected a valid number. (Message = " + e.getMessage() + ").");
+ }
+ }
+
+ public FileDistributionOptions build(Element fileDistributionElement) {
+ FileDistributionOptions options = FileDistributionOptions.defaultOptions();
+ if (fileDistributionElement != null) {
+ setIfPresent(options, "uploadbitrate", fileDistributionElement);
+ setIfPresent(options, "downloadbitrate", fileDistributionElement);
+ }
+ return options;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java
new file mode 100644
index 00000000000..741b6f99944
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.HttpFilter;
+import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
+import org.w3c.dom.Element;
+
+/**
+ * @author tonytv
+ */
+public class DomFilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> {
+ @Override
+ protected Component doBuild(AbstractConfigProducer ancestor, Element element) {
+ return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element, false));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
new file mode 100644
index 00000000000..39a3ab553d2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
+import org.w3c.dom.Element;
+
+/**
+ * @author gjoranv
+ * @since 5.1.6
+ */
+public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> {
+ private final boolean legacyMode;
+
+ public DomHandlerBuilder(boolean legacyMode) {
+ this.legacyMode = legacyMode;
+ }
+
+ public DomHandlerBuilder() {
+ this(false);
+ }
+
+ @Override
+ protected Handler doBuild(AbstractConfigProducer ancestor, Element handlerElement) {
+ Handler<? super Component<?, ?>> handler = getHandler(handlerElement);
+
+ for (Element binding : XML.getChildren(handlerElement, "binding"))
+ handler.addServerBindings(XML.getValue(binding));
+
+ for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding"))
+ handler.addClientBindings(XML.getValue(clientBinding));
+
+ DomComponentBuilder.addChildren(ancestor, handlerElement, handler);
+
+ return handler;
+ }
+
+ protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) {
+ BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement, legacyMode);
+ return new Handler<>(
+ new ComponentModel(bundleSpec));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java
new file mode 100644
index 00000000000..49d475ec057
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.admin.Metric;
+import com.yahoo.vespa.model.admin.MetricsConsumer;
+import org.w3c.dom.Element;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helper class for parsing yamasmetric config.
+ *
+ * @author trygve
+ * @since 5.1
+ */
+public class DomMetricBuilderHelper {
+
+
+ /**
+ * Build metricConsumer config
+ *
+ * @param spec xml element
+ * @return a map from metric name to a {@link MetricsConsumer}
+ */
+ protected static Map<String, MetricsConsumer> buildMetricsConsumers(Element spec) {
+ Map<String, MetricsConsumer> metricsConsumers = new LinkedHashMap<>();
+ List<Element> consumersElem = XML.getChildren(spec, "consumer");
+ for (Element consumer : consumersElem) {
+ String consumerName = consumer.getAttribute("name");
+ Map<String, Metric> metrics = new LinkedHashMap<>();
+ List<Element> metricsEl = XML.getChildren(consumer, "metric");
+ if (metricsEl != null) {
+ for (Element metric : metricsEl) {
+ String metricName = metric.getAttribute("name");
+ String outputName = metric.getAttribute("output-name");
+ metrics.put(metricName, new Metric(metricName, outputName));
+ }
+ }
+ MetricsConsumer metricsConsumer = new MetricsConsumer(consumerName, metrics);
+ metricsConsumers.put(consumerName, metricsConsumer);
+ }
+ return metricsConsumers;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java
new file mode 100644
index 00000000000..dad00f529fb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.application.Xml;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.messagebus.routing.*;
+import com.yahoo.text.XML;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.routing.Routing;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Builds the Routing plugin
+ *
+ * @author vegardh
+ */
+public class DomRoutingBuilder extends ConfigModelBuilder<Routing> {
+
+ public DomRoutingBuilder() {
+ super(Routing.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() {
+ return Arrays.asList(ConfigModelId.fromName("routing"));
+ }
+
+ // Overrides ConfigModelBuilder.
+ @Override
+ public void doBuild(Routing plugin, Element spec, ConfigModelContext modelContext) {
+ ApplicationSpec app = null;
+ RoutingSpec routing = null;
+ if (spec != null) {
+ app = new ApplicationSpec();
+ for (Element node : Xml.mergeElems(spec, "services", modelContext.getApplicationPackage(), ApplicationPackage.ROUTINGTABLES_DIR)) {
+ addServices(app, node);
+ }
+ routing = new RoutingSpec();
+ for (Element node : Xml.mergeElems(spec, "routingtable", modelContext.getApplicationPackage(), ApplicationPackage.ROUTINGTABLES_DIR)) {
+ addRoutingTable(routing, node);
+ }
+ }
+ plugin.setExplicitApplicationSpec(app);
+ plugin.setExplicitRoutingSpec(routing);
+ }
+
+ /**
+ * This function updates the given application with the data contained in the given xml element.
+ *
+ * @param app The application to update.
+ * @param element The element to base the services on.
+ */
+ private static void addServices(ApplicationSpec app, Element element) {
+ String protocol = element.getAttribute("protocol");
+ for (Element node : XML.getChildren(element, "service")) {
+ app.addService(protocol, node.getAttribute("name"));
+ }
+ }
+
+ /**
+ * This function updates the given routing spec with the data contained in the given xml element.
+ *
+ * @param routing The routing spec to update.
+ * @param element The element to base the route config on.
+ */
+ private static void addRoutingTable(RoutingSpec routing, Element element) {
+ boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true;
+ RoutingTableSpec table = new RoutingTableSpec(element.getAttribute("protocol"), verify);
+
+ NodeList children = element.getChildNodes();
+ for (int i = 0; i < children.getLength(); i++) {
+ Node node = children.item(i);
+ if ("hop".equals(node.getNodeName())) {
+ table.addHop(createHopSpec((Element)node));
+ } else if ("route".equals(node.getNodeName())) {
+ table.addRoute(createRouteSpec((Element)node));
+ }
+ }
+
+ routing.addTable(table);
+ }
+
+ /**
+ * This function creates a route from the content of the given xml element.
+ *
+ * @param element The element to base the route config on.
+ * @return The corresponding route spec.
+ */
+ private static RouteSpec createRouteSpec(Element element) {
+ boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true;
+ RouteSpec route = new RouteSpec(element.getAttribute("name"), verify);
+ String hops = element.getAttribute("hops");
+ int from = 0;
+ for (int to = 0, depth = 0, len = hops.length(); to < len; ++to) {
+ if (hops.charAt(to) == '[') {
+ ++depth;
+ } else if (hops.charAt(to) == ']') {
+ --depth;
+ } else if (hops.charAt(to) == ' ' && depth == 0) {
+ if (to > from) {
+ route.addHop(hops.substring(from, to));
+ }
+ from = to + 1;
+ }
+ }
+ if (from < hops.length()) {
+ route.addHop(hops.substring(from));
+ }
+ return route;
+ }
+
+ /**
+ * This function creates a hop from the content of the given xml element.
+ *
+ * @param element The element to base the hop config on.
+ * @return The corresponding hop spec.
+ */
+ private static HopSpec createHopSpec(Element element) {
+ boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true;
+ HopSpec hop = new HopSpec(element.getAttribute("name"), element.getAttribute("selector"), verify);
+ if (Boolean.valueOf(element.getAttribute("ignore-result"))) {
+ hop.setIgnoreResult(true);
+ }
+ NodeList children = element.getElementsByTagName("recipient");
+ for (int i = 0; i < children.getLength(); i++) {
+ Element node = (Element)children.item(i);
+ hop.addRecipient(node.getAttribute("session"));
+ }
+ return hop;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java
new file mode 100644
index 00000000000..aa2dfd84eb5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java
@@ -0,0 +1,247 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.search.Tuning;
+import org.w3c.dom.Element;
+
+/**
+ * Builder for the tuning config for a search cluster.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Tuning> {
+
+ @Override
+ protected Tuning doBuild(AbstractConfigProducer parent, Element spec) {
+ Tuning tuning = new Tuning(parent);
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("dispatch", e)) {
+ handleDispatch(e, tuning);
+ } else if (equals("searchnode", e)) {
+ handleSearchNode(e, tuning);
+ }
+ }
+ return tuning;
+ }
+
+ private static boolean equals(String name, Element e) {
+ return name.equals(e.getNodeName());
+ }
+
+ private static String asString(Element e) {
+ return e.getFirstChild().getNodeValue();
+ }
+
+ private static Long asLong(Element e) {
+ return Long.parseLong(e.getFirstChild().getNodeValue());
+ }
+
+ private static Integer asInt(Element e) {
+ return Integer.parseInt(e.getFirstChild().getNodeValue());
+ }
+
+ private static Double asDouble(Element e) {
+ return Double.parseDouble(e.getFirstChild().getNodeValue());
+ }
+
+ private void handleDispatch(Element spec, Tuning t) {
+ t.dispatch = new Tuning.Dispatch();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("max-hits-per-partition", e)) {
+ t.dispatch.maxHitsPerPartition = asInt(e);
+ }
+ }
+ }
+
+ private void handleSearchNode(Element spec, Tuning t) {
+ t.searchNode = new Tuning.SearchNode();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("requestthreads", e)) {
+ handleRequestThreads(e, t.searchNode);
+ } else if (equals("flushstrategy", e)) {
+ handleFlushStrategy(e, t.searchNode);
+ } else if (equals("resizing", e)) {
+ handleResizing(e, t.searchNode);
+ } else if (equals("index", e)) {
+ handleIndex(e, t.searchNode);
+ } else if (equals("attribute", e)) {
+ handleAttribute(e, t.searchNode);
+ } else if (equals("summary", e)) {
+ handleSummary(e, t.searchNode);
+ }
+ }
+ }
+
+ private void handleRequestThreads(Element spec, Tuning.SearchNode sn) {
+ sn.threads = new Tuning.SearchNode.RequestThreads();
+ Tuning.SearchNode.RequestThreads rt = sn.threads;
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("search", e)) {
+ rt.numSearchThreads = asInt(e);
+ } else if (equals("persearch", e)) {
+ rt.numThreadsPerSearch = asInt(e);
+ } else if (equals("summary", e)) {
+ rt.numSummaryThreads = asInt(e);
+ }
+ }
+ }
+
+ private void handleFlushStrategy(Element spec, Tuning.SearchNode sn) {
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("native", e)) {
+ handleNativeStrategy(e, sn);
+ }
+ }
+ }
+
+ private void handleNativeStrategy(Element spec, Tuning.SearchNode sn) {
+ sn.strategy = new Tuning.SearchNode.FlushStrategy();
+ Tuning.SearchNode.FlushStrategy fs = sn.strategy;
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("total", e)) {
+ for (Element e2 : XML.getChildren(e)) {
+ if (equals("maxmemorygain", e2)) {
+ fs.totalMaxMemoryGain = asLong(e2);
+ } else if (equals("diskbloatfactor", e2)) {
+ fs.totalDiskBloatFactor = asDouble(e2);
+ }
+ }
+ } else if (equals("component", e)) {
+ for (Element e2 : XML.getChildren(e)) {
+ if (equals("maxmemorygain", e2)) {
+ fs.componentMaxMemoryGain = asLong(e2);
+ } else if (equals("diskbloatfactor", e2)) {
+ fs.componentDiskBloatFactor = asDouble(e2);
+ } else if (equals("maxage", e2)) {
+ fs.componentMaxage = asDouble(e2);
+ }
+ }
+ } else if (equals("transactionlog", e)) {
+ for (Element e2 : XML.getChildren(e)) {
+ if (equals("maxentries", e2)) {
+ fs.transactionLogMaxEntries = asLong(e2);
+ } else if (equals("maxsize", e2)) {
+ fs.transactionLogMaxSize = asLong(e2);
+ }
+ }
+ }
+ }
+ }
+
+ private void handleResizing(Element spec, Tuning.SearchNode sn) {
+ sn.resizing = new Tuning.SearchNode.Resizing();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("initialdocumentcount", e)) {
+ sn.resizing.initialDocumentCount = asInt(e);
+ }
+ }
+ }
+
+ private void handleIndex(Element spec, Tuning.SearchNode sn) {
+ sn.index = new Tuning.SearchNode.Index();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("io", e)) {
+ sn.index.io = new Tuning.SearchNode.Index.Io();
+ Tuning.SearchNode.Index.Io io = sn.index.io;
+ for (Element e2 : XML.getChildren(e)) {
+ if (equals("write", e2)) {
+ io.write = Tuning.SearchNode.IoType.fromString(asString(e2));
+ } else if (equals("read", e2)) {
+ io.read = Tuning.SearchNode.IoType.fromString(asString(e2));
+ } else if (equals("search", e2)) {
+ io.search = Tuning.SearchNode.IoType.fromString(asString(e2));
+ }
+ }
+ }
+ }
+ }
+
+ private void handleAttribute(Element spec, Tuning.SearchNode sn) {
+ sn.attribute = new Tuning.SearchNode.Attribute();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("io", e)) {
+ sn.attribute.io = new Tuning.SearchNode.Attribute.Io();
+ for (Element e2 : XML.getChildren(e)) {
+ if (equals("write", e2)) {
+ sn.attribute.io.write = Tuning.SearchNode.IoType.fromString(asString(e2));
+ }
+ }
+ }
+ }
+ }
+
+ private void handleSummary(Element spec, Tuning.SearchNode sn) {
+ sn.summary = new Tuning.SearchNode.Summary();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("io", e)) {
+ sn.summary.io = new Tuning.SearchNode.Summary.Io();
+ for (Element e2 : XML.getChildren(e)) {
+ if (equals("write", e2)) {
+ sn.summary.io.write = Tuning.SearchNode.IoType.fromString(asString(e2));
+ } else if (equals("read", e2)) {
+ sn.summary.io.read = Tuning.SearchNode.IoType.fromString(asString(e2));
+ }
+ }
+ } else if (equals("store", e)) {
+ handleSummaryStore(e, sn.summary);
+ }
+ }
+ }
+
+ private void handleSummaryStore(Element spec, Tuning.SearchNode.Summary s) {
+ s.store = new Tuning.SearchNode.Summary.Store();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("cache", e)) {
+ s.store.cache = new Tuning.SearchNode.Summary.Store.Component();
+ handleSummaryStoreComponent(e, s.store.cache);
+ } else if (equals("logstore", e)) {
+ handleSummaryLogStore(e, s.store);
+ }
+ }
+ }
+
+ private void handleSummaryStoreComponent(Element spec, Tuning.SearchNode.Summary.Store.Component c) {
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("maxsize", e)) {
+ c.maxSize = asLong(e);
+ } else if (equals("maxentries", e)) {
+ c.maxEntries = asLong(e);
+ } else if (equals("initialentries", e)) {
+ c.initialEntries = asLong(e);
+ } else if (equals("compression", e)) {
+ c.compression = new Tuning.SearchNode.Summary.Store.Compression();
+ handleSummaryStoreCompression(e, c.compression);
+ }
+ }
+ }
+
+ private void handleSummaryStoreCompression(Element spec, Tuning.SearchNode.Summary.Store.Compression c) {
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("type", e)) {
+ c.type = Tuning.SearchNode.Summary.Store.Compression.Type.fromString(asString(e));
+ } else if (equals("level", e)) {
+ c.level = asInt(e);
+ }
+ }
+ }
+
+ private void handleSummaryLogStore(Element spec, Tuning.SearchNode.Summary.Store s) {
+ s.logStore = new Tuning.SearchNode.Summary.Store.LogStore();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("maxfilesize", e)) {
+ s.logStore.maxFileSize = asLong(e);
+ } else if (equals("maxdiskbloatfactor", e)) {
+ s.logStore.maxDiskBloatFactor = asDouble(e);
+ } else if (equals("minfilesizefactor", e)) {
+ s.logStore.minFileSizeFactor = asDouble(e);
+ } else if (equals("numthreads", e)) {
+ s.logStore.numThreads = asInt(e);
+ } else if (equals("chunk", e)) {
+ s.logStore.chunk = new Tuning.SearchNode.Summary.Store.Component(true);
+ handleSummaryStoreComponent(e, s.logStore.chunk);
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
new file mode 100644
index 00000000000..65519637bd5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
@@ -0,0 +1,576 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.ConfigModelUtils;
+import com.yahoo.vespa.config.content.spooler.SpoolerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.SimpleConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder;
+import com.yahoo.vespa.model.clients.Clients;
+import com.yahoo.vespa.model.clients.HttpGatewayOwner;
+import com.yahoo.vespa.model.clients.VespaSpoolMaster;
+import com.yahoo.vespa.model.clients.VespaSpooler;
+import com.yahoo.vespa.model.clients.VespaSpoolerProducer;
+import com.yahoo.vespa.model.clients.VespaSpoolerService;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.search.ContainerHttpGateway;
+import com.yahoo.vespa.model.container.search.ContainerSearch;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChain;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.vespaclient.config.FeederConfig;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.logging.Level;
+
+/**
+ * Builds the Clients plugin
+ *
+ * @author vegardh
+ */
+public class DomV20ClientsBuilder {
+
+ public static final String vespaClientBundleSpecification = "vespaclient-container-plugin";
+
+ // The parent docproc plugin to register data with.
+ private final Clients clients;
+
+ DomV20ClientsBuilder(Clients clients, String version) {
+ this.clients = clients;
+ if (!version.equals("2.0")) {
+ throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported.");
+ }
+ }
+
+ public void build(Element spec) {
+ NodeList children = spec.getElementsByTagName("gateways");
+ if (children.getLength() > 0 && clients.getConfigProducer()!=null)
+ clients.getConfigProducer().deployLogger().log(Level.WARNING, "The 'gateways' element is deprecated, and will be disallowed in a " +
+ "later version of Vespa. Use 'document-api' under 'jdisc' instead, see: " +
+ ConfigModelUtils.createDocLink("reference/services-jdisc.html"));
+ for (int i = 0; i < children.getLength(); i++) {
+ createGateways(clients.getConfigProducer(), (Element) children.item(i), clients);
+ }
+
+ children = spec.getElementsByTagName("spoolers");
+ for (int i = 0; i < children.getLength(); i++) {
+ createSpoolers(clients.getConfigProducer(), (Element) children.item(i), clients);
+ }
+
+ children = spec.getElementsByTagName("load-types");
+ for (int i = 0; i < children.getLength(); i++) {
+ createLoadTypes((Element) children.item(i), clients);
+ }
+ }
+
+ static Boolean getBooleanNodeValue(Node node) {
+ return Boolean.valueOf(node.getFirstChild().getNodeValue());
+ }
+
+ static boolean getHttpFileServerEnabled(Element parentHttpFileServer, Element httpFileServer) {
+ boolean ret=false;
+ if (parentHttpFileServer != null) {
+ for (Element child : XML.getChildren(parentHttpFileServer)) {
+ if ("enabled".equals(child.getNodeName())) {
+ ret = getBooleanNodeValue(child);
+ }
+ }
+ }
+ if (httpFileServer != null) {
+ for (Element child : XML.getChildren(httpFileServer)) {
+ if ("enabled".equals(child.getNodeName())) {
+ ret = getBooleanNodeValue(child);
+ }
+ }
+ }
+ return ret;
+ }
+
+ static String getHttpFileServerRootDir(Element parentHttpFileServer, Element httpFileServer) {
+ String ret="";
+ if (parentHttpFileServer != null) {
+ for (Element child : XML.getChildren(parentHttpFileServer)) {
+ if ("rootdir".equals(child.getNodeName())) {
+ ret = child.getFirstChild().getNodeValue();
+ }
+ }
+ }
+ if (httpFileServer != null) {
+ for (Element child : XML.getChildren(httpFileServer)) {
+ if ("rootdir".equals(child.getNodeName())) {
+ ret = child.getFirstChild().getNodeValue();
+ }
+ }
+ }
+ return ret;
+ }
+
+ private void createLoadTypes(Element element, Clients clients) {
+ for (Element e : XML.getChildren(element, "type")) {
+ String priority = e.getAttribute("default-priority");
+ clients.getLoadTypes().addType(e.getAttribute("name"), priority.length() > 0 ? priority : null);
+ }
+ }
+
+ /**
+ * Creates HttpGateway objects using the given xml Element.
+ *
+ * @param pcp AbstractConfigProducer
+ * @param element The xml Element
+ */
+ private void createGateways(AbstractConfigProducer pcp, Element element, Clients clients) {
+ String jvmArgs = null;
+ if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME);
+
+ Element gatewaysFeederOptions = findFeederOptions(element);
+
+ HttpGatewayOwner owner = new HttpGatewayOwner(pcp, getFeederConfig(null, gatewaysFeederOptions));
+ ContainerCluster cluster = new ContainerHttpGatewayClusterBuilder().build(owner, element);
+
+ int index = 0;
+ for (Element e : XML.getChildren(element, "gateway")) {
+ ContainerHttpGateway qrs = new ContainerHttpGatewayBuilder(cluster, index).build(cluster, e);
+
+ if ("".equals(qrs.getJvmArgs()) && jvmArgs!=null) qrs.setJvmArgs(jvmArgs);
+ index++;
+ }
+ clients.setContainerHttpGateways(cluster);
+ }
+
+ /**
+ * Creates VespaSpooler objects using the given xml Element.
+ */
+ private void createSpoolers(AbstractConfigProducer pcp, Element element, Clients clients) {
+ String jvmArgs = null;
+ if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME);
+ SimpleConfigProducer spoolerCfg = new VespaDomBuilder.DomSimpleConfigProducerBuilder(element.getNodeName()).
+ build(pcp, element);
+ Element spoolersFeederOptions = findFeederOptions(element);
+ createSpoolMasters(spoolerCfg, element);
+ for (Element e : XML.getChildren(element, "spooler")) {
+ String configId = e.getAttribute("id").trim();
+ FeederConfig.Builder feederConfig = getFeederConfig(spoolersFeederOptions, e);
+ SpoolerConfig.Builder spoolConfig = getSpoolConfig(e);
+ if (configId.length() == 0) {
+ int index = clients.getVespaSpoolers().size();
+ VespaSpoolerService spoolerService = new VespaSpoolerServiceBuilder(index, new VespaSpooler(feederConfig, spoolConfig)).
+ build(spoolerCfg, e);
+ if ("".equals(spoolerService.getJvmArgs()) && jvmArgs!=null) spoolerService.setJvmArgs(jvmArgs);
+ spoolerService.setProp("index", String.valueOf(index));
+ clients.getVespaSpoolers().add(spoolerService);
+ } else {
+ new VespaSpoolerProducerBuilder(configId, new VespaSpooler(feederConfig, spoolConfig)).
+ build(spoolerCfg, e);
+ }
+ }
+ }
+
+ private void createSpoolMasters(SimpleConfigProducer producer,
+ Element element) {
+ int i=0;
+ for (Element e : XML.getChildren(element, "spoolmaster")) {
+ VespaSpoolMaster master = new VespaSpoolMasterBuilder(i).build(producer, e);
+ i++;
+ }
+ }
+
+ private SpoolerConfig.Builder getSpoolConfig(Element conf) {
+ SpoolerConfig.Builder builder = new SpoolerConfig.Builder();
+ if (conf.getAttributes().getNamedItem("directory") != null) {
+ builder.directory(Defaults.getDefaults().underVespaHome(conf.getAttributes().getNamedItem("directory").getNodeValue()));
+ }
+ if (conf.getAttributes().getNamedItem("keepsuccess") != null) {
+ builder.keepsuccess(getBooleanFromAttribute(conf, "keepsuccess"));
+ }
+ if (conf.getAttributes().getNamedItem("maxfailuresize") != null) {
+ builder.maxfailuresize(getIntegerFromAttribute(conf, "maxfailuresize"));
+ }
+ if (conf.getAttributes().getNamedItem("maxfatalfailuresize") != null) {
+ builder.maxfatalfailuresize(getIntegerFromAttribute(conf, "maxfatalfailuresize"));
+ }
+ if (conf.getAttributes().getNamedItem("threads") != null) {
+ builder.threads(getIntegerFromAttribute(conf, "threads"));
+ }
+ if (conf.getAttributes().getNamedItem("maxretries") != null) {
+ builder.maxretries(getIntegerFromAttribute(conf, "maxretries"));
+ }
+
+ NodeList children = conf.getElementsByTagName("parsers");
+ if (children.getLength() == 1) {
+ children = ((Element)children.item(0)).getElementsByTagName("parser");
+ }
+
+ for (int i=0; i < children.getLength(); i++) {
+ Element e = (Element)children.item(i);
+
+ String type = e.getAttributes().getNamedItem("type").getNodeValue();
+ NodeList params = e.getElementsByTagName("parameter");
+
+ SpoolerConfig.Parsers.Builder parserBuilder = new SpoolerConfig.Parsers.Builder();
+ parserBuilder.classname(type);
+ if (params.getLength() > 0) {
+ List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>();
+ for (int j = 0; j < params.getLength(); j++) {
+ SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder();
+ Element p = (Element) params.item(j);
+ parametersBuilder.key(getStringFromAttribute(p, "key"));
+ parametersBuilder.value(getStringFromAttribute(p, "value"));
+ parametersBuilders.add(parametersBuilder);
+ }
+ parserBuilder.parameters(parametersBuilders);
+ }
+
+ builder.parsers.add(parserBuilder);
+ }
+ return builder;
+ }
+
+ Boolean getBooleanFromAttribute(Element e, String attributeName) {
+ return Boolean.parseBoolean(e.getAttributes().getNamedItem(attributeName).getNodeValue());
+ }
+
+ Integer getIntegerFromAttribute(Element e, String attributeName) {
+ return Integer.parseInt(e.getAttributes().getNamedItem(attributeName).getNodeValue());
+ }
+
+ String getStringFromAttribute(Element e, String attributeName) {
+ return e.getAttributes().getNamedItem(attributeName).getNodeValue();
+ }
+
+ private FeederConfig.Builder getFeederConfig(Element gatewaysFeederOptions, Element e) {
+ FeederOptionsParser foParser = new FeederOptionsParser();
+ if (gatewaysFeederOptions!=null) {
+ foParser.parseFeederOptions(gatewaysFeederOptions).getFeederConfig();
+ }
+ foParser.parseFeederOptions(e);
+ return foParser.getFeederConfig();
+ }
+
+ /**
+ * Finds the feederoptions subelement in the given xml Element.
+ *
+ * @param element The xml Element
+ * @return The feederoptions xml Element
+ */
+ private Element findFeederOptions(Element element) {
+ for (Element child : XML.getChildren(element)) {
+ if (child.getNodeName().equals("feederoptions")) {
+ return child;
+ }
+ }
+ return null;
+ }
+
+ private static class VespaSpoolerServiceBuilder extends DomConfigProducerBuilder<VespaSpoolerService> {
+ private int index;
+ private VespaSpooler spoolerConfig;
+
+ public VespaSpoolerServiceBuilder(int index, VespaSpooler spoolerConfig) {
+ this.index = index;
+ this.spoolerConfig = spoolerConfig;
+ }
+
+ @Override
+ protected VespaSpoolerService doBuild(AbstractConfigProducer parent,
+ Element spec) {
+ return new VespaSpoolerService(parent, index, spoolerConfig);
+ }
+ }
+
+ private static class VespaSpoolerProducerBuilder extends DomConfigProducerBuilder<VespaSpoolerProducer> {
+ private String name=null;
+ private VespaSpooler spooler;
+
+ public VespaSpoolerProducerBuilder(String name, VespaSpooler spooler) {
+ this.name = name;
+ this.spooler = spooler;
+ }
+
+ @Override
+ protected VespaSpoolerProducer doBuild(AbstractConfigProducer parent,
+ Element producerSpec) {
+ return new VespaSpoolerProducer(parent, name, spooler);
+ }
+ }
+
+ private static class VespaSpoolMasterBuilder extends DomConfigProducerBuilder<VespaSpoolMaster> {
+ int index;
+
+ public VespaSpoolMasterBuilder(int index) {
+ super();
+ this.index = index;
+ }
+
+ @Override
+ protected VespaSpoolMaster doBuild(AbstractConfigProducer parent,
+ Element spec) {
+ return new VespaSpoolMaster(parent, index);
+ }
+ }
+
+ public static class ContainerHttpGatewayClusterBuilder extends DomConfigProducerBuilder<ContainerCluster> {
+ @Override
+ protected ContainerCluster doBuild(AbstractConfigProducer parent,
+ Element spec) {
+
+ ContainerCluster cluster = new ContainerCluster(parent, "gateway", "gateway");
+
+ SearchChains searchChains = new SearchChains(cluster, "searchchain");
+ Set<ComponentSpecification> inherited = new TreeSet<>();
+ //inherited.add(new ComponentSpecification("vespa", null, null));
+ {
+ SearchChain mySearchChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"),
+ new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>()));
+ Searcher getComponent = newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher");
+ mySearchChain.addInnerComponent(getComponent);
+ searchChains.add(mySearchChain);
+ }
+ {
+ SearchChain mySearchChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"),
+ new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>()));
+ Searcher getComponent = newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher");
+ mySearchChain.addInnerComponent(getComponent);
+ searchChains.add(mySearchChain);
+ }
+
+ ContainerSearch containerSearch = new ContainerSearch(cluster, searchChains, new ContainerSearch.Options());
+ cluster.setSearch(containerSearch);
+
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "http://*/feed"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "http://*/remove"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "http://*/removelocation"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "http://*/get"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "http://*/visit"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "http://*/document"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerStatus", "http://*/feedstatus"));
+ final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(
+ cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler");
+ searchHandler.addServerBindings("http://*/search/*");
+ cluster.addComponent(searchHandler);
+
+ ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster);
+
+ //BEGIN HACK for docproc chains:
+ DocprocChains docprocChains = getDocprocChains(cluster, spec);
+ if (docprocChains != null) {
+ ContainerDocproc containerDocproc = new ContainerDocproc(cluster, docprocChains);
+ cluster.setDocproc(containerDocproc);
+ }
+ //END HACK
+
+ return cluster;
+ }
+
+ private Handler newVespaClientHandler(String componentId, String binding) {
+ Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel(
+ BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), ""));
+ handler.addServerBindings(binding);
+ handler.addServerBindings(binding + '/');
+ return handler;
+ }
+
+ private Searcher newVespaClientSearcher(String componentSpec) {
+ return new Searcher<>(new ChainedComponentModel(
+ BundleInstantiationSpecification.getFromStrings(componentSpec, null, vespaClientBundleSpecification),
+ new Dependencies(null, null, null)));
+ }
+
+ //BEGIN HACK for docproc chains:
+ private DocprocChains getDocprocChains(AbstractConfigProducer qrs, Element gateways) {
+ Element clients = (Element) gateways.getParentNode();
+ Element services = (Element) clients.getParentNode();
+ if (services == null) {
+ return null;
+ }
+
+ Element docproc = XML.getChild(services, "docproc");
+ if (docproc == null) {
+ return null;
+ }
+
+ String version = docproc.getAttribute("version");
+ if (version.startsWith("1.")) {
+ return null;
+ } else if (version.startsWith("2.")) {
+ return null;
+ } else if (version.startsWith("3.")) {
+ return getDocprocChainsV3(qrs, docproc);
+ } else {
+ throw new IllegalArgumentException("Docproc version " + version + " unknown.");
+ }
+ }
+
+ private DocprocChains getDocprocChainsV3(AbstractConfigProducer qrs, Element docproc) {
+ Element docprocChainsElem = XML.getChild(docproc, "docprocchains");
+ if (docprocChainsElem == null) {
+ return null;
+ }
+ return new DomDocprocChainsBuilder(null, true).build(qrs, docprocChainsElem);
+ }
+ //END HACK
+ }
+
+ public static class ContainerHttpGatewayBuilder extends DomConfigProducerBuilder<ContainerHttpGateway> {
+ int index;
+ ContainerCluster cluster;
+
+ public ContainerHttpGatewayBuilder(ContainerCluster cluster, int index) {
+ this.index = index;
+ this.cluster = cluster;
+ }
+
+ @Override
+ protected ContainerHttpGateway doBuild(AbstractConfigProducer parent,
+ Element spec) {
+ // TODO: remove port handling
+ int port = 19020;
+ if (spec != null && spec.hasAttribute("baseport")) {
+ port = Integer.parseInt(spec.getAttribute("baseport"));
+ }
+ ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port);
+ List<Container> containers = new ArrayList<>();
+ containers.add(httpGateway);
+
+ cluster.addContainers(containers);
+ return httpGateway;
+ }
+ }
+
+ /**
+ * This class parses the feederoptions xml tag and produces Vespa config output.
+ *
+ * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ */
+ private class FeederOptionsParser implements Serializable {
+ private static final long serialVersionUID = 1L;
+ // All member variables are objects so that we can switch on null values.
+ private Boolean abortondocumenterror = null;
+ private String route = null;
+ private Integer maxpendingdocs = null;
+ private Integer maxpendingbytes = null;
+ private Boolean retryenabled = null;
+ private Double retrydelay = null;
+ private Double timeout = null;
+ private Integer tracelevel = null;
+ private Integer mbusport = null;
+ private String docprocChain = null;
+
+ /**
+ * Constructs an empty feeder options object with all members set to null.
+ */
+ public FeederOptionsParser() {
+ // empty
+ }
+
+ /**
+ * Parses the content of the given XML element as feeder options.
+ *
+ * @param conf The XML element to parse.
+ */
+ public FeederOptionsParser parseFeederOptions(Element conf) {
+ for (Node node : XML.getChildren(conf)) {
+ String nodename = node.getNodeName();
+ Node firstchild = node.getFirstChild();
+ String childval = (firstchild != null) ? firstchild.getNodeValue() : null;
+
+ switch (nodename) {
+ case "abortondocumenterror":
+ abortondocumenterror = Boolean.valueOf(childval);
+ break;
+ case "maxpendingdocs":
+ maxpendingdocs = new Integer(childval);
+ break;
+ case "maxpendingbytes":
+ maxpendingbytes = new Integer(childval);
+ break;
+ case "retryenabled":
+ retryenabled = Boolean.valueOf(childval);
+ break;
+ case "retrydelay":
+ retrydelay = new Double(childval);
+ break;
+ case "timeout":
+ timeout = new Double(childval);
+ break;
+ case "route":
+ route = childval;
+ break;
+ case "tracelevel":
+ tracelevel = new Integer(childval);
+ break;
+ case "mbusport":
+ mbusport = new Integer(childval);
+ break;
+ case "docprocchain":
+ docprocChain = childval;
+ break;
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Returns a feeder options config string of the content of this.
+ *
+ * @return A config string.
+ */
+ public FeederConfig.Builder getFeederConfig() {
+ FeederConfig.Builder builder = new FeederConfig.Builder();
+ if (abortondocumenterror != null) {
+ builder.abortondocumenterror(abortondocumenterror);
+ }
+ if (route != null && route.length() > 0) {
+ builder.route(route);
+ }
+ if (maxpendingdocs != null) {
+ builder.maxpendingdocs(maxpendingdocs);
+ }
+ if (maxpendingbytes != null) {
+ builder.maxpendingbytes(maxpendingbytes);
+ }
+ if (retryenabled != null) {
+ builder.retryenabled(retryenabled);
+ }
+ if (retrydelay != null) {
+ builder.retrydelay(retrydelay);
+ }
+ if (timeout != null) {
+ builder.timeout(timeout);
+ }
+ if (tracelevel != null) {
+ builder.tracelevel(tracelevel);
+ }
+ if (mbusport != null) {
+ builder.mbusport(mbusport);
+ }
+ if (docprocChain != null && docprocChain.length() > 0) {
+ builder.docprocchain(docprocChain);
+ }
+ return builder;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java
new file mode 100644
index 00000000000..87a88fae176
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.ConfigModelInstanceFactory;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import org.w3c.dom.Element;
+
+/**
+ * A model builder that can be used to deal with toplevel config overrides and create another
+ * producer in between. This should not be used by new model plugins.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public abstract class LegacyConfigModelBuilder<MODEL extends ConfigModel> extends ConfigModelBuilder<MODEL> {
+
+ public LegacyConfigModelBuilder(Class<MODEL> configModelClass) {
+ super(configModelClass);
+ }
+
+ @Override
+ public MODEL build(ConfigModelInstanceFactory<MODEL> factory, Element spec, ConfigModelContext context) {
+ VespaDomBuilder.DomSimpleConfigProducerBuilder builder = new VespaDomBuilder.DomSimpleConfigProducerBuilder(context.getProducerId());
+ AbstractConfigProducer producer = builder.build(context.getParentProducer(), spec);
+ return super.build(factory, spec, context.modifyParent(producer));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
new file mode 100644
index 00000000000..a60e0dd01e4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
@@ -0,0 +1,211 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.utils.Duration;
+
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+/**
+ * A w3c Element wrapper whith a better API.
+ *
+ * Author unknown.
+ */
+public class ModelElement {
+
+ private final Element xml;
+
+ public ModelElement(Element xml) {
+ this.xml = xml;
+ if (xml == null) {
+ throw new NullPointerException("Can not create ModelElement with null element");
+ }
+ if (xml.getNodeName() == null) {
+ throw new NullPointerException("Can not create ModelElement with unnamed element");
+ }
+ }
+
+ public Element getXml() {
+ return xml;
+ }
+
+ /**
+ * If not found, return null.
+ */
+ public ModelElement getChild(String name) {
+ Element e = XML.getChild(xml, name);
+
+ if (e != null) {
+ return new ModelElement(e);
+ }
+
+ return null;
+ }
+
+ public ModelElement getChildByPath(String path) {
+ StringTokenizer tokenizer = new StringTokenizer(path, ".");
+ ModelElement curElem = this;
+ while (tokenizer.hasMoreTokens() && curElem != null) {
+ String pathElem = tokenizer.nextToken();
+ ModelElement child = curElem.getChild(pathElem);
+ if (!tokenizer.hasMoreTokens()) {
+ if (child != null) {
+ return child;
+ }
+ }
+ curElem = child;
+ }
+ return null;
+ }
+
+ public String childAsString(String path) {
+ StringTokenizer tokenizer = new StringTokenizer(path, ".");
+ ModelElement curElem = this;
+ while (tokenizer.hasMoreTokens() && curElem != null) {
+ String pathElem = tokenizer.nextToken();
+ ModelElement child = curElem.getChild(pathElem);
+ if (!tokenizer.hasMoreTokens()) {
+ String attr = curElem.getStringAttribute(pathElem);
+ if (attr != null) {
+ return attr;
+ } else if (child != null) {
+ return child.asString();
+ }
+ }
+ curElem = child;
+ }
+ return null;
+ }
+
+ public String asString() {
+ return xml.getFirstChild().getTextContent();
+ }
+
+ public double asDouble() {
+ return Double.parseDouble(asString());
+ }
+
+ public long asLong() {
+ return (long) BinaryUnit.valueOf(asString());
+ }
+
+ public Duration asDuration() {
+ return new Duration(asString());
+ }
+
+ public Long childAsLong(String path) {
+ String child = childAsString(path);
+ if (child == null) {
+ return null;
+ }
+ return Long.parseLong(child.trim());
+ }
+
+ public Integer childAsInteger(String path) {
+ String child = childAsString(path);
+ if (child == null) {
+ return null;
+ }
+ return Integer.parseInt(child.trim());
+ }
+
+ public Double childAsDouble(String path) {
+ String child = childAsString(path);
+ if (child == null) {
+ return null;
+ }
+ return Double.parseDouble(child.trim());
+ }
+
+ public Boolean childAsBoolean(String path) {
+ String child = childAsString(path);
+ if (child == null) {
+ return null;
+ }
+ return Boolean.parseBoolean(child.trim());
+ }
+
+ public Duration childAsDuration(String path) {
+ String child = childAsString(path);
+ if (child == null) {
+ return null;
+ }
+ return new Duration(child);
+ }
+
+ /** Returns the given attribute or throws IllegalArgumentException if not present */
+ public int requiredIntegerAttribute(String name) {
+ if (getStringAttribute(name) == null)
+ throw new IllegalArgumentException("Required attribute '" + name + "' is missing");
+ return getIntegerAttribute(name, null);
+ }
+
+ /** Returns the value of this attribute or null if not present */
+ public Integer getIntegerAttribute(String name) {
+ return getIntegerAttribute(name, null);
+ }
+
+ public Integer getIntegerAttribute(String name, Integer defaultValue) {
+ String value = getStringAttribute(name);
+ if (value == null) {
+ return defaultValue;
+ }
+ return (int) BinaryUnit.valueOf(value);
+ }
+
+ public boolean getBooleanAttribute(String name) {
+ return getBooleanAttribute(name, false);
+ }
+
+ public boolean getBooleanAttribute(String name, boolean defaultValue) {
+ String value = getStringAttribute(name);
+ if (value == null) {
+ return defaultValue;
+ }
+ return Boolean.parseBoolean(value);
+ }
+
+ public Long getLongAttribute(String name) {
+ String value = getStringAttribute(name);
+ if (value == null) {
+ return null;
+ }
+ return (long) BinaryUnit.valueOf(value);
+ }
+
+ public Double getDoubleAttribute(String name) {
+ String value = getStringAttribute(name);
+ if (value == null) {
+ return null;
+ }
+ return Double.parseDouble(value);
+ }
+
+ public String getStringAttribute(String name) {
+ if (!xml.hasAttribute(name)) {
+ return null;
+ }
+
+ return xml.getAttribute(name);
+ }
+
+ public List<ModelElement> subElements(String name) {
+ List<Element> elements = XML.getChildren(xml, name);
+
+ List<ModelElement> helpers = new ArrayList<>();
+ for (Element e : elements) {
+ helpers.add(new ModelElement(e));
+ }
+
+ return helpers;
+ }
+
+ @Override
+ public String toString() {
+ return xml.getNodeName();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
new file mode 100644
index 00000000000..a27739b42ef
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -0,0 +1,105 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.HostSystem;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A common utility class to represent a requirement for some nodes during model building.
+ * Such a requirement is commonly specified as a <code>nodes</code> element.
+ *
+ * @author bratseth
+ */
+ // TODO: Use this for all nodes tags and unify with NodesUtil
+public class NodesSpecification {
+
+ private final boolean dedicated;
+
+ private final int count;
+
+ private final int groups;
+
+ private final Optional<String> flavor;
+
+ private final Optional<String> dockerImage;
+
+ private NodesSpecification(boolean dedicated, int count, int groups, Optional<String> flavor, Optional<String> dockerImage) {
+ this.dedicated = dedicated;
+ this.count = count;
+ this.groups = groups;
+ this.flavor = flavor;
+ this.dockerImage = dockerImage;
+ }
+
+ private NodesSpecification(boolean dedicated, ModelElement nodesElement) {
+ this(dedicated,
+ nodesElement.requiredIntegerAttribute("count"),
+ nodesElement.getIntegerAttribute("groups", 1),
+ Optional.ofNullable(nodesElement.getStringAttribute("flavor")),
+ Optional.ofNullable(nodesElement.getStringAttribute("docker-image")));
+ }
+
+ /**
+ * Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element
+ */
+ public static NodesSpecification from(ModelElement nodesElement) {
+ return new NodesSpecification(true, nodesElement);
+ }
+
+ /**
+ * Returns a requirement for dedicated nodes taken from the <code>nodes</code> element
+ * contained in the given parent element, or empty if the parent element is null, or the nodes elements
+ * is not present.
+ */
+ public static Optional<NodesSpecification> fromParent(ModelElement parentElement) {
+ if (parentElement == null) return Optional.empty();
+ ModelElement nodesElement = parentElement.getChild("nodes");
+ if (nodesElement == null) return Optional.empty();
+ return Optional.of(from(nodesElement));
+ }
+
+ /**
+ * Returns a requirement for undedicated or dedicated nodes taken from the <code>nodes</code> element
+ * contained in the given parent element, or empty if the parent element is null, or the nodes elements
+ * is not present.
+ */
+ public static Optional<NodesSpecification> optionalDedicatedFromParent(ModelElement parentElement) {
+ if (parentElement == null) return Optional.empty();
+ ModelElement nodesElement = parentElement.getChild("nodes");
+ if (nodesElement == null) return Optional.empty();
+ return Optional.of(new NodesSpecification(nodesElement.getBooleanAttribute("dedicated", false), nodesElement));
+ }
+
+ /** Returns a requirement from <code>count</code> nondedicated nodes in one group */
+ public static NodesSpecification nonDedicated(int count) {
+ return new NodesSpecification(false, count, 1, Optional.empty(), Optional.empty());
+ }
+
+ /**
+ * Returns whether this requires dedicated nodes.
+ * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible.
+ */
+ public boolean isDedicated() { return dedicated; }
+
+ /** Returns the number of nodes required */
+ public int count() { return count; }
+
+ /** Returns the number of host groups this specifies. Default is 1 */
+ public int groups() { return groups; }
+
+ public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, Optional<ClusterSpec.Group> clusterGroup, DeployLogger logger) {
+ if (clusterGroup.isPresent() && groups > 1)
+ throw new IllegalArgumentException("Cannot both specify a group and request multiple groups");
+ ClusterSpec cluster = ClusterSpec.from(clusterType, clusterId, clusterGroup, dockerImage);
+ return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor), groups, logger);
+ }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java
new file mode 100644
index 00000000000..147f6517754
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.container.component.Servlet;
+import com.yahoo.vespa.model.container.component.ServletProvider;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author stiankri
+ * @since 5.32
+ */
+public class ServletBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Servlet> {
+ @Override
+ protected ServletProvider doBuild(AbstractConfigProducer ancestor, Element servletElement) {
+ SimpleComponent servlet = createServletComponent(servletElement);
+ ServletProvider servletProvider = createServletProvider(servletElement, servlet);
+
+ return servletProvider;
+ }
+
+ private SimpleComponent createServletComponent(Element servletElement) {
+ ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement, false));
+ return new SimpleComponent(componentModel);
+ }
+
+ private ServletProvider createServletProvider(Element servletElement, SimpleComponent servlet) {
+ Map<String, String> servletConfig = getServletConfig(servletElement);
+ return new ServletProvider(servlet, getPath(servletElement), servletConfig);
+ }
+
+ private String getPath(Element servletElement) {
+ Element pathElement = XML.getChild(servletElement, "path");
+ return XML.getValue(pathElement);
+ }
+
+ private Map<String, String> getServletConfig(Element servletElement) {
+ Map<String, String> servletConfig = new HashMap<>();
+
+ Element servletConfigElement = XML.getChild(servletElement, "servlet-config");
+ XML.getChildren(servletConfigElement).forEach( parameter ->
+ servletConfig.put(parameter.getTagName(), XML.getValue(parameter))
+ );
+
+ return servletConfig;
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
new file mode 100644
index 00000000000..08c3db81091
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
@@ -0,0 +1,301 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.config.model.*;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.UserConfigRepo;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.builder.UserConfigBuilder;
+import com.yahoo.vespa.model.builder.VespaModelBuilder;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder;
+import com.yahoo.vespa.model.generic.service.ServiceCluster;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Builds Vespa model components using the w3c dom api
+ *
+ * @author vegardh
+ */
+public class VespaDomBuilder extends VespaModelBuilder {
+
+ public static final String JVMARGS_ATTRIB_NAME="jvmargs";
+ public static final String PRELOAD_ATTRIB_NAME="preload";
+ public static final String MMAP_NOCORE_LIMIT="mmap-nocore-limit";
+ private static final String CPU_SOCKET_ATTRIB_NAME = "cpu-socket";
+ public static final String CPU_SOCKET_AFFINITY_ATTRIB_NAME = "cpu-socket-affinity";
+
+ public static final Logger log = Logger.getLogger(VespaDomBuilder.class.getPackage().toString());
+
+ /**
+ * Get all aliases for one host from a list of 'alias' xml nodes.
+ *
+ * @param hostAliases List of xml nodes, each representing one hostalias
+ * @return a list of alias strings.
+ */
+ // TODO Move and change scope
+ public static List<String> getHostAliases(NodeList hostAliases) {
+ List<String> aliases = new LinkedList<>();
+ for (int i=0; i < hostAliases.getLength(); i++) {
+ Node n = hostAliases.item(i);
+ if (! (n instanceof Element)) {
+ continue;
+ }
+ Element e = (Element)n;
+ if (! e.getNodeName().equals("alias")) {
+ throw new RuntimeException("Unexpected tag: '" + e.getNodeName() + "' at node " +
+ XML.getNodePath(e, " > ") + ", expected 'alias'.");
+ }
+ String alias = e.getFirstChild().getNodeValue();
+ if ((alias == null) || (alias.equals(""))) {
+ throw new RuntimeException("Missing value for the alias tag at node " +
+ XML.getNodePath(e, " > ") + "'.");
+ }
+ aliases.add(alias);
+ }
+ return aliases;
+ }
+
+
+ @Override
+ public ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent) {
+ try {
+ return new DomRootBuilder(name, deployState).
+ build(parent, XmlHelper.getDocument(deployState.getApplicationPackage().getServices()).getDocumentElement());
+ } catch (Exception e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * @param spec The element containing the xml specification for this Service.
+ * @return the user's desired port, which is retrieved from the xml spec.
+ */
+ public static int getXmlWantedPort(Element spec) {
+ return getXmlIntegerAttribute(spec, "baseport");
+ }
+
+ /**
+ * Base class for builders of producers using DOM. The purpose is to always
+ * include hostalias, baseport and user config overrides generically.
+ *
+ * @param <T> an {@link com.yahoo.config.model.producer.AbstractConfigProducer}
+ * @author vegardh
+ */
+ public static abstract class DomConfigProducerBuilder<T extends AbstractConfigProducer> {
+
+ // TODO: find good way to provide access to app package
+ public final T build(AbstractConfigProducer ancestor, Element producerSpec) {
+ T t = doBuild(ancestor, producerSpec);
+
+ if (t instanceof AbstractService) {
+ initializeService((AbstractService)t, ancestor, producerSpec);
+ } else {
+ initializeProducer(t, ancestor, producerSpec);
+ }
+
+ return t;
+ }
+
+ protected abstract T doBuild(AbstractConfigProducer ancestor, Element producerSpec);
+
+ private void initializeProducer(AbstractConfigProducer child,
+ AbstractConfigProducer ancestor,
+ Element producerSpec) {
+ UserConfigRepo userConfigs = UserConfigBuilder.build(producerSpec, ancestor.getRoot().getDeployState(), ancestor.getRoot().deployLogger());
+ // TODO: must be made to work:
+ //userConfigs.applyWarnings(child);
+ log.log(LogLevel.DEBUG, "Adding user configs " + userConfigs + " for " + producerSpec);
+ child.mergeUserConfigs(userConfigs);
+ }
+
+ private void initializeService(AbstractService t,
+ AbstractConfigProducer ancestor,
+ Element producerSpec) {
+ initializeProducer(t, ancestor, producerSpec);
+ if (producerSpec != null) {
+ if (producerSpec.hasAttribute(JVMARGS_ATTRIB_NAME)) {
+ t.appendJvmArgs(producerSpec.getAttribute(JVMARGS_ATTRIB_NAME));
+ }
+ if (producerSpec.hasAttribute(PRELOAD_ATTRIB_NAME)) {
+ t.setPreLoad(producerSpec.getAttribute(PRELOAD_ATTRIB_NAME));
+ }
+ if (producerSpec.hasAttribute(MMAP_NOCORE_LIMIT)) {
+ t.setMMapNoCoreLimit(Long.parseLong(producerSpec.getAttribute(MMAP_NOCORE_LIMIT)));
+ }
+ if (producerSpec.hasAttribute(CPU_SOCKET_ATTRIB_NAME)) {
+ t.setAffinity(new Affinity.Builder().cpuSocket(Integer.parseInt(producerSpec.getAttribute(CPU_SOCKET_ATTRIB_NAME))).build());
+ }
+ int port = getXmlWantedPort(producerSpec);
+ if (port > 0) {
+ t.setBasePort(port);
+ }
+ allocateHost(t, ancestor.getHostSystem(), producerSpec);
+ }
+ // This depends on which constructor in AbstractService is used, but the best way
+ // is to let this method do initialize.
+ if (!t.isInitialized()) {
+ t.initService();
+ }
+ }
+
+ /**
+ * Allocates a host to the service using host file or create service spec for provisioner to use later
+ * Pre-condition: producerSpec is non-null
+ * @param service the service to allocate a host for
+ * @param hostSystem a {@link HostSystem}
+ * @param producerSpec xml element for the service
+ */
+ private void allocateHost(final AbstractService service, HostSystem hostSystem, Element producerSpec) {
+ // TODO store service on something else than HostSystem, to not make that overloaded
+ service.setHostResource(hostSystem.getHost(producerSpec.getAttribute("hostalias")));
+ }
+ }
+
+ /**
+ * The SimpleConfigProducer is the producer for elements such as qrservers, topleveldispatchers, gateways.
+ * Must support overrides for that too, hence this builder
+ *
+ * @author vegardh
+ */
+ public static class DomSimpleConfigProducerBuilder extends DomConfigProducerBuilder<SimpleConfigProducer> {
+ private String configId = null;
+
+ public DomSimpleConfigProducerBuilder(String configId) {
+ this.configId = configId;
+ }
+
+ @Override
+ protected SimpleConfigProducer doBuild(AbstractConfigProducer parent,
+ Element producerSpec) {
+ return new SimpleConfigProducer(parent, configId);
+ }
+ }
+
+ public static class DomRootBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ApplicationConfigProducerRoot> {
+ private final String name;
+ private final DeployState deployState;
+
+ /**
+ * @param name The name of the Vespa to create. Usually 'root' when there is only one.
+ */
+ public DomRootBuilder(String name, DeployState deployState) {
+ this.name = name;
+ this.deployState = deployState;
+ }
+
+ @Override
+ protected ApplicationConfigProducerRoot doBuild(AbstractConfigProducer parent, Element producerSpec) {
+ ApplicationConfigProducerRoot root = new ApplicationConfigProducerRoot(parent, name,
+ deployState.getDocumentModel(), deployState.getProperties().vespaVersion(), deployState.getProperties().applicationId());
+ root.setHostSystem(new HostSystem(root, "hosts", deployState.getProvisioner()));
+ new Client(root);
+ return root;
+ }
+ }
+
+ /**
+ * Gets the index from a service's spec
+ *
+ * @param spec The element containing the xml specification for this Service.
+ * @return the index of the service, which is retrieved from the xml spec.
+ */
+ static int getIndex(Element spec) {
+ return getXmlIntegerAttribute(spec, "index");
+ }
+
+ /**
+ * Gets an integer attribute value from a service's spec
+ *
+ * @param spec XML element
+ * @param attributeName nam of attribute to get value from
+ * @return value of attribute, or 0 if it does not exist or is empty
+ */
+ static int getXmlIntegerAttribute(Element spec, String attributeName) {
+ String value = (spec == null) ? null : spec.getAttribute(attributeName);
+ if (value == null || value.equals("")) {
+ return 0;
+ } else {
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException
+ ("Illegal format for attribute '" + attributeName + "' at node " +
+ XML.getNodePath(spec, " > ") + ", must be an integer", e);
+ }
+ }
+ }
+
+ /**
+ * Processing that requires access across different plugins
+ *
+ * @param root root config producer
+ * @param configModelRepo a {@link ConfigModelRepo}
+ */
+ public void postProc(AbstractConfigProducer root, ConfigModelRepo configModelRepo) {
+ createTlds(configModelRepo);
+ setContentSearchClusterIndexes(configModelRepo);
+ createDocprocMBusServersAndClients(configModelRepo);
+ }
+
+ private void createDocprocMBusServersAndClients(ConfigModelRepo pc) {
+ for (ContainerCluster cluster: ContainerModel.containerClusters(pc)) {
+ addServerAndClientsForChains(cluster.getDocproc());
+ }
+ }
+
+ private void addServerAndClientsForChains(ContainerDocproc docproc) {
+ if (docproc != null)
+ docproc.getChains().addServersAndClientsForChains();
+ }
+
+ private void createTlds(ConfigModelRepo pc) {
+ for (ConfigModel p : pc.asMap().values()) {
+ if (p instanceof Content) {
+ ((Content)p).createTlds(pc);
+ }
+ }
+ }
+
+ /**
+ * For some reason, search clusters need to be enumerated.
+ * @param configModelRepo a {@link ConfigModelRepo}
+ */
+ private void setContentSearchClusterIndexes(ConfigModelRepo configModelRepo) {
+ int index = 0;
+ for (AbstractSearchCluster sc : Content.getSearchClusters(configModelRepo)) {
+ sc.setClusterIndex(index++);
+ }
+ }
+
+ @Override
+ public List<ServiceCluster> getClusters(ApplicationPackage pkg,
+ AbstractConfigProducer parent) {
+ List<ServiceCluster> clusters = new ArrayList<>();
+ Document services = XmlHelper.getDocument(pkg.getServices());
+ for (Element clusterSpec : XML.getChildren(services.getDocumentElement(), "cluster")) {
+ DomServiceClusterBuilder clusterBuilder = new DomServiceClusterBuilder(clusterSpec.getAttribute("name"));
+ clusters.add(clusterBuilder.build(parent.getRoot(), clusterSpec));
+ }
+ return clusters;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java
new file mode 100644
index 00000000000..9dac8a23587
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import org.w3c.dom.Element;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+
+/**
+ * Creates a partial ChainSpecification without inner components
+ * @author tonytv
+ */
+public class ChainSpecificationBuilder {
+ private final ComponentId componentId;
+ private final ChainSpecification.Inheritance inheritance;
+ private final Collection<Phase> phases;
+
+ public ChainSpecificationBuilder(Element chainElement) {
+ componentId = readComponentId(chainElement);
+ inheritance = readInheritance(chainElement);
+ phases = readPhases(chainElement);
+ }
+
+ private Set<Phase> readPhases(Element parentElement) {
+ Set<Phase> phases = new LinkedHashSet<>();
+
+ for (Element phaseSpec : XML.getChildren(parentElement, "phase")) {
+ String name = XmlHelper.getIdString(phaseSpec);
+ Dependencies dependencies = new DependenciesBuilder(phaseSpec).build();
+ phases.add(new Phase(name, dependencies));
+ }
+ return phases;
+ }
+
+ private ComponentId readComponentId(Element spec) {
+ return XmlHelper.getId(spec);
+ }
+
+ private ChainSpecification.Inheritance readInheritance(Element spec) {
+ return new InheritanceBuilder(spec).build();
+ }
+
+ public ChainSpecification build(Set<ComponentSpecification> outerComponentReferences) {
+ return new ChainSpecification(componentId, inheritance, phases, outerComponentReferences);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java
new file mode 100644
index 00000000000..7414723c164
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
+import org.w3c.dom.Element;
+
+/**
+ * Builds a regular ChainedComponentModel from an element.
+ * @author tonytv
+ */
+public class ChainedComponentModelBuilder extends GenericChainedComponentModelBuilder {
+ protected final BundleInstantiationSpecification bundleInstantiationSpec;
+
+ public ChainedComponentModelBuilder(Element spec) {
+ super(spec);
+ bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec, false);
+ }
+
+ public ChainedComponentModel build() {
+ return new ChainedComponentModel(bundleInstantiationSpec, dependencies);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java
new file mode 100644
index 00000000000..9c848e3f284
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ */
+public class ChainsBuilder<COMPONENT extends ChainedComponent<?>, CHAIN extends Chain<COMPONENT>> {
+
+ protected final List<CHAIN> chains = new ArrayList<>();
+ private final Map<String, Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>>> chainType2BuilderClass;
+
+ // NOTE: The chain type string (key in chainType2BuilderClass) must match the xml tag name for the chain.
+ public ChainsBuilder(AbstractConfigProducer ancestor, List<Element> chainsElems,
+ Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName,
+ Map<String, Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>>> chainType2BuilderClass) {
+
+ this.chainType2BuilderClass = chainType2BuilderClass;
+ readChains(ancestor, chainsElems, outerComponentTypeByComponentName);
+ }
+
+ public Collection<CHAIN> getChains() {
+ return Collections.unmodifiableCollection(chains);
+ }
+
+ private void readChains(AbstractConfigProducer ancestor, List<Element> chainsElems,
+ Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+
+ for (Map.Entry<String, Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>>>
+ chainType : chainType2BuilderClass.entrySet()) {
+ for (Element elemContainingChainElems : chainsElems) {
+ for (Element chainElem : XML.getChildren(elemContainingChainElems, chainType.getKey())) {
+ readChain(ancestor, chainElem, chainType.getValue(), outerSearcherTypeByComponentName);
+ }
+ }
+ }
+ }
+
+ private void readChain(AbstractConfigProducer ancestor, Element chainElem,
+ Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>> builderClass,
+ Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+
+ DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN> builder =
+ DomBuilderCreator.create(builderClass, outerSearcherTypeByComponentName);
+ chains.add(builder.build(ancestor, chainElem));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java
new file mode 100644
index 00000000000..b7aef6b9b1f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java
@@ -0,0 +1,156 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocumentProcessorBuilder;
+import com.yahoo.vespa.model.container.http.xml.FilterBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessorBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomFederationSearcherBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearcherBuilder;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+import com.yahoo.vespa.model.container.http.Filter;
+import com.yahoo.vespa.model.container.processing.Processor;
+import com.yahoo.vespa.model.container.docproc.DocumentProcessor;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * Creates component models and component references from xml for a given scope.
+ * @author tonytv
+ */
+public class ComponentsBuilder<T extends ChainedComponent<?>> {
+
+ // NOTE: the 'name' string must match the xml tag name for the component in services.
+ public static class ComponentType<T extends ChainedComponent<?>> {
+ static ArrayList<ComponentType> values = new ArrayList<>();
+ public static final ComponentType<DocumentProcessor> documentprocessor = new ComponentType<>("documentprocessor", DomDocumentProcessorBuilder.class);
+ public static final ComponentType<Searcher<?>> searcher = new ComponentType<>("searcher", DomSearcherBuilder.class);
+ public static final ComponentType<Processor> processor = new ComponentType<>("processor", DomProcessorBuilder.class);
+ public static final ComponentType<Searcher<?>> federation = new ComponentType<>("federation", DomFederationSearcherBuilder.class);
+ public static final ComponentType<Filter> filter = new ComponentType<>("filter", FilterBuilder.class);
+
+
+ final String name;
+
+ private final Class<? extends VespaDomBuilder.DomConfigProducerBuilder<T>> builderClass;
+
+ private ComponentType(String name, Class<? extends VespaDomBuilder.DomConfigProducerBuilder<T>> builderClass) {
+ this.name = name;
+ this.builderClass = builderClass;
+ values.add(this);
+ }
+
+ public VespaDomBuilder.DomConfigProducerBuilder<T> createBuilder() {
+ return DomBuilderCreator.create(builderClass);
+ }
+ }
+
+ private final Set<ComponentSpecification> outerComponentReferences = new LinkedHashSet<>();
+ private final List<T> componentDefinitions = new ArrayList<>();
+ private final Map<String, ComponentType> componentTypesByComponentName = new LinkedHashMap<>();
+
+ /**
+ * @param ancestor The parent config producer
+ * @param componentTypes The allowed component types for 'elementContainingComponentElements' - MUST match &lt;T&gt;
+ * @param elementsContainingComponentElems All elements containing elements with name matching ComponentType.name
+ * @param outerComponentTypeByComponentName Use null if this is the outermost scope, i.e.
+ * every component is a definition, not a reference.
+ */
+ ComponentsBuilder(AbstractConfigProducer ancestor,
+ Collection<ComponentType<T>> componentTypes,
+ List<Element> elementsContainingComponentElems,
+ Map<String, ComponentType> outerComponentTypeByComponentName) {
+
+ readComponents(ancestor, componentTypes, elementsContainingComponentElems, unmodifiable(outerComponentTypeByComponentName));
+ }
+
+ private void readComponents(AbstractConfigProducer ancestor,
+ Collection<ComponentType<T>> componentTypes,
+ List<Element> elementsContainingComponentElems,
+ Map<String, ComponentType> outerComponentTypeByComponentName) {
+
+ for (ComponentType<T> componentType : componentTypes) {
+ for (Element elemContainingComponentElems : elementsContainingComponentElems) {
+ for (Element componentElement : XML.getChildren(elemContainingComponentElems, componentType.name)) {
+ readComponent(ancestor, componentElement, componentType, outerComponentTypeByComponentName);
+ }
+ }
+ }
+ }
+
+ private void readComponent(AbstractConfigProducer ancestor,
+ Element componentElement,
+ ComponentType<T> componentType,
+ Map<String, ComponentType> outerComponentTypeByComponentName) {
+
+ ComponentSpecification componentSpecification = XmlHelper.getIdRef(componentElement);
+
+ if (outerComponentTypeByComponentName.containsKey(componentSpecification.getName())) {
+ readComponentReference(componentElement, componentType, componentSpecification, outerComponentTypeByComponentName);
+ } else {
+ readComponentDefinition(ancestor, componentElement, componentType);
+ }
+ }
+
+ private void readComponentReference(Element componentElement, ComponentType componentType,
+ ComponentSpecification componentSpecification,
+ Map<String, ComponentType> outerComponentTypeByComponentName) {
+
+ String componentName = componentSpecification.getName();
+ ensureTypesMatch(componentType, outerComponentTypeByComponentName.get(componentName), componentName);
+ ensureNotDefinition(componentName, componentElement);
+ outerComponentReferences.add(componentSpecification);
+ }
+
+ private void readComponentDefinition(AbstractConfigProducer ancestor, Element componentElement, ComponentType<T> componentType) {
+ T component = componentType.createBuilder().build(ancestor, componentElement);
+ componentDefinitions.add(component);
+ updateComponentTypes(component.getComponentId(), componentType);
+ }
+
+ private void updateComponentTypes(ComponentId componentId, ComponentType componentType) {
+ ComponentType oldType = componentTypesByComponentName.put(componentId.getName(), componentType);
+ if (oldType != null) {
+ ensureTypesMatch(componentType, oldType, componentId.getName());
+ }
+ }
+
+ private void ensureNotDefinition(String componentName, Element componentSpec) {
+ if (componentSpec.getAttributes().getLength() > 1 || !XML.getChildren(componentSpec).isEmpty())
+ throw new RuntimeException("Expecting " + componentName +
+ " to be a reference to a global component with the same name," +
+ " so no additional attributes or nested elements are allowed");
+ }
+
+ private void ensureTypesMatch(ComponentType type1, ComponentType type2, String componentName) {
+ if (!type1.equals(type2)) {
+ throw new RuntimeException("Two different types declared for the component with name '" + componentName + "' ("
+ + type1.name + " != " + type2.name + ").");
+ }
+ }
+
+ private Map<String, ComponentType> unmodifiable(Map<String, ComponentType> outerComponentTypeByComponentName) {
+ return (outerComponentTypeByComponentName != null)?
+ Collections.unmodifiableMap(outerComponentTypeByComponentName):
+ Collections.<String, ComponentType>emptyMap();
+ }
+
+ public Collection<T> getComponentDefinitions() {
+ return Collections.unmodifiableCollection(componentDefinitions);
+ }
+
+ public Map<String, ComponentType> getComponentTypeByComponentName() {
+ return Collections.unmodifiableMap(componentTypesByComponentName);
+ }
+
+ public Set<ComponentSpecification> getOuterComponentReferences() {
+ return Collections.unmodifiableSet(outerComponentReferences);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java
new file mode 100644
index 00000000000..85ed36d0356
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import org.w3c.dom.Element;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Builds Dependencies (provides, before, after) from an element.
+ * @author tonytv
+ */
+public class DependenciesBuilder {
+ private final Dependencies dependencies;
+
+ public DependenciesBuilder(Element spec) {
+
+ Set<String> provides = read(spec, "provides");
+ Set<String> before = read(spec, "before");
+ Set<String> after = read(spec, "after");
+
+ dependencies = new Dependencies(provides, before, after);
+ }
+
+ public Dependencies build() {
+ return dependencies;
+ }
+
+ private Set<String> read(Element spec, String name) {
+ Set<String> symbols = new HashSet<>();
+ symbols.addAll(XmlHelper.valuesFromElements(spec, name));
+ symbols.addAll(XmlHelper.spaceSeparatedSymbolsFromAttribute(spec, name));
+
+ return symbols;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java
new file mode 100644
index 00000000000..1cf91f11ed1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+/**
+ * Utility class for instantiating a builder using reflection.
+ * @author tonytv
+ */
+public class DomBuilderCreator {
+ public static <T> T create(Class<T> builderClass, Object... parameters) {
+ try {
+ return getConstructor(builderClass).newInstance(parameters);
+ } catch (InstantiationException | InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> Constructor<T> getConstructor(Class<T> builderClass) {
+ Constructor<?>[] constructors = builderClass.getConstructors();
+ assert(constructors.length == 1);
+ return (Constructor<T>) constructors[0];
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java
new file mode 100644
index 00000000000..3233424f4ac
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Base functionality for all chain builders (docprocChain, searchChain, provider, source)
+ * @author tonytv
+ */
+public abstract class DomChainBuilderBase<COMPONENT extends ChainedComponent<?>, CHAIN extends Chain<COMPONENT>>
+ extends VespaDomBuilder.DomConfigProducerBuilder<CHAIN> {
+
+ private Collection<ComponentsBuilder.ComponentType<COMPONENT>> allowedComponentTypes;
+ protected final Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName;
+
+ public DomChainBuilderBase(Collection<ComponentsBuilder.ComponentType<COMPONENT>> allowedComponentTypes,
+ Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) {
+ this.allowedComponentTypes = allowedComponentTypes;
+ this.outerComponentTypeByComponentName = outerComponentTypeByComponentName;
+ }
+
+ public final CHAIN doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ ComponentsBuilder<COMPONENT> componentsBuilder =
+ new ComponentsBuilder<>(ancestor, allowedComponentTypes, Arrays.asList(producerSpec), outerComponentTypeByComponentName);
+ ChainSpecification specWithoutInnerComponents =
+ new ChainSpecificationBuilder(producerSpec).build(componentsBuilder.getOuterComponentReferences());
+
+ CHAIN chain = buildChain(ancestor, producerSpec, specWithoutInnerComponents);
+ addInnerComponents(chain, componentsBuilder.getComponentDefinitions());
+
+ return chain;
+ }
+
+ private void addInnerComponents(CHAIN chain, Collection<COMPONENT> componentDefinitions) {
+ for (COMPONENT innerComponent : componentDefinitions) {
+ chain.addInnerComponent(innerComponent);
+ }
+ }
+
+ protected abstract CHAIN buildChain(AbstractConfigProducer ancestor, Element producerSpec,
+ ChainSpecification specWithoutInnerComponents);
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java
new file mode 100644
index 00000000000..9555d916d2e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.config.application.Xml;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+import com.yahoo.vespa.model.container.component.chain.Chains;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * NOTE: This class _must_ be abstract, due to calling subclass method in ctor.
+ * @author tonytv
+ * @author gjoranv
+ */
+public abstract
+class DomChainsBuilder<COMPONENT extends ChainedComponent<?>, CHAIN extends Chain<COMPONENT>, CHAINS extends Chains<CHAIN>>
+ extends VespaDomBuilder.DomConfigProducerBuilder<CHAINS> {
+
+ private final Collection<ComponentType<COMPONENT>> allowedComponentTypes;
+ private final String appPkgChainsDir;
+ private final Element outerChainsElem;
+
+ protected DomChainsBuilder(Element outerChainsElem,
+ Collection<ComponentType<COMPONENT>> allowedComponentTypes,
+ String appPkgChainsDir) {
+
+ this.outerChainsElem = outerChainsElem;
+
+ // XXX: hack to prevent 'config' in the outer chains elem.
+ // TODO: Remove when this can be handled by schema, i.e. when we don't need the non-cluster qrservers syntax anymore
+ if (outerChainsElem != null) {
+ if (XML.getChildren(outerChainsElem, "config").size() > 0) {
+ throw new RuntimeException("At node " + XML.getNodePath(outerChainsElem, " > ") + ": " +
+ "'config' is not allowed in the outer chains element, please move it up one level!");
+ }
+ }
+
+ this.allowedComponentTypes = new ArrayList<>(allowedComponentTypes);
+ this.appPkgChainsDir = appPkgChainsDir;
+ }
+
+ protected abstract CHAINS newChainsInstance(AbstractConfigProducer parent);
+
+ @Override
+ protected final CHAINS doBuild(AbstractConfigProducer parent, Element chainsElement) {
+ CHAINS chains = newChainsInstance(parent);
+
+ List<Element> allChainElements = allChainElements(parent, chainsElement);
+ if (! allChainElements.isEmpty()) {
+ ComponentsBuilder<COMPONENT> outerComponentsBuilder = readOuterComponents(chains, allChainElements);
+ ChainsBuilder<COMPONENT, CHAIN> chainsBuilder = readChains(chains, allChainElements,
+ outerComponentsBuilder.getComponentTypeByComponentName());
+
+ addOuterComponents(chains, outerComponentsBuilder);
+ addChains(chains, chainsBuilder);
+ }
+ return chains;
+ }
+
+ private List<Element> allChainElements(AbstractConfigProducer ancestor, Element chainsElement) {
+ List<Element> chainsElements = new ArrayList<>();
+ if (outerChainsElem != null)
+ chainsElements.add(outerChainsElem);
+ chainsElements.add(chainsElement);
+
+ if (appPkgChainsDir != null)
+ chainsElements.addAll(Xml.allElemsFromPath(ancestor.getRoot().getDeployState().getApplicationPackage(), appPkgChainsDir));
+
+ return chainsElements;
+ }
+
+ private ComponentsBuilder<COMPONENT> readOuterComponents(AbstractConfigProducer ancestor, List<Element> chainsElems) {
+ return new ComponentsBuilder<>(ancestor, allowedComponentTypes, chainsElems, null);
+ }
+
+ protected abstract
+ ChainsBuilder<COMPONENT, CHAIN> readChains(AbstractConfigProducer ancestor, List<Element> allChainsElems,
+ Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName);
+
+ private void addOuterComponents(CHAINS chains, ComponentsBuilder<COMPONENT> outerComponentsBuilder) {
+ assert (outerComponentsBuilder.getOuterComponentReferences().isEmpty());
+
+ for (ChainedComponent outerComponent : outerComponentsBuilder.getComponentDefinitions()) {
+ chains.add(outerComponent);
+ }
+ }
+
+ private void addChains(CHAINS chains, ChainsBuilder<COMPONENT, CHAIN> chainsBuilder) {
+ for (CHAIN chain : chainsBuilder.getChains()) {
+ chains.add(chain);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java
new file mode 100644
index 00000000000..a7f5a306f8b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import org.w3c.dom.Element;
+
+/**
+ * reads the common attributes and elements of all chained component elements.
+ * @author tonytv
+ */
+public abstract class GenericChainedComponentModelBuilder {
+ //The componentId might be used as a spec later(for example as class or
+ //bundle), so we must treat it as a specification until then.
+ protected final ComponentSpecification componentId;
+ protected final Dependencies dependencies;
+
+ public GenericChainedComponentModelBuilder(Element spec) {
+ componentId = readComponentId(spec);
+ dependencies = readDependencies(spec);
+ }
+
+ private Dependencies readDependencies(Element spec) {
+ return new DependenciesBuilder(spec).build();
+ }
+
+ protected ComponentSpecification readComponentId(Element spec) {
+ return XmlHelper.getIdRef(spec);
+ }
+
+ protected abstract ChainedComponentModel build();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java
new file mode 100644
index 00000000000..e617ea93405
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.text.XML;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * Build an Inheritance object from an inheritance section.
+ * @author tonytv
+ */
+public class InheritanceBuilder {
+ final ChainSpecification.Inheritance inheritance;
+
+ public InheritanceBuilder(Element spec) {
+ inheritance = new ChainSpecification.Inheritance(
+ // XXX: for this to work, the tagname in the spec must match the tagname inside the 'inherits' elem, e.g. 'searchchain->inherits->searchchain'
+ read(spec, "inherits", spec.getTagName()),
+ read(spec, "excludes", "exclude"));
+ }
+
+ public ChainSpecification.Inheritance build() {
+ return inheritance;
+ }
+
+ private Set<ComponentSpecification> read(Element spec, String attributeName, String elementName) {
+ Set<ComponentSpecification> componentSpecifications = new LinkedHashSet<>();
+
+ componentSpecifications.addAll(spaceSeparatedComponentSpecificationsFromAttribute(spec, attributeName));
+
+ // TODO: the 'inherits' element is undocumented, and can be removed in an upcoming version of Vespa
+ componentSpecifications.addAll(idRefFromElements(XML.getChild(spec, "inherits"), elementName));
+
+
+ return componentSpecifications;
+ }
+
+ private Collection<ComponentSpecification> idRefFromElements(Element spec, String elementName) {
+ Collection<ComponentSpecification> result = new ArrayList<>();
+ if (spec == null)
+ return result;
+
+ for (Element element : XML.getChildren(spec, elementName)) {
+ result.add(XmlHelper.getIdRef(element));
+ }
+ return result;
+ }
+
+ private Collection<ComponentSpecification> spaceSeparatedComponentSpecificationsFromAttribute(Element spec, String attributeName) {
+ return toComponentSpecifications(XmlHelper.spaceSeparatedSymbolsFromAttribute(spec, attributeName));
+ }
+
+ private Set<ComponentSpecification> toComponentSpecifications(Collection<String> symbols) {
+ Set<ComponentSpecification> specifications = new LinkedHashSet<>();
+ for (String symbol : symbols) {
+ specifications.add(new ComponentSpecification(symbol));
+ }
+ return specifications;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java
new file mode 100644
index 00000000000..8aabea965eb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.docproc;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+import com.yahoo.vespa.model.container.docproc.DocumentProcessor;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * Creates all docproc chains from xml.
+ *
+ * @author gjoranv
+ */
+public class DocprocChainsBuilder extends ChainsBuilder<DocumentProcessor, DocprocChain> {
+
+ private static final Map<String, Class<? extends DomChainBuilderBase<? extends DocumentProcessor, ? extends DocprocChain>>>
+ chainType2builderClass = Collections.unmodifiableMap(
+ new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends DocumentProcessor, ? extends DocprocChain>>>() {{
+ put("docprocchain", DomDocprocChainBuilder.class);
+ put("chain", DomDocprocChainBuilder.class);
+ }});
+
+ public DocprocChainsBuilder(AbstractConfigProducer ancestor, List<Element> docprocChainsElements,
+ Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(ancestor, docprocChainsElements, outerSearcherTypeByComponentName, chainType2builderClass);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java
new file mode 100644
index 00000000000..e2c5afb3c86
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.docproc;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder;
+import com.yahoo.vespa.model.container.docproc.model.DocumentProcessorModel;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentProcessorModelBuilder extends ChainedComponentModelBuilder {
+
+ private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>();
+
+ public DocumentProcessorModelBuilder(Element spec) {
+ super(spec);
+ readFieldNameSchemaMap(spec);
+ }
+
+ private void readFieldNameSchemaMap(Element spec) {
+ fieldNameSchemaMap = parseFieldNameSchemaMap(spec);
+ }
+
+ @Override
+ public DocumentProcessorModel build() {
+ return new DocumentProcessorModel(bundleInstantiationSpec, dependencies, fieldNameSchemaMap);
+ }
+
+ /**
+ * Parses a schemamapping element and generates a map of field mappings
+ *
+ * @param e a schemamapping element
+ * @return doctype,in-document → in-processor
+ */
+ public static Map<Pair<String,String>, String> parseFieldNameSchemaMap(Element e) {
+ Map<Pair<String, String>, String> ret = new HashMap<>();
+ for (Element sm : XML.getChildren(e, "map")) {
+ for (Element fm : XML.getChildren(sm, "field")) {
+ String from = fm.getAttribute("in-document");
+ String to = fm.getAttribute("in-processor");
+ String doctype = fm.getAttribute("doctype");
+ if ("".equals(doctype)) doctype=null;
+ ret.put(new Pair<>(doctype, from), to);
+ }
+ }
+ return ret;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java
new file mode 100644
index 00000000000..476275ca290
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.docproc;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+import com.yahoo.vespa.model.container.docproc.DocumentProcessor;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Builds a docproc chain from xml
+ *
+ * @author gjoranv
+ */
+public class DomDocprocChainBuilder extends DomChainBuilderBase<DocumentProcessor, DocprocChain> {
+
+ public DomDocprocChainBuilder(Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) {
+ super(Arrays.asList(ComponentsBuilder.ComponentType.documentprocessor), outerComponentTypeByComponentName);
+ }
+
+ protected DocprocChain buildChain(AbstractConfigProducer ancestor, Element producerSpec,
+ ChainSpecification specWithoutInnerComponents) {
+ Map<Pair<String, String>, String> fieldNameSchemaMap = DocumentProcessorModelBuilder.parseFieldNameSchemaMap(producerSpec);
+ return new DocprocChain(specWithoutInnerComponents, fieldNameSchemaMap);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java
new file mode 100644
index 00000000000..07dc6e337fd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.docproc;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder;
+import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+import com.yahoo.vespa.model.container.docproc.DocumentProcessor;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Builds the docproc chains model from xml.
+ *
+ * @author gjoranv
+ */
+public class DomDocprocChainsBuilder extends DomChainsBuilder<DocumentProcessor, DocprocChain, DocprocChains> {
+ public DomDocprocChainsBuilder(Element outerChainsElem, boolean supportDocprocChainsDir) {
+ super(outerChainsElem, Arrays.asList(ComponentType.documentprocessor),
+ supportDocprocChainsDir ? ApplicationPackage.DOCPROCCHAINS_DIR: null);
+ }
+
+ @Override
+ protected DocprocChains newChainsInstance(AbstractConfigProducer parent) {
+ return new DocprocChains(parent, "docprocchains");
+ }
+
+ @Override
+ protected DocprocChainsBuilder readChains(AbstractConfigProducer ancestor, List<Element> docprocChainsElements,
+ Map<String, ComponentType> outerComponentTypeByComponentName) {
+ return new DocprocChainsBuilder(ancestor, docprocChainsElements, outerComponentTypeByComponentName);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java
new file mode 100644
index 00000000000..43b652ccff5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.docproc;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.docproc.DocumentProcessor;
+import org.w3c.dom.Element;
+
+/**
+ * Builds a DocumentProcessor from XML.
+ *
+ * @author gjoranv
+ */
+public class DomDocumentProcessorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<DocumentProcessor> {
+
+ protected DocumentProcessor doBuild(AbstractConfigProducer ancestor, Element documentProcessorElement) {
+ DocumentProcessorModelBuilder modelBuilder = new DocumentProcessorModelBuilder(documentProcessorElement);
+ return new DocumentProcessor(modelBuilder.build());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java
new file mode 100644
index 00000000000..4533b078b28
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.processing;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder;
+import com.yahoo.vespa.model.container.processing.ProcessingChain;
+import com.yahoo.vespa.model.container.processing.ProcessingChains;
+import com.yahoo.vespa.model.container.processing.Processor;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Root builder of the processing model
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class DomProcessingBuilder extends DomChainsBuilder<Processor, ProcessingChain, ProcessingChains> {
+
+ public DomProcessingBuilder(Element outerChainsElem) {
+ super(outerChainsElem, Arrays.asList(ComponentsBuilder.ComponentType.processor), ApplicationPackage.PROCESSORCHAINS_DIR);
+ }
+
+ @Override
+ protected ProcessingChains newChainsInstance(AbstractConfigProducer parent) {
+ return new ProcessingChains(parent, "processing");
+ }
+
+ @Override
+ protected ProcessingChainsBuilder readChains(AbstractConfigProducer ancestor, List<Element> processingChainsElements,
+ Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) {
+ return new ProcessingChainsBuilder(ancestor, processingChainsElements, outerComponentTypeByComponentName);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java
new file mode 100644
index 00000000000..a3cbff2910f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.processing;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.processing.ProcessingChain;
+import com.yahoo.vespa.model.container.processing.Processor;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class DomProcessingChainBuilder extends DomChainBuilderBase<Processor, ProcessingChain> {
+
+ public DomProcessingChainBuilder(Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) {
+ super(Arrays.asList(ComponentsBuilder.ComponentType.processor), outerComponentTypeByComponentName);
+ }
+
+ protected ProcessingChain buildChain(AbstractConfigProducer ancestor, Element producerSpec,
+ ChainSpecification specWithoutInnerComponents) {
+ return new ProcessingChain(specWithoutInnerComponents);
+ }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java
new file mode 100644
index 00000000000..b48d3dcab2f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.processing;
+
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder;
+import com.yahoo.vespa.model.container.processing.Processor;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import org.w3c.dom.Element;
+
+/**
+ * Builds a processor from XML.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class DomProcessorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Processor> {
+
+ protected Processor doBuild(AbstractConfigProducer ancestor, Element processorElement) {
+ ChainedComponentModelBuilder modelBuilder = new ChainedComponentModelBuilder(processorElement);
+ return new Processor(modelBuilder.build());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java
new file mode 100644
index 00000000000..5806de9343e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.processing;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.processing.ProcessingChain;
+import com.yahoo.vespa.model.container.processing.Processor;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * Creates all processing chains from xml.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class ProcessingChainsBuilder extends ChainsBuilder<Processor, ProcessingChain> {
+
+ private static final Map<String, Class<? extends DomChainBuilderBase<? extends Processor, ? extends ProcessingChain>>>
+ chainType2builderClass = Collections.unmodifiableMap(
+ new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Processor, ? extends ProcessingChain>>>() {{
+ put("chain", DomProcessingChainBuilder.class);
+ }});
+
+ public ProcessingChainsBuilder(AbstractConfigProducer ancestor, List<Element> processingChainsElements,
+ Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(ancestor, processingChainsElements, outerSearcherTypeByComponentName, chainType2builderClass);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java
new file mode 100644
index 00000000000..c06d62b7d43
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.search.searchchain.model.federation.FederationSearcherModel;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.GenericChainedComponentModelBuilder;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.search.searchchain.FederationSearcher;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+import scala.Option;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Builds a federation searcher config producer from an element.
+ * @author tonytv
+ */
+public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Searcher<?>> {
+ static class FederationSearcherModelBuilder extends GenericChainedComponentModelBuilder {
+ private final List<FederationSearcherModel.TargetSpec> sources;
+ private final boolean inheritDefaultSources;
+
+ FederationSearcherModelBuilder(Element searcherSpec) {
+ super(searcherSpec);
+ sources = readSources(searcherSpec);
+ inheritDefaultSources = readSourceSet(searcherSpec);
+ }
+
+ private boolean readSourceSet(Element searcherSpec) {
+ return XML.getChild(searcherSpec, "source-set") != null;
+ }
+
+
+ private List<FederationSearcherModel.TargetSpec> readSources(Element searcherSpec) {
+ List<FederationSearcherModel.TargetSpec> sources = new ArrayList<>();
+ for (Element source : XML.getChildren(searcherSpec, "source")) {
+ sources.add(readSource(source));
+ }
+ return sources;
+ }
+
+ private FederationSearcherModel.TargetSpec readSource(Element source) {
+ ComponentSpecification componentSpecification = XmlHelper.getIdRef(source);
+
+ FederationOptions federationOptions =
+ readFederationOptions(XML.getChild(source, FederationOptionsBuilder.federationOptionsElement));
+
+ return new FederationSearcherModel.TargetSpec(componentSpecification, federationOptions);
+ }
+
+ private FederationOptions readFederationOptions(Element federationOptionsElement) {
+ if (federationOptionsElement == null) {
+ return new FederationOptions();
+ } else {
+ return new FederationOptionsBuilder(federationOptionsElement).build();
+ }
+ }
+
+ protected FederationSearcherModel build() {
+ return new FederationSearcherModel(componentId, dependencies, sources, inheritDefaultSources);
+ }
+ }
+
+ protected FederationSearcher doBuild(AbstractConfigProducer ancestor, Element searcherElement) {
+ FederationSearcherModel model = new FederationSearcherModelBuilder(searcherElement).build();
+ Optional<Component> targetSelector = buildTargetSelector(ancestor, searcherElement, model.getComponentId());
+
+ return new FederationSearcher(model, targetSelector);
+ }
+
+ private Optional<Component> buildTargetSelector(AbstractConfigProducer ancestor, Element searcherElement, ComponentId namespace) {
+ Element targetSelectorElement = XML.getChild(searcherElement, "target-selector");
+ if (targetSelectorElement == null)
+ return Optional.empty();
+
+ return Optional.of(new DomComponentBuilder(namespace).build(ancestor, targetSelectorElement));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java
new file mode 100644
index 00000000000..59ecbe99b40
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.search.searchchain.GenericTarget;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ * Base functionality for all target chain builders (provider, source)
+ */
+abstract public class DomGenericTargetBuilder<T extends GenericTarget> extends DomChainBuilderBase<Searcher<?>, T> {
+
+ DomGenericTargetBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation),
+ outerSearcherTypeByComponentName);
+ }
+
+ protected static FederationOptions readFederationOptions(Element sourceElement) {
+ Element optionsElement = XML.getChild(sourceElement, FederationOptionsBuilder.federationOptionsElement);
+ if (optionsElement == null) {
+ return new FederationOptions();
+ } else {
+ return new FederationOptionsBuilder(optionsElement).build();
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java
new file mode 100644
index 00000000000..bfd7dbc3a3c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java
@@ -0,0 +1,283 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.binaryprefix.BinaryPrefix;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.search.searchchain.model.federation.HttpProviderSpec;
+import com.yahoo.search.searchchain.model.federation.LocalProviderSpec;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.BinaryScaledAmountParser;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.container.search.searchchain.HttpProvider;
+import com.yahoo.vespa.model.container.search.searchchain.HttpProviderSearcher;
+import com.yahoo.vespa.model.container.search.searchchain.LocalProvider;
+import com.yahoo.vespa.model.container.search.searchchain.Provider;
+import com.yahoo.vespa.model.container.search.searchchain.Source;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Builds a provider from xml.
+ * The demangling of provider types is taken care of here,
+ * since the mangling is an intrinsic of the configuration language,
+ * not the model itself.
+ *
+ * @author tonytv
+ */
+public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> {
+ /**
+ * Retrieves all possible provider specific parameters
+ */
+ private static class ProviderReader {
+ final String type;
+ final String path;
+ final Double cacheWeight;
+ final Integer retries;
+ final Double readTimeout;
+ final Double connectionTimeout;
+ final Double connectionPoolTimeout;
+ final String clusterName;
+ final List<HttpProviderSpec.Node> nodes;
+ final String ycaApplicationId;
+ final Integer ycaTtl;
+ final Integer ycaRetryWait;
+ final HttpProviderSpec.Node ycaProxy; // Just re-using the Node class, as it matches our needs
+ final Integer cacheSizeMB;
+
+ ProviderReader(Element providerElement) {
+ type = readType(providerElement);
+ path = readPath(providerElement);
+ cacheWeight = readCacheWeight(providerElement);
+ cacheSizeMB = readCacheSize(providerElement);
+ clusterName = readCluster(providerElement);
+ readTimeout = readReadTimeout(providerElement);
+ connectionTimeout = readConnectionTimeout(providerElement);
+ connectionPoolTimeout = readConnectionPoolTimeout(providerElement);
+ retries = readRetries(providerElement);
+ nodes = readNodes(providerElement);
+ ycaApplicationId = readYcaApplicationId(providerElement);
+ ycaTtl = readYcaTtl(providerElement);
+ ycaRetryWait = readYcaRetryWait(providerElement);
+ ycaProxy = readYcaProxy(providerElement);
+ }
+
+
+ private String getAttributeOrNull(Element element, String name) {
+ String value = element.getAttribute(name);
+ return value.isEmpty() ? null : value;
+ }
+
+ private String readPath(Element providerElement) {
+ return getAttributeOrNull(providerElement, "path");
+ }
+
+ private String readCluster(Element providerElement) {
+ return getAttributeOrNull(providerElement, "cluster");
+ }
+
+ private Double readCacheWeight(Element providerElement) {
+ String cacheWeightString = getAttributeOrNull(providerElement, "cacheweight");
+ return (cacheWeightString == null)? null : Double.parseDouble(cacheWeightString);
+ }
+
+ private Integer readCacheSize(Element providerElement) {
+ String cacheSize = getAttributeOrNull(providerElement, "cachesize");
+ return (cacheSize == null)? null : (int)BinaryScaledAmountParser.parse(cacheSize).as(BinaryPrefix.mega);
+ }
+
+ private Integer readRetries(Element providerElement) {
+ String retriesString = getAttributeOrNull(providerElement, "retries");
+ return (retriesString == null) ? null : Integer.parseInt(retriesString);
+ }
+
+ private Double readReadTimeout(Element providerElement) {
+ String timeoutString = getAttributeOrNull(providerElement, "readtimeout");
+ return (timeoutString == null) ? null : TimeParser.seconds(timeoutString);
+ }
+
+ private Double readConnectionTimeout(Element providerElement) {
+ String timeoutString = getAttributeOrNull(providerElement, "connectiontimeout");
+ return (timeoutString == null) ? null : TimeParser.seconds(timeoutString);
+ }
+
+ private Double readConnectionPoolTimeout(Element providerElement) {
+ String timeoutString = getAttributeOrNull(providerElement, "connectionpooltimeout");
+ return (timeoutString == null) ? null : TimeParser.seconds(timeoutString);
+ }
+
+ private String readYcaApplicationId(Element providerElement) {
+ return getAttributeOrNull(providerElement, "yca-application-id");
+ }
+
+ private Integer readYcaTtl(Element providerElement) {
+ String x = getAttributeOrNull(providerElement, "yca-cache-ttl");
+ return (x == null) ? null : TimeParser.seconds(x).intValue();
+ }
+
+ private Integer readYcaRetryWait(Element providerElement) {
+ String x = getAttributeOrNull(providerElement, "yca-cache-retry-wait");
+ return (x == null) ? null : TimeParser.seconds(x).intValue();
+ }
+
+ private HttpProviderSpec.Node readYcaProxy(Element providerElement) {
+ Element ycaProxySpec = XML.getChild(providerElement, "yca-proxy");
+ if (ycaProxySpec == null) {
+ return null; // no proxy
+ }
+ if(getAttributeOrNull(ycaProxySpec, "host") == null) {
+ return new HttpProviderSpec.Node(null, 0); // default proxy
+ }
+ return readNode(ycaProxySpec);
+ }
+
+ private List<HttpProviderSpec.Node> readNodes(Element providerElement) {
+ Element nodesSpec = XML.getChild(providerElement, "nodes");
+ if (nodesSpec == null) {
+ return null;
+ }
+
+ List<HttpProviderSpec.Node> nodes = new ArrayList<>();
+ for (Element nodeSpec : XML.getChildren(nodesSpec, "node")) {
+ nodes.add(readNode(nodeSpec));
+ }
+ return nodes;
+ }
+
+ private HttpProviderSpec.Node readNode(Element nodeElement) {
+ String host = getAttributeOrNull(nodeElement, "host");
+ // The direct calls to parse methods below works because the schema
+ // guarantees us no null references
+ int port = Integer.parseInt(getAttributeOrNull(nodeElement, "port"));
+ return new HttpProviderSpec.Node(host, port);
+ }
+
+ private String readType(Element providerElement) {
+ return getAttributeOrNull(providerElement, "type");
+ }
+ }
+
+ public DomProviderBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(outerSearcherTypeByComponentName);
+ }
+
+ @Override
+ protected Provider buildChain(AbstractConfigProducer ancestor, Element providerElement,
+ ChainSpecification specWithoutInnerComponents) {
+
+ ProviderReader providerReader = new ProviderReader(providerElement);
+ if (providerReader.ycaApplicationId == null && providerReader.ycaProxy != null) {
+ throw new IllegalArgumentException(
+ "Provider '" + specWithoutInnerComponents.componentId +
+ "' must have a YCA application ID, since a YCA proxy is given");
+ }
+
+ FederationOptions federationOptions = readFederationOptions(providerElement);
+
+ Provider provider = buildProvider(specWithoutInnerComponents, providerReader, federationOptions);
+
+ Collection<Source> sources = buildSources(ancestor, providerElement);
+ addSources(provider, sources);
+
+ return provider;
+ }
+
+
+ private Collection<Source> buildSources(AbstractConfigProducer ancestor, Element providerElement) {
+ List<Source> sources = new ArrayList<>();
+ for (Element sourceElement : XML.getChildren(providerElement, "source")) {
+ sources.add(new DomSourceBuilder(outerComponentTypeByComponentName).build(ancestor, sourceElement));
+ }
+ return sources;
+ }
+
+ private void addSources(Provider provider, Collection<Source> sources) {
+ for (Source source : sources) {
+ provider.addSource(source);
+ }
+ }
+
+ private Provider buildProvider(ChainSpecification specWithoutInnerSearchers,
+ ProviderReader providerReader, FederationOptions federationOptions) {
+
+ if (providerReader.type == null) {
+ return buildEmptyHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions);
+ } else if (HttpProviderSpec.includesType(providerReader.type)) {
+ return buildHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions);
+ } else if (LocalProviderSpec.includesType(providerReader.type)) {
+ return buildLocalProvider(specWithoutInnerSearchers, providerReader, federationOptions);
+ } else {
+ throw new RuntimeException("Unknown provider type '" + providerReader.type + "'");
+ }
+ }
+
+ private Provider buildLocalProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) {
+ try {
+ ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.cacheWeight, providerReader.path, providerReader.nodes,
+ providerReader.readTimeout, providerReader.connectionTimeout, providerReader.connectionPoolTimeout,
+ providerReader.retries, providerReader.ycaApplicationId, providerReader.ycaTtl,
+ providerReader.ycaRetryWait, providerReader.ycaProxy);
+
+ return new LocalProvider(specWithoutInnerSearchers,
+ federationOptions,
+ new LocalProviderSpec(providerReader.clusterName, providerReader.cacheSizeMB));
+ } catch (Exception e) {
+ throw new RuntimeException("Failed creating local provider " + specWithoutInnerSearchers.componentId, e);
+ }
+ }
+
+ private Provider buildHttpProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) {
+ ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.clusterName);
+
+ Provider httpProvider = buildEmptyHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions);
+
+ httpProvider.addInnerComponent(new HttpProviderSearcher(
+ new ChainedComponentModel(
+ HttpProviderSpec.toBundleInstantiationSpecification(HttpProviderSpec.Type.valueOf(providerReader.type)),
+ Dependencies.emptyDependencies())));
+
+ return httpProvider;
+ }
+
+
+ private Provider buildEmptyHttpProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) {
+ ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.clusterName);
+
+ return new HttpProvider(specWithoutInnerSearchers,
+ federationOptions,
+ new HttpProviderSpec(
+ providerReader.cacheWeight,
+ providerReader.path,
+ providerReader.nodes,
+ providerReader.ycaApplicationId,
+ providerReader.ycaTtl,
+ providerReader.ycaRetryWait,
+ providerReader.ycaProxy,
+ providerReader.cacheSizeMB,
+ connectionParameters(providerReader)));
+ }
+
+ private HttpProviderSpec.ConnectionParameters connectionParameters(ProviderReader providerReader) {
+ return new HttpProviderSpec.ConnectionParameters(
+ providerReader.readTimeout,
+ providerReader.connectionTimeout,
+ providerReader.connectionPoolTimeout,
+ providerReader.retries);
+ }
+
+ private void ensureEmpty(ComponentId componentId, Object... objects) {
+ for (Object object : objects) {
+ if (object != null) {
+ throw new RuntimeException("Invalid provider option in provider '" + componentId + "': value='" + object + "'");
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java
new file mode 100644
index 00000000000..d79ea1a536b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChain;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Builds a Search chain from xml.
+ * @author tonytv
+ */
+public class DomSearchChainBuilder extends DomChainBuilderBase<Searcher<?>, SearchChain> {
+
+ public DomSearchChainBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation),
+ outerSearcherTypeByComponentName);
+ }
+
+ protected SearchChain buildChain(AbstractConfigProducer ancestor, Element producerSpec,
+ ChainSpecification specWithoutInnerComponents) {
+ return new SearchChain(specWithoutInnerComponents);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java
new file mode 100644
index 00000000000..5cadc8cc271
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChain;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Builds the search chains model from xml.
+ *
+ * @author tonytv
+ */
+public class DomSearchChainsBuilder extends DomChainsBuilder<Searcher<?>, SearchChain, SearchChains> {
+
+ public DomSearchChainsBuilder(Element outerChainsElem, boolean supportSearchChainsDir) {
+ super(outerChainsElem, Arrays.asList(ComponentType.searcher, ComponentType.federation),
+ supportSearchChainsDir ? ApplicationPackage.SEARCHCHAINS_DIR: null);
+ }
+
+ // For unit testing without outer chains
+ public DomSearchChainsBuilder() {
+ this(null, false);
+ }
+
+ @Override
+ protected SearchChains newChainsInstance(AbstractConfigProducer parent) {
+ return new SearchChains(parent, "searchchains");
+ }
+
+ @Override
+ protected SearchChainsBuilder readChains(AbstractConfigProducer ancestor, List<Element> searchChainsElements,
+ Map<String, ComponentType> outerComponentTypeByComponentName) {
+ return new SearchChainsBuilder(ancestor, searchChainsElements, outerComponentTypeByComponentName);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java
new file mode 100644
index 00000000000..44f4e83b1c8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+
+/**
+ * Builds a Searcher from XML.
+ * @author tonytv
+ */
+public class DomSearcherBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Searcher<?>> {
+
+ protected Searcher<ChainedComponentModel> doBuild(AbstractConfigProducer ancestor, Element searcherElement) {
+ ChainedComponentModelBuilder modelBuilder = new ChainedComponentModelBuilder(searcherElement);
+ return new Searcher<>(modelBuilder.build());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java
new file mode 100644
index 00000000000..090ac6a6df3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.container.search.searchchain.Source;
+import org.w3c.dom.Element;
+
+import java.util.Map;
+
+/**
+ * Builds a source from xml.
+ * @author tonytv
+ */
+public class DomSourceBuilder extends DomGenericTargetBuilder<Source> {
+ DomSourceBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(outerSearcherTypeByComponentName);
+ }
+
+ protected Source buildChain(AbstractConfigProducer ancestor, Element producerSpec, ChainSpecification specWithoutInnerComponents) {
+ Source.GroupOption groupOption =
+ XmlHelper.isReference(producerSpec) ?
+ Source.GroupOption.participant :
+ Source.GroupOption.leader;
+
+ return new Source(specWithoutInnerComponents, readFederationOptions(producerSpec), groupOption);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java
new file mode 100644
index 00000000000..ba84e66cb97
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import org.w3c.dom.Element;
+
+
+/**
+ * Builds federation options from a federations options element
+ * @author tonytv
+ */
+public class FederationOptionsBuilder {
+ public static final String federationOptionsElement = "federationoptions";
+
+ private final FederationOptions federationOptions;
+
+ FederationOptionsBuilder(Element spec) {
+ federationOptions =
+ new FederationOptions().
+ setUseByDefault(readUseByDefault(spec)).
+ setOptional(readOptional(spec)).
+ setTimeoutInMilliseconds(readTimeout(spec)).
+ setRequestTimeoutInMilliseconds(readRequestTimeout(spec));
+ }
+
+
+ private Integer readTimeout(Element spec) {
+ String timeout = spec.getAttribute("timeout");
+
+ return (timeout.isEmpty())?
+ null :
+ TimeParser.asMilliSeconds(timeout);
+ }
+
+ private Integer readRequestTimeout(Element spec) {
+ String requestTimeout = spec.getAttribute("requestTimeout");
+
+ return (requestTimeout.isEmpty())?
+ null :
+ TimeParser.asMilliSeconds(requestTimeout);
+ }
+
+ private Boolean readOptional(Element spec) {
+ String optional = spec.getAttribute("optional");
+ return (optional.isEmpty()) ?
+ null :
+ Boolean.parseBoolean(optional);
+ }
+
+ private Boolean readUseByDefault(Element spec) {
+ String useByDefault = spec.getAttribute("default");
+ return (useByDefault.isEmpty()) ?
+ null :
+ Boolean.parseBoolean(useByDefault);
+ }
+
+ FederationOptions build() {
+ return federationOptions;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java
new file mode 100644
index 00000000000..9b4a8908afe
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChain;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ * Creates top level search chains(searchchain, provider) from xml.
+ */
+public class SearchChainsBuilder extends ChainsBuilder<Searcher<?>, SearchChain> {
+
+ private static final Map<String, Class<? extends DomChainBuilderBase<? extends Searcher<?>, ? extends SearchChain>>>
+ chainType2builderClass = Collections.unmodifiableMap(
+ new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Searcher<?>, ? extends SearchChain>>>() {{
+ put("chain", DomSearchChainBuilder.class);
+ put("searchchain", DomSearchChainBuilder.class);
+ put("provider", DomProviderBuilder.class);
+ }});
+
+ public SearchChainsBuilder(AbstractConfigProducer ancestor, List<Element> searchChainsElements,
+ Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) {
+ super(ancestor, searchChainsElements, outerSearcherTypeByComponentName, chainType2builderClass);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java
new file mode 100644
index 00000000000..7aa84d3a14b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.builder.xml.dom.chains.search;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Utility class for parsing timeout fields.
+ *
+ * @author tonytv
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class TimeParser {
+ private static final Pattern timeoutPattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*(m)?s");
+ private static final double milliSecondsPerSecond = 1000.0d;
+
+ public static Double seconds(String timeout) {
+ Matcher matcher = timeoutPattern.matcher(timeout);
+ if (!matcher.matches()) {
+ throw new RuntimeException("Timeout pattern not in sync with schema");
+ }
+
+ double value = Double.parseDouble(matcher.group(1));
+ if (matcher.group(3) != null) {
+ value /= milliSecondsPerSecond;
+ }
+ return new Double(value);
+ }
+
+ public static int asMilliSeconds(String timeout) {
+ Matcher matcher = timeoutPattern.matcher(timeout);
+ if (!matcher.matches()) {
+ throw new RuntimeException("Timeout pattern not in sync with schema");
+ }
+
+ double value = Double.parseDouble(matcher.group(1));
+ if (matcher.group(3) == null) {
+ value *= milliSecondsPerSecond;
+ }
+
+ return (int) value;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java
new file mode 100644
index 00000000000..e51d7e4acee
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.builder.xml.dom;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java
new file mode 100644
index 00000000000..799d0b437ae
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.search.ContainerHttpGateway;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This is the clients plugin for the Vespa model. It is responsible for creating
+ * all clients services.
+ *
+ * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ */
+public class Clients extends ConfigModel {
+
+ private static final long serialVersionUID = 1L;
+ private ContainerCluster containerHttpGateways = null;
+ private List<VespaSpoolerService> vespaSpoolers = new LinkedList<>();
+ private LoadTypeSet loadTypes = new LoadTypeSet();
+ private final AbstractConfigProducer parent;
+
+ public Clients(ConfigModelContext modelContext) {
+ super(modelContext);
+ this.parent = modelContext.getParentProducer();
+ }
+
+ public AbstractConfigProducer getConfigProducer() {
+ return parent;
+ }
+
+ @Override
+ public void prepare(ConfigModelRepo configModelRepo) {
+ if (containerHttpGateways != null) {
+ containerHttpGateways.prepare();
+ }
+ }
+
+ public void setContainerHttpGateways(ContainerCluster containerHttpGateways) {
+ this.containerHttpGateways = containerHttpGateways;
+ }
+
+ public List<VespaSpoolerService> getVespaSpoolers() {
+ return vespaSpoolers;
+ }
+
+ public LoadTypeSet getLoadTypes() {
+ return loadTypes;
+ }
+
+ public void getConfig(LoadTypeConfig.Builder builder) {
+ for (LoadType t : loadTypes.getNameMap().values()) {
+ if (t != LoadType.DEFAULT) {
+ builder.type(getLoadTypeConfig(t));
+ }
+ }
+ }
+
+ private LoadTypeConfig.Type.Builder getLoadTypeConfig(LoadType loadType) {
+ LoadTypeConfig.Type.Builder builder = new LoadTypeConfig.Type.Builder();
+ builder.name(loadType.getName());
+ builder.id(loadType.getId());
+ builder.priority(loadType.getPriority().toString());
+ return builder;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
new file mode 100644
index 00000000000..55cfc8b2fba
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java
@@ -0,0 +1,165 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+import com.yahoo.vespa.model.container.search.ContainerSearch;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChain;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import com.yahoo.vespa.model.container.search.searchchain.Searcher;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.11
+ */
+public class ContainerDocumentApi implements FeederConfig.Producer {
+
+ public static final String vespaClientBundleSpecification = "vespaclient-container-plugin";
+ private final Options options;
+
+ public ContainerDocumentApi(ContainerCluster cluster, Options options) {
+ this.options = options;
+ setupLegacySearchers(cluster);
+ setupHandlers(cluster);
+ }
+
+ private void setupHandlers(ContainerCluster cluster) {
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "feed"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "get"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "document/v1/*"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "document"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerStatus", "feedstatus"));
+ cluster.addComponent(newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"));
+ }
+
+ private void setupLegacySearchers(ContainerCluster cluster) {
+ Set<ComponentSpecification> inherited = new TreeSet<>();
+
+ SearchChain vespaGetChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"),
+ new ChainSpecification.Inheritance(inherited, null), new ArrayList<Phase>(), new TreeSet<ComponentSpecification>()));
+ vespaGetChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher"));
+
+ SearchChain vespaVisitChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"),
+ new ChainSpecification.Inheritance(inherited, null), new ArrayList<Phase>(), new TreeSet<ComponentSpecification>()));
+ vespaVisitChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher"));
+
+ SearchChains chains;
+ if (cluster.getSearch() != null) {
+ chains = cluster.getSearchChains();
+ } else {
+ chains = new SearchChains(cluster, "searchchain");
+ }
+ chains.add(vespaGetChain);
+ chains.add(vespaVisitChain);
+
+ if (cluster.getSearch() == null) {
+ ContainerSearch containerSearch = new ContainerSearch(cluster, chains, new ContainerSearch.Options());
+ cluster.setSearch(containerSearch);
+
+ final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(
+ chains, "com.yahoo.search.handler.SearchHandler");
+ searchHandler.addServerBindings("http://*/search/*", "https://*/search/*");
+ cluster.addComponent(searchHandler);
+ }
+ }
+
+ private Handler newVespaClientHandler(String componentId, String bindingSuffix) {
+ Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel(
+ BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), ""));
+
+ for (String rootBinding : options.bindings) {
+ handler.addServerBindings(rootBinding + bindingSuffix,
+ rootBinding + bindingSuffix + '/');
+ }
+ return handler;
+ }
+
+ private Searcher newVespaClientSearcher(String componentSpec) {
+ return new Searcher<>(new ChainedComponentModel(
+ BundleInstantiationSpecification.getFromStrings(componentSpec, null, vespaClientBundleSpecification),
+ new Dependencies(null, null, null)));
+ }
+
+ @Override
+ public void getConfig(FeederConfig.Builder builder) {
+ if (options.abortondocumenterror != null)
+ builder.abortondocumenterror(options.abortondocumenterror);
+ if (options.route!= null)
+ builder.route(options.route);
+ if (options.maxpendingdocs != null)
+ builder.maxpendingdocs(options.maxpendingdocs);
+ if (options.maxpendingbytes != null)
+ builder.maxpendingbytes(options.maxpendingbytes);
+ if (options.retryenabled != null)
+ builder.retryenabled(options.retryenabled);
+ if (options.retrydelay != null)
+ builder.retrydelay(options.retrydelay);
+ if (options.timeout != null)
+ builder.timeout(options.timeout);
+ if (options.tracelevel != null)
+ builder.tracelevel(options.tracelevel);
+ if (options.mbusport != null)
+ builder.mbusport(options.mbusport);
+ if (options.docprocChain != null)
+ builder.docprocchain(options.docprocChain);
+ }
+
+ public static final class Options {
+ private final Collection<String> bindings;
+ private final Boolean abortondocumenterror;
+ private final String route;
+ private final Integer maxpendingdocs;
+ private final Integer maxpendingbytes;
+ private final Boolean retryenabled;
+ private final Double retrydelay;
+ private final Double timeout;
+ private final Integer tracelevel;
+ private final Integer mbusport;
+ private final String docprocChain;
+
+ public Options(Collection<String> bindings,
+ Boolean abortondocumenterror,
+ String route,
+ Integer maxpendingdocs,
+ Integer maxpendingbytes,
+ Boolean retryenabled,
+ Double retrydelay,
+ Double timeout,
+ Integer tracelevel,
+ Integer mbusport,
+ String docprocChain) {
+
+ this.bindings = Collections.unmodifiableCollection(bindings);
+ this.abortondocumenterror = abortondocumenterror;
+ this.route = route;
+ this.maxpendingdocs = maxpendingdocs;
+ this.maxpendingbytes = maxpendingbytes;
+ this.retryenabled = retryenabled;
+ this.retrydelay = retrydelay;
+ this.timeout = timeout;
+ this.tracelevel = tracelevel;
+ this.mbusport = mbusport;
+ this.docprocChain = docprocChain;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java
new file mode 100644
index 00000000000..b5118e6c6c7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+public class HttpGatewayOwner extends AbstractConfigProducer implements FeederConfig.Producer {
+ private final FeederConfig.Builder feederConfig;
+
+ public HttpGatewayOwner(AbstractConfigProducer parent, FeederConfig.Builder feederConfig) {
+ super(parent, "gateways");
+ this.feederConfig = feederConfig;
+ }
+
+ @Override
+ public void getConfig(FeederConfig.Builder builder) {
+ ConfigInstanceUtil.setValues(builder, feederConfig);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java
new file mode 100644
index 00000000000..1546b2d8433
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+
+/**
+ * The spoolmaster program, which is used when multiple spooler instances are used to provide
+ * multi colo HTTP feeding.
+ * @author vegardh
+ *
+ */
+public class VespaSpoolMaster extends AbstractService {
+
+ public VespaSpoolMaster(AbstractConfigProducer parent, int index) {
+ super(parent, "spoolmaster."+index);
+ }
+
+ @Override
+ public int getPortCount() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public String getStartupCommand() {
+ return "exec spoolmaster";
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java
new file mode 100644
index 00000000000..7a4ec77b2da
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.vespa.config.content.spooler.SpoolerConfig;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+/**
+ * Holds configuration for VespaSpoolers. Actual services use VespaSpoolerService,
+ * while virtual services can be generated for external spoolers (VespaSpoolerProducer).
+ *
+ * @author <a href="mailto:thomasg@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ * @author Vidar Larsen
+ */
+public class VespaSpooler {
+ private final SpoolerConfig.Builder spoolConfig;
+ private final FeederConfig.Builder feederConfig;
+
+ public VespaSpooler(FeederConfig.Builder feederConfig, SpoolerConfig.Builder spoolConfig) {
+ this.feederConfig = feederConfig;
+ this.spoolConfig = spoolConfig;
+ }
+
+ public void getConfig(SpoolerConfig.Builder builder) {
+ ConfigInstanceUtil.setValues(builder, spoolConfig);
+ }
+
+ public void getConfig(FeederConfig.Builder builder) {
+ ConfigInstanceUtil.setValues(builder, feederConfig);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java
new file mode 100644
index 00000000000..671603553ec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.vespa.config.content.spooler.SpoolerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+/**
+ * This model represents a config producer for spooler used for feeding documents to Vespa.
+ *
+ * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ * @author Vidar Larsen
+ */
+public class VespaSpoolerProducer extends AbstractConfigProducer implements SpoolerConfig.Producer, FeederConfig.Producer {
+ private static final long serialVersionUID = 1L;
+ private VespaSpooler spoolerConfig;
+
+ public VespaSpoolerProducer(AbstractConfigProducer parent, String configId, VespaSpooler spooler) {
+ super(parent, configId);
+ spoolerConfig = spooler;
+ }
+
+ @Override
+ public void getConfig(SpoolerConfig.Builder builder) {
+ spoolerConfig.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(FeederConfig.Builder builder) {
+ spoolerConfig.getConfig(builder);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java
new file mode 100644
index 00000000000..8e62b797441
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.clients;
+
+import com.yahoo.vespa.config.content.spooler.SpoolerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+/**
+ * This model represents a spooler used for feeding documents to Vespa.
+ *
+ * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ * @author Vidar Larsen
+ */
+public class VespaSpoolerService extends AbstractService implements SpoolerConfig.Producer, FeederConfig.Producer {
+ private static final long serialVersionUID = 1L;
+ private VespaSpooler spooler;
+
+ public VespaSpoolerService(AbstractConfigProducer parent, int index, VespaSpooler spooler) {
+ super(parent, "spooler." + index);
+ this.spooler = spooler;
+ monitorService("spooler");
+ }
+
+ public int getPortCount() {
+ return 0;
+ }
+
+ public String getStartupCommand() {
+ return "exec vespaspooler "+getJvmArgs();
+ }
+
+ @Override
+ public void getConfig(SpoolerConfig.Builder builder) {
+ spooler.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(FeederConfig.Builder builder) {
+ spooler.getConfig(builder);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
new file mode 100644
index 00000000000..a13c7c9cec4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -0,0 +1,408 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.ComponentsConfig;
+import com.yahoo.container.QrConfig;
+import com.yahoo.container.core.ContainerHttpConfig;
+import com.yahoo.container.jdisc.ContainerMbusConfig;
+import com.yahoo.container.jdisc.JdiscBindingsConfig;
+import com.yahoo.container.jdisc.config.PortOverridesConfig;
+import com.yahoo.search.config.QrStartConfig;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.ComponentGroup;
+import com.yahoo.vespa.model.container.component.ComponentsConfigGenerator;
+import com.yahoo.vespa.model.container.component.DiscBindingsConfigGenerator;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import com.yahoo.vespa.model.container.http.ConnectorFactory;
+import com.yahoo.vespa.model.container.http.Http;
+import com.yahoo.vespa.model.container.http.JettyHttpServer;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import static com.yahoo.container.QrConfig.Filedistributor;
+import static com.yahoo.container.QrConfig.Rpc;
+
+
+/**
+ * @author gjoranv
+ * @author einarmr
+ * @author tonytv
+ */
+//qr is restart because it is handled by ConfiguredApplication.start
+@RestartConfigs({QrStartConfig.class, QrConfig.class})
+public class Container extends AbstractService implements
+ QrConfig.Producer,
+ ComponentsConfig.Producer,
+ JdiscBindingsConfig.Producer,
+ ContainerHttpConfig.Producer,
+ PortOverridesConfig.Producer,
+ ContainerMbusConfig.Producer
+{
+ public static final class PortOverride {
+ public final ComponentSpecification serverId;
+ public final int port;
+
+ public PortOverride(ComponentSpecification serverId, int port) {
+ this.serverId = serverId;
+ this.port = port;
+ }
+ }
+
+ public static final int BASEPORT = Defaults.getDefaults().vespaWebServicePort();
+ public static final String SINGLENODE_CONTAINER_SERVICESPEC = "default_singlenode_container";
+
+ private final AbstractConfigProducer parent;
+ private final String name;
+ private String clusterName = null;
+ private boolean rpcServerEnabled = true;
+
+ // TODO: move these up to cluster
+ private boolean httpServerEnabled = true;
+ private boolean messageBusEnabled = true;
+
+ private final boolean retired;
+
+ private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler");
+ private final ComponentGroup<Component<?, ?>> components = new ComponentGroup(this, "components");
+
+ private final JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"));
+
+ private final List<PortOverride> portOverrides;
+
+ private final int numHttpServerPorts;
+ private final int numRpcServerPorts = 2;
+ private static String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage";
+
+ public Container(AbstractConfigProducer parent, String name) {
+ this(parent, name, Collections.<PortOverride>emptyList());
+ }
+ public Container(AbstractConfigProducer parent, String name, boolean retired) {
+ this(parent, name, retired, Collections.<PortOverride>emptyList());
+ }
+ public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides) {
+ this(parent, name, false, portOverrides);
+ }
+ public Container(AbstractConfigProducer parent, String name, boolean retired, List<PortOverride> portOverrides) {
+ super(parent, name);
+ this.name = name;
+ this.parent = parent;
+ this.portOverrides = Collections.unmodifiableList(new ArrayList<>(portOverrides));
+ this.retired = retired;
+
+ if (getHttp() == null) {
+ numHttpServerPorts = 2;
+ addChild(defaultHttpServer);
+ } else if (getHttp().getHttpServer() == null) {
+ numHttpServerPorts = 0;
+ } else {
+ numHttpServerPorts = getHttp().getHttpServer().getConnectorFactories().size();
+ }
+ addBuiltinHandlers();
+
+ addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext"));
+ addChild(new SimpleComponent("com.yahoo.container.jdisc.ContainerPortsOverride"));
+ }
+
+ /** True if this container is retired (slated for removal) */
+ public boolean isRetired() { return retired; }
+
+ public ComponentGroup<Handler<?>> getHandlers() {
+ return handlers;
+ }
+
+ public ComponentGroup getComponents() {
+ return components;
+ }
+
+ public void addComponent(Component c) {
+ components.addComponent(c);
+ }
+
+ public void addHandler(Handler h) {
+ handlers.addComponent(h);
+ }
+
+ public Http getHttp() {
+ return (parent instanceof ContainerCluster) ?
+ ((ContainerCluster) parent).getHttp():
+ null;
+ }
+
+ public JettyHttpServer getDefaultHttpServer() {
+ return defaultHttpServer;
+ }
+
+ // We cannot set bindings yet, as baseport is not initialized
+ public void addBuiltinHandlers() {
+ }
+
+ @Override
+ public void initService() {
+ // XXX: Must be called first, to set the baseport
+ super.initService();
+
+ if (getHttp() == null) {
+ initDefaultJettyConnector();
+ } else {
+ reserveHttpPortsPrepended();
+ }
+
+ tagServers();
+ monitorService();
+ }
+
+ private void tagServers() {
+ if (numHttpServerPorts > 0) {
+ portsMeta.on(0).tag("http").tag("query").tag("external").tag("state");
+ }
+
+ for (int i = 1; i < numHttpServerPorts; i++)
+ portsMeta.on(i).tag("http").tag("external");
+
+ if (rpcServerEnabled) {
+ portsMeta.on(numHttpServerPorts + 0).tag("rpc").tag("messaging");
+ portsMeta.on(numHttpServerPorts + 1).tag("rpc").tag("admin");
+ }
+ }
+
+ private void reserveHttpPortsPrepended() {
+ if (getHttp().getHttpServer() != null) {
+ for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) {
+ reservePortPrepended(getPort(connectorFactory, portOverrides));
+ }
+ }
+ }
+
+ private int getPort(ConnectorFactory connectorFactory, List<PortOverride> portOverrides) {
+ ComponentId id = ComponentId.fromString(connectorFactory.getName());
+ for (PortOverride override : portOverrides) {
+ if (override.serverId.matches(id)) {
+ return override.port;
+ }
+ }
+ return connectorFactory.getListenPort();
+ }
+
+ private void initDefaultJettyConnector() {
+ defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", getSearchPort(), null));
+ }
+
+ private boolean hasDocproc() {
+ return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null);
+ }
+
+ // TODO: hack to retain old service names, e.g. in ymon config, vespa.log etc.
+ @Override
+ public String getServiceType() {
+ if (parent instanceof ContainerCluster) {
+ ContainerCluster cluster = (ContainerCluster)parent;
+ if (cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) {
+ return "qrserver";
+ }
+ if (cluster.getSearch() == null && cluster.getDocproc() != null) {
+ return "docprocservice";
+ }
+ }
+ return super.getServiceType();
+ }
+
+ public void setClusterName(String name) {
+ this.clusterName = name;
+ }
+
+ @Override
+ public int getWantedPort() {
+ return getHttp() == null ?
+ BASEPORT:
+ 0;
+ }
+
+ /**
+ * First Qrserver or container must run on ports familiar to the user.
+ */
+ @Override
+ public boolean requiresWantedPort() {
+ return getHttp() == null;
+
+ }
+
+ public boolean requiresConsecutivePorts() {
+ return false;
+ }
+
+ /**
+ * @return the number of ports needed by the Container - those reserved manually(reservePortPrepended)
+ */
+ public int getPortCount() {
+ int httpPorts = (getHttp() != null) ? 0 : numHttpServerPorts + 2; // TODO remove +2, only here to keep irrelevant unit tests from failing.
+ int rpcPorts = (isRpcServerEnabled()) ? numRpcServerPorts : 0;
+ return httpPorts + rpcPorts;
+ }
+
+ /**
+ * @return the actual search port
+ * TODO: Remove. Use {@link #getPortsMeta()} and check tags in conjunction with {@link #getRelativePort(int)}.
+ */
+ public int getSearchPort(){
+ if (getHttp() != null)
+ throw new AssertionError("getSearchPort must not be used when http section is present.");
+
+ return getRelativePort(0);
+ }
+
+ private int getRpcPort() {
+ return isRpcServerEnabled() ?
+ getRelativePort(numHttpServerPorts + 1) :
+ 0;
+ }
+
+ private int getMessagingPort() {
+ return getRelativePort(numHttpServerPorts);
+ }
+
+ @Override
+ public int getHealthPort() {
+ final Http http = getHttp();
+ if (http != null) {
+ // TODO: allow the user to specify health port manually
+ if (http.getHttpServer() == null) {
+ return -1;
+ } else {
+ return getRelativePort(0);
+ }
+ } else {
+ return httpServerEnabled ? getSearchPort() : -1;
+ }
+ }
+
+ public String getStartupCommand() {
+ return "PRELOAD=" + getPreLoad() + " exec vespa-start-container-daemon " + getJvmArgs() + " ";
+ }
+
+ public boolean isRpcServerEnabled() {
+ return rpcServerEnabled;
+ }
+
+ @Override
+ public void getConfig(PortOverridesConfig.Builder builder) {
+ for (PortOverride portOverride: portOverrides) {
+ builder.server(new PortOverridesConfig.Server.Builder().
+ id(portOverride.serverId.stringValue()).
+ port(portOverride.port));
+ }
+ }
+
+ @Override
+ public void getConfig(QrConfig.Builder builder) {
+ builder.
+ rpc(new Rpc.Builder()
+ .enabled(isRpcServerEnabled())
+ .port(getRpcPort())
+ .slobrokId(serviceSlobrokId())).
+ filedistributor(filedistributorConfig());
+ if (clusterName != null) {
+ builder.discriminator(clusterName+"."+name);
+ } else {
+ builder.discriminator(name);
+ }
+ }
+
+ @Override
+ public String getJvmArgs() {
+ String jvmArgs = super.getJvmArgs();
+ return isHostedVespa() && hasDocproc()
+ ? ("".equals(jvmArgs) ? defaultHostedJVMArgs : defaultHostedJVMArgs + " " + jvmArgs)
+ : jvmArgs;
+ }
+ private String serviceSlobrokId() {
+ return "vespa/service/" + getConfigId();
+ }
+
+ private Filedistributor.Builder filedistributorConfig() {
+ Filedistributor.Builder builder = new Filedistributor.Builder();
+
+ FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer();
+ if (fileDistribution != null)
+ builder.configid(fileDistribution.getFileDistributionServiceConfigId(getHost()));
+
+ return builder;
+ }
+
+ @Override
+ public void getConfig(ComponentsConfig.Builder builder) {
+ builder.components.addAll(
+ ComponentsConfigGenerator.generate(allEnabledComponents()));
+ }
+
+ private Collection<Component<?, ?>> allEnabledComponents() {
+ Collection<Component<?, ?>> allComponents = new ArrayList<>();
+ addAllEnabledComponents(allComponents, this);
+ return Collections.unmodifiableCollection(allComponents);
+ }
+
+ private void addAllEnabledComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) {
+ for (AbstractConfigProducer<?> child: current.getChildren().values()) {
+ if (!httpServerEnabled && isHttpServer(child))
+ continue;
+
+ if (child instanceof Component) {
+ allComponents.add((Component<?, ?>) child);
+ }
+
+ addAllEnabledComponents(allComponents, child);
+ }
+ }
+
+ private boolean isHttpServer(AbstractConfigProducer<?> component) {
+ return component instanceof JettyHttpServer;
+ }
+
+ @Override
+ public final void getConfig(JdiscBindingsConfig.Builder builder) {
+ builder.handlers(DiscBindingsConfigGenerator.generate(handlers.getComponents()));
+ }
+
+ @Override
+ public void getConfig(ContainerHttpConfig.Builder builder) {
+ builder
+ .enabled(httpServerEnabled)
+ .port(new ContainerHttpConfig.Port.Builder()
+ .search(getSearchPort()));
+ }
+
+ @Override
+ public void getConfig(ContainerMbusConfig.Builder builder) {
+ builder.enabled(messageBusEnabled).
+ port(getMessagingPort());
+ }
+
+ @Override
+ public HashMap<String,String> getDefaultMetricDimensions(){
+ HashMap<String, String> dimensions = new HashMap<>();
+ if (clusterName != null) {
+ dimensions.put("clustername", clusterName);
+ }
+ return dimensions;
+ }
+
+ public void setRpcServerEnabled(boolean rpcServerEnabled) {
+ this.rpcServerEnabled = rpcServerEnabled;
+ }
+
+ public void setHttpServerEnabled(boolean httpServerEnabled) {
+ this.httpServerEnabled = httpServerEnabled;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
new file mode 100755
index 00000000000..ae158567587
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -0,0 +1,816 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container;
+
+import com.yahoo.cloud.config.ClusterInfoConfig;
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.cloud.config.RoutingProviderConfig;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.FileReference;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.docproc.DocprocConfig;
+import com.yahoo.config.docproc.SchemamappingConfig;
+import com.yahoo.config.application.api.ApplicationMetaData;
+import com.yahoo.config.application.api.ComponentInfo;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.config.provision.Rotation;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.container.BundlesConfig;
+import com.yahoo.container.ComponentsConfig;
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.core.ApplicationMetadataConfig;
+import com.yahoo.container.core.document.ContainerDocumentConfig;
+import com.yahoo.container.handler.ThreadPoolProvider;
+import com.yahoo.container.jdisc.ContainerMbusConfig;
+import com.yahoo.container.jdisc.JdiscBindingsConfig;
+import com.yahoo.container.jdisc.config.HealthMonitorConfig;
+import com.yahoo.container.jdisc.config.MetricDefaultsConfig;
+import com.yahoo.container.jdisc.messagebus.MbusServerProvider;
+import com.yahoo.container.jdisc.state.StateHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.container.usability.BindingsOverviewHandler;
+import com.yahoo.container.xml.providers.DatatypeFactoryProvider;
+import com.yahoo.container.xml.providers.DocumentBuilderFactoryProvider;
+import com.yahoo.container.xml.providers.JAXBContextFactoryProvider;
+import com.yahoo.container.xml.providers.SAXParserFactoryProvider;
+import com.yahoo.container.xml.providers.SchemaFactoryProvider;
+import com.yahoo.container.xml.providers.TransformerFactoryProvider;
+import com.yahoo.container.xml.providers.XMLEventFactoryProvider;
+import com.yahoo.container.xml.providers.XMLInputFactoryProvider;
+import com.yahoo.container.xml.providers.XMLOutputFactoryProvider;
+import com.yahoo.container.xml.providers.XPathFactoryProvider;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.jdisc.http.ServletPathsConfig;
+import com.yahoo.metrics.simple.runtime.MetricProperties;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.prelude.semantics.SemanticRulesConfig;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.search.config.QrStartConfig;
+import com.yahoo.search.pagetemplates.PageTemplatesConfig;
+import com.yahoo.search.query.profile.config.QueryProfilesConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.vespa.model.PortsMeta;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.admin.MonitoringSystem;
+import com.yahoo.vespa.model.clients.ContainerDocumentApi;
+import com.yahoo.vespa.model.container.component.AccessLogComponent;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.ComponentGroup;
+import com.yahoo.vespa.model.container.component.ComponentsConfigGenerator;
+import com.yahoo.vespa.model.container.component.ConfigProducerGroup;
+import com.yahoo.vespa.model.container.component.DiscBindingsConfigGenerator;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import com.yahoo.vespa.model.container.component.Servlet;
+import com.yahoo.vespa.model.container.component.StatisticsComponent;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.http.Http;
+import com.yahoo.vespa.model.container.jersey.Jersey2Servlet;
+import com.yahoo.vespa.model.container.jersey.JerseyHandler;
+import com.yahoo.vespa.model.container.jersey.RestApi;
+import com.yahoo.vespa.model.container.processing.ProcessingChains;
+import com.yahoo.vespa.model.container.search.ContainerSearch;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.utils.FileSender;
+import com.yahoo.vespaclient.config.FeederConfig;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+import java.io.Reader;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX;
+
+/**
+ * @author gjoranv
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @author tonytv
+ */
+public final class ContainerCluster
+ extends AbstractConfigProducer<AbstractConfigProducer<?>>
+ implements
+ ComponentsConfig.Producer,
+ JdiscBindingsConfig.Producer,
+ DocumentmanagerConfig.Producer,
+ ContainerMbusConfig.Producer,
+ ContainerDocumentConfig.Producer,
+ HealthMonitorConfig.Producer,
+ ApplicationMetadataConfig.Producer,
+ BundlesConfig.Producer,
+ FeederConfig.Producer,
+ IndexInfoConfig.Producer,
+ IlscriptsConfig.Producer,
+ SchemamappingConfig.Producer,
+ QrSearchersConfig.Producer,
+ QrStartConfig.Producer,
+ QueryProfilesConfig.Producer,
+ PageTemplatesConfig.Producer,
+ SemanticRulesConfig.Producer,
+ DocprocConfig.Producer,
+ MetricDefaultsConfig.Producer,
+ ClusterInfoConfig.Producer,
+ ServletPathsConfig.Producer,
+ RoutingProviderConfig.Producer,
+ ConfigserverConfig.Producer {
+ /**
+ * URI prefix used for internal, usually programmatic, APIs. URIs using this
+ * prefix should never considered available for direct use by customers, and
+ * normal compatibility concerns only applies to libraries using the URIs in
+ * question, not contents served from the URIs themselves.
+ */
+ public static final String RESERVED_URI_PREFIX = "reserved-for-internal-use";
+
+ public static final String ROOT_HANDLER_BINDING = "*://*/";
+
+ private String name;
+
+ private List<Container> containers = new ArrayList<>();
+
+ private Http http;
+ private ProcessingChains processingChains;
+ private ContainerSearch containerSearch;
+ private ContainerDocproc containerDocproc;
+ private ContainerDocumentApi containerDocumentApi;
+
+ private MbusParams mbusParams;
+
+ private final Set<FileReference> applicationBundles = new LinkedHashSet<>();
+ private final Set<Path> platformBundles = new LinkedHashSet<>();
+
+ private final List<String> serviceAliases = new ArrayList<>();
+ private final List<String> endpointAliases = new ArrayList<>();
+ protected final ComponentGroup<Component<?, ?>> componentGroup;
+ private final ConfigProducerGroup<RestApi> restApiGroup;
+ private final ConfigProducerGroup<Servlet> servletGroup;
+
+ private Map<String, String> concreteDocumentTypes = new LinkedHashMap<>();
+ private MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory;
+
+ private ApplicationMetaData applicationMetaData = null;
+
+ /** The zone this is deployed in, or the default zone if not on hosted Vespa */
+ private Zone zone;
+
+ public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name) {
+ super(parent, subId);
+ this.name = name;
+ this.zone = getRoot() != null ? getRoot().getDeployState().zone() : Zone.defaultZone();
+ componentGroup = new ComponentGroup<>(this, "component");
+ restApiGroup = new ConfigProducerGroup<>(this, "rest-api");
+ servletGroup = new ConfigProducerGroup<>(this, "servlet");
+
+ addComponent(new StatisticsComponent());
+ addSimpleComponent(AccessLog.class);
+ // TODO better modelling
+ addSimpleComponent(ThreadPoolProvider.class);
+ addSimpleComponent("com.yahoo.jdisc.http.filter.SecurityFilterInvoker");
+ addSimpleComponent("com.yahoo.language.provider.SimpleLinguisticsProvider");
+ addSimpleComponent("com.yahoo.container.jdisc.SslKeyStoreFactoryProvider");
+ addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider");
+ addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider");
+ addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider");
+ addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider");
+ addSimpleComponent("com.yahoo.container.jdisc.metric.MetricUpdater");
+ addSimpleComponent(com.yahoo.metrics.simple.MetricManager.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME);
+ addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME);
+ addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor");
+ addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory");
+ addSimpleComponent("com.yahoo.container.protect.FreezeDetector");
+ addSimpleComponent("com.yahoo.container.core.slobrok.SlobrokConfigurator");
+ addSimpleComponent("com.yahoo.container.handler.VipStatus");
+ addJaxProviders();
+ }
+
+ public void setZone(Zone zone) {
+ this.zone = zone;
+ }
+
+ public void addMetricStateHandler() {
+ Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>(
+ new ComponentModel("com.yahoo.container.jdisc.state.StateHandler", null, null, null));
+ stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT,
+ "https://*" + StateHandler.STATE_API_ROOT,
+ "http://*" + StateHandler.STATE_API_ROOT + "/*",
+ "https://*" + StateHandler.STATE_API_ROOT + "/*");
+ addComponent(stateHandler);
+ }
+
+ public void addDefaultRootHandler() {
+ if (hasHandlerWithBinding(ROOT_HANDLER_BINDING))
+ return;
+
+ Handler<AbstractConfigProducer<?>> handler = new Handler<>(
+ new ComponentModel(BundleInstantiationSpecification.getFromStrings(
+ BindingsOverviewHandler.class.getName(), null, null), null)); // null bundle, as the handler is in container-disc
+ handler.addServerBindings(ROOT_HANDLER_BINDING);
+ addComponent(handler);
+ }
+
+ private boolean hasHandlerWithBinding(String binding) {
+ Collection<Handler<?>> handlers = getHandlers();
+ for (Handler handler : handlers) {
+ if (handler.getServerBindings().contains(binding))
+ return true;
+ }
+ return false;
+ }
+
+ public void addApplicationStatusHandler() {
+ Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>(
+ new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(
+ "com.yahoo.container.handler.observability.ApplicationStatusHandler", null), null));
+ statusHandler.addServerBindings("http://*/ApplicationStatus",
+ "https://*/ApplicationStatus");
+ addComponent(statusHandler);
+ }
+
+ public void addVipHandler() {
+ Handler<?> vipHandler = Handler.fromClassName("com.yahoo.container.handler.VipStatusHandler");
+ vipHandler.addServerBindings("http://*/status.html", "https://*/status.html");
+ addComponent(vipHandler);
+ }
+
+ public void addStatisticsHandler() {
+ Handler<?> statsHandler = Handler.fromClassName("com.yahoo.container.config.StatisticsRequestHandler");
+ statsHandler.addServerBindings("http://*/statistics/*", "https://*/statistics/*");
+ addComponent(statsHandler);
+ }
+
+ public void addJaxProviders() {
+ addSimpleComponent(DatatypeFactoryProvider.class);
+ addSimpleComponent(DocumentBuilderFactoryProvider.class);
+ addSimpleComponent(JAXBContextFactoryProvider.class);
+ addSimpleComponent(SAXParserFactoryProvider.class);
+ addSimpleComponent(SchemaFactoryProvider.class);
+ addSimpleComponent(TransformerFactoryProvider.class);
+ addSimpleComponent(XMLEventFactoryProvider.class);
+ addSimpleComponent(XMLInputFactoryProvider.class);
+ addSimpleComponent(XMLOutputFactoryProvider.class);
+ addSimpleComponent(XPathFactoryProvider.class);
+ }
+
+ public final void addComponent(Component<?, ?> component) {
+ componentGroup.addComponent(component);
+ }
+
+ public final void addComponents(Collection<Component<?, ?>> components) {
+ for (Component<?, ?> component : components) {
+ addComponent(component);
+ }
+ }
+
+ public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) {
+ addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec)));
+ }
+
+ /**
+ * Removes a component by id
+ *
+ * @return the removed component, or null if it was not present
+ */
+ public Component removeComponent(ComponentId componentId) {
+ return componentGroup.removeComponent(componentId);
+ }
+
+ public void addSimpleComponent(Class<?> clazz) {
+ addSimpleComponent(clazz.getName());
+ }
+
+ private void addSimpleComponent(String className) {
+ addComponent(new SimpleComponent(className));
+ }
+
+ public void prepare() {
+ addAndSendApplicationBundles();
+ sendUserConfiguredFiles();
+ setApplicationMetaData();
+ for (RestApi restApi : restApiGroup.getComponents())
+ restApi.prepare();
+ }
+
+ private void setApplicationMetaData() {
+ applicationMetaData = getRoot().getDeployState().getApplicationPackage().getMetaData();
+ }
+
+ public void addMbusServer(ComponentId chainId) {
+ ComponentId serviceId = chainId.nestInNamespace(ComponentId.fromString("MbusServer"));
+
+ addComponent(
+ new Component<>(new ComponentModel(new BundleInstantiationSpecification(
+ serviceId,
+ ComponentSpecification.fromString(MbusServerProvider.class.getName()),
+ null))));
+ }
+
+ private void addAndSendApplicationBundles() {
+ for (ComponentInfo component : getRoot().getDeployState().getApplicationPackage().getComponentsInfo(getRoot().getDeployState().getProperties().vespaVersion())) {
+ FileReference reference = FileSender.sendFileToServices(component.getPathRelativeToAppDir(), containers);
+ applicationBundles.add(reference);
+ }
+ }
+
+ private void sendUserConfiguredFiles() {
+ // Files referenced from user configs to all components.
+ for (Component<?, ?> component : getAllComponents()) {
+ FileSender.sendUserConfiguredFiles(component, containers, deployLogger());
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<Container> getContainers() {
+ return Collections.unmodifiableList(containers);
+ }
+
+ public void addContainers(Collection<Container> containers) {
+ int index = this.containers.size();
+ for (Container container : containers) {
+ container.setClusterName(name);
+ container.setProp("clustername", name)
+ .setProp("index", index++);
+ setRotations(container, getRotations(), getGlobalServiceId(), name);
+ container.setProp("activeRotation", Boolean.toString(getActiveRotation()));
+ }
+ this.containers.addAll(containers);
+ }
+
+ private Optional<String> getGlobalServiceId() {
+ Optional<String> globalServiceId = Optional.empty();
+ Optional<DeploymentSpec> deploymentSpec = getDeploymentSpec();
+ if (deploymentSpec.isPresent()) {
+ globalServiceId = deploymentSpec.get().globalServiceId();
+ }
+ return globalServiceId;
+ }
+
+ private Set<Rotation> getRotations() {
+ return Optional.ofNullable(getRoot())
+ .map(root -> root.getDeployState().getRotations())
+ .orElse(Collections.emptySet());
+ }
+
+ private boolean getActiveRotation() {
+ return Optional.ofNullable(getRoot())
+ .map(root -> root.getDeployState().getProperties().zone())
+ .map(this::zoneHasActiveRotation)
+ .orElse(false);
+ }
+
+ private boolean zoneHasActiveRotation(Zone zone) {
+ return getDeploymentSpec()
+ .flatMap(spec -> spec.zones().stream()
+ .filter(dz -> dz.matches(zone.environment(), Optional.of(zone.region())))
+ .findFirst())
+ .map(DeploymentSpec.DeclaredZone::active)
+ .orElse(false);
+ }
+
+ private Optional<DeploymentSpec> getDeploymentSpec() {
+ Optional<DeploymentSpec> deploymentSpec = Optional.empty();
+ AbstractConfigProducerRoot root = getRoot();
+ if (root != null) {
+ final Optional<Reader> deployment = root.getDeployState().getApplicationPackage().getDeployment();
+ if (deployment.isPresent()) {
+ deploymentSpec = Optional.of(DeploymentSpec.fromXml(deployment.get()));
+ }
+ }
+ return deploymentSpec;
+ }
+
+ private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) {
+ if (!rotations.isEmpty() && globalServiceId.isPresent()) {
+ if (containerClusterName.equals(globalServiceId.get())) {
+ container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(",")));
+ }
+ }
+ }
+
+ public void addContainer(Container container) {
+ addContainers(Collections.singletonList(container));
+ }
+
+ public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) {
+ if (this.processingChains != null)
+ throw new IllegalStateException("ProcessingChains should only be set once.");
+
+ this.processingChains = processingChains;
+
+ // Cannot use the class object for ProcessingHandler, because its superclass is not accessible
+ ProcessingHandler<?> processingHandler = new ProcessingHandler<>(
+ processingChains,
+ "com.yahoo.processing.handler.ProcessingHandler");
+
+ for (String binding: serverBindings)
+ processingHandler.addServerBindings(binding);
+
+ addComponent(processingHandler);
+ }
+
+ public ProcessingChains getProcessingChains() {
+ return processingChains;
+ }
+
+ @NonNull
+ public SearchChains getSearchChains() {
+ if (containerSearch == null)
+ throw new IllegalStateException("Null search components!");
+ return containerSearch.getChains();
+ }
+
+ @Nullable
+ public ContainerSearch getSearch() {
+ return containerSearch;
+ }
+
+ public void setSearch(ContainerSearch containerSearch) {
+ this.containerSearch = containerSearch;
+ }
+
+ public void setHttp(Http http) {
+ this.http = http;
+ addChild(http);
+ }
+
+ @Nullable
+ public Http getHttp() {
+ return http;
+ }
+
+ public final void addRestApi(@NonNull RestApi restApi) {
+ restApiGroup.addComponent(ComponentId.fromString(restApi.getBindingPath()), restApi);
+ }
+
+ public Map<ComponentId, RestApi> getRestApiMap() {
+ return restApiGroup.getComponentMap();
+ }
+
+ public Map<ComponentId, Servlet> getServletMap() {
+ return servletGroup.getComponentMap();
+ }
+
+ public final void addServlet(@NonNull Servlet servlet) {
+ servletGroup.addComponent(servlet.getGlobalComponentId(), servlet);
+ }
+
+ @Nullable
+ public ContainerDocproc getDocproc() {
+ return containerDocproc;
+ }
+
+ public void setDocproc(ContainerDocproc containerDocproc) {
+ this.containerDocproc = containerDocproc;
+ }
+
+ @Nullable
+ public ContainerDocumentApi getDocumentApi() {
+ return containerDocumentApi;
+ }
+
+ public void setDocumentApi(ContainerDocumentApi containerDocumentApi) {
+ this.containerDocumentApi = containerDocumentApi;
+ }
+
+ @NonNull
+ public DocprocChains getDocprocChains() {
+ if (containerDocproc == null)
+ throw new IllegalStateException("Null docproc components!");
+ return containerDocproc.getChains();
+ }
+
+ @SuppressWarnings("unchecked")
+ public Collection<Handler<?>> getHandlers() {
+ return (Collection<Handler<?>>)(Collection)componentGroup.getComponents(Handler.class);
+ }
+
+ public Map<ComponentId, Component<?, ?>> getComponentsMap() {
+ return componentGroup.getComponentMap();
+ }
+
+ /** Returns all components in this cluster (generic, handlers, chained) */
+ public Collection<Component<?, ?>> getAllComponents() {
+ List<Component<?, ?>> allComponents = new ArrayList<>();
+ recursivelyFindAllComponents(allComponents, this);
+ // We need consistent ordering
+ Collections.sort(allComponents);
+ return Collections.unmodifiableCollection(allComponents);
+ }
+
+ private void recursivelyFindAllComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) {
+ for (AbstractConfigProducer<?> child: current.getChildren().values()) {
+ if (child instanceof Component)
+ allComponents.add((Component<?, ?>) child);
+
+ if (!(child instanceof Container))
+ recursivelyFindAllComponents(allComponents, child);
+ }
+ }
+
+ @Override
+ public final void getConfig(ComponentsConfig.Builder builder) {
+ builder.components.addAll(ComponentsConfigGenerator.generate(getAllComponents()));
+ builder.components(new ComponentsConfig.Components.Builder().id("com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack"));
+ }
+
+ @Override
+ public final void getConfig(JdiscBindingsConfig.Builder builder) {
+ builder.handlers.putAll(DiscBindingsConfigGenerator.generate(getHandlers()));
+
+ allJersey1Handlers().forEach(handler ->
+ builder.handlers.putAll(DiscBindingsConfigGenerator.generate(handler))
+ );
+ }
+
+ private Stream<JerseyHandler> allJersey1Handlers() {
+ return restApiGroup.getComponents().stream().flatMap(streamOf(RestApi::getJersey1Handler));
+ }
+
+ @Override
+ public void getConfig(ServletPathsConfig.Builder builder) {
+ allServlets().forEach(servlet ->
+ builder.servlets(servlet.getComponentId().stringValue(),
+ servlet.toConfigBuilder())
+ );
+ }
+
+ private Stream<Servlet> allServlets() {
+ return Stream.concat(allJersey2Servlets(),
+ servletGroup.getComponents().stream());
+ }
+
+ private Stream<Jersey2Servlet> allJersey2Servlets() {
+ return restApiGroup.getComponents().stream().flatMap(streamOf(RestApi::getJersey2Servlet));
+ }
+
+ private <T, R> Function<T, Stream<R>> streamOf(Function<T, Optional<R>> f) {
+ return t ->
+ f.apply(t).
+ <Stream<R>>map(Stream::of).
+ orElse(Stream.empty());
+ }
+
+ @Override
+ public void getConfig(DocumentmanagerConfig.Builder builder) {
+ if (containerDocproc != null && containerDocproc.isCompressDocuments())
+ builder.enablecompression(true);
+ }
+
+ @Override
+ public void getConfig(ContainerDocumentConfig.Builder builder) {
+ for (Map.Entry<String, String> e : concreteDocumentTypes.entrySet()) {
+ ContainerDocumentConfig.Doctype.Builder dtb = new ContainerDocumentConfig.Doctype.Builder();
+ dtb.type(e.getKey());
+ dtb.factorycomponent(e.getValue());
+ builder.doctype(dtb);
+ }
+ }
+
+ @Override
+ public void getConfig(HealthMonitorConfig.Builder builder) {
+ MonitoringSystem monitoringSystem = getMonitoringService();
+ if (monitoringSystem != null) {
+ builder.snapshot_interval(monitoringSystem.getIntervalSeconds());
+ }
+ }
+
+ @Override
+ public void getConfig(ApplicationMetadataConfig.Builder builder) {
+ if (applicationMetaData != null) {
+ builder.name(applicationMetaData.getApplicationName()).
+ user(applicationMetaData.getDeployedByUser()).
+ path(applicationMetaData.getDeployPath()).
+ timestamp(applicationMetaData.getDeployTimestamp()).
+ checksum(applicationMetaData.getCheckSum()).
+ generation(applicationMetaData.getGeneration());
+ }
+ }
+
+ /**
+ * Adds a bundle present at a known location at the target container nodes.
+ *
+ * @param bundlePath usually an absolute path, e.g. '$VESPA_HOME/lib/jars/foo.jar'
+ */
+ public final void addPlatformBundle(Path bundlePath) {
+ platformBundles.add(bundlePath);
+ }
+
+ @Override
+ public void getConfig(BundlesConfig.Builder builder) {
+ Stream.concat(applicationBundles.stream().map(FileReference::value),
+ platformBundles.stream().map(ContainerCluster::toFileReferenceString))
+ .forEach(builder::bundle);
+ }
+
+ private static String toFileReferenceString(Path path) {
+ return DISK_BUNDLE_PREFIX + path.toString();
+ }
+
+ @Override
+ public void getConfig(QrSearchersConfig.Builder builder) {
+ if (containerSearch!=null) containerSearch.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(QrStartConfig.Builder builder) {
+ if (containerSearch!=null) containerSearch.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(DocprocConfig.Builder builder) {
+ if (containerDocproc != null) {
+ containerDocproc.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(PageTemplatesConfig.Builder builder) {
+ if (containerSearch != null) {
+ containerSearch.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(SemanticRulesConfig.Builder builder) {
+ if (containerSearch != null) {
+ containerSearch.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(QueryProfilesConfig.Builder builder) {
+ if (containerSearch != null) {
+ containerSearch.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(SchemamappingConfig.Builder builder) {
+ if (containerDocproc!=null) containerDocproc.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ if (containerSearch!=null) containerSearch.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(FeederConfig.Builder builder) {
+ if (containerDocumentApi != null) {
+ containerDocumentApi.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(ContainerMbusConfig.Builder builder) {
+ if (mbusParams != null) {
+ if (mbusParams.maxConcurrentFactor != null)
+ builder.maxConcurrentFactor(mbusParams.maxConcurrentFactor);
+ if (mbusParams.documentExpansionFactor != null)
+ builder.documentExpansionFactor(mbusParams.documentExpansionFactor);
+ if (mbusParams.containerCoreMemory != null)
+ builder.containerCoreMemory(mbusParams.containerCoreMemory);
+ }
+ if (containerDocproc != null)
+ containerDocproc.getConfig(builder);
+ }
+
+ public void setMbusParams(MbusParams mbusParams) {
+ this.mbusParams = mbusParams;
+ }
+
+ public void initialize(Map<String, AbstractSearchCluster> clusterMap) {
+ if (containerSearch != null)
+ containerSearch.connectSearchClusters(clusterMap);
+ }
+
+ public void addDefaultSearchAccessLog() {
+ addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.queryAccessLog, getName()));
+ }
+
+ @Override
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ List<AbstractSearchCluster> searchClusters = new ArrayList<>();
+ searchClusters.addAll(Content.getSearchClusters(getRoot().configModelRepo()));
+ for (AbstractSearchCluster searchCluster : searchClusters) {
+ searchCluster.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(MetricDefaultsConfig.Builder builder) {
+ if (defaultMetricConsumerFactory != null) {
+ builder.factory(defaultMetricConsumerFactory);
+ }
+ }
+
+ @Override
+ public void getConfig(ClusterInfoConfig.Builder builder) {
+ builder.clusterId(name);
+ builder.nodeCount(containers.size());
+
+ for (Service service : getDescendantServices()) {
+ builder.services.add(new ClusterInfoConfig.Services.Builder()
+ .index(Integer.parseInt(service.getServicePropertyString("index", "99999")))
+ .hostname(service.getHostName())
+ .ports(getPorts(service)));
+ }
+ }
+
+ /**
+ * Returns a config server config containing the right zone settings (and defaults for the rest).
+ * This is useful to allow applications to find out in which zone they are runnung by having the Zone
+ * object (which is constructed from this config) injected.
+ */
+ @Override
+ public void getConfig(ConfigserverConfig.Builder builder) {
+ builder.environment(zone.environment().value());
+ builder.region(zone.region().value());
+ }
+
+ private List<ClusterInfoConfig.Services.Ports.Builder> getPorts(Service service) {
+ List<ClusterInfoConfig.Services.Ports.Builder> builders = new ArrayList<>();
+ PortsMeta portsMeta = service.getPortsMeta();
+ for (int i = 0; i < portsMeta.getNumPorts(); i++) {
+ builders.add(new ClusterInfoConfig.Services.Ports.Builder()
+ .number(service.getRelativePort(i))
+ .tags(ApplicationConfigProducerRoot.getPortTags(portsMeta, i))
+ );
+ }
+ return builders;
+ }
+
+ public void setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory) {
+ Objects.requireNonNull(defaultMetricConsumerFactory, "defaultMetricConsumerFactory");
+ this.defaultMetricConsumerFactory = defaultMetricConsumerFactory;
+ }
+
+ @Override
+ public void getConfig(RoutingProviderConfig.Builder builder) {
+ builder.enabled(isHostedVespa());
+ }
+
+ public static class MbusParams {
+ //the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%)
+ public final Double maxConcurrentFactor;
+
+ //the amount that documents expand temporarily when processing them
+ public final Double documentExpansionFactor;
+
+ //the space to reserve for container, docproc stuff (memory that cannot be used for processing documents), in MB
+ public final Integer containerCoreMemory;
+
+ public MbusParams(Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) {
+ this.maxConcurrentFactor = maxConcurrentFactor;
+ this.documentExpansionFactor = documentExpansionFactor;
+ this.containerCoreMemory = containerCoreMemory;
+ }
+ }
+
+ public Map<String, String> concreteDocumentTypes() {
+ return concreteDocumentTypes;
+ }
+
+ /**
+ * The configured service aliases for the service in this cluster
+ * @return alias list
+ */
+ public List<String> serviceAliases() {
+ return serviceAliases;
+ }
+
+ /**
+ * The configured endpoint aliases (fqdn) for the service in this cluster
+ * @return alias list
+ */
+ public List<String> endpointAliases() {
+ return endpointAliases;
+ }
+
+ @Override
+ public String toString() {
+ return "container cluster '" + getName() + "'";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
new file mode 100644
index 00000000000..29846ded00f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container;
+
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+
+/**
+ * A model of a container cluster.
+ *
+ * @author tonytv
+ */
+public class ContainerModel extends ConfigModel {
+
+ // TODO: Move to referer
+ public static final String DOCPROC_RESERVED_NAME = "docproc";
+
+ private ContainerCluster containerCluster;
+
+ public ContainerModel(ConfigModelContext context) {
+ super(context);
+ }
+
+ public void setCluster(ContainerCluster containerCluster) { this.containerCluster = containerCluster; }
+
+ public ContainerCluster getCluster() { return containerCluster; }
+
+ @Override
+ public void prepare(ConfigModelRepo plugins) {
+ assert (getCluster() != null) : "Null container cluster!";
+ getCluster().prepare();
+ }
+
+ @Override
+ public void initialize(ConfigModelRepo configModelRepo) {
+ List<AbstractSearchCluster> searchClusters = Content.getSearchClusters(configModelRepo);
+
+ Map<String, AbstractSearchCluster> searchClustersByName = new TreeMap<>();
+ for (AbstractSearchCluster c : searchClusters) {
+ searchClustersByName.put(c.getClusterName(), c);
+ }
+
+ getCluster().initialize(searchClustersByName);
+ }
+
+ public static Collection<ContainerCluster> containerClusters(ConfigModelRepo models) {
+ List<ContainerCluster> containerClusters = new ArrayList<>();
+
+ for (ContainerModel model: models.getModels(ContainerModel.class))
+ containerClusters.add(model.getCluster());
+
+ return containerClusters;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
new file mode 100644
index 00000000000..75ca2c1e958
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.container.core.AccessLogConfig;
+import com.yahoo.container.logging.VespaAccessLog;
+import com.yahoo.container.logging.YApacheAccessLog;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ * @since 5.1.4
+ */
+public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer {
+
+
+ public enum AccessLogType { queryAccessLog, yApacheAccessLog }
+
+ private final String fileNamePattern;
+ private final String rotationInterval;
+ private final RotateScheme.Enum rotationScheme;
+ private final String symlinkName;
+
+ public AccessLogComponent(AccessLogType logType, String clusterName) {
+ this(logType,
+ String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"),
+ null, null,
+ capitalize(logType.name()) + "." + clusterName);
+ }
+
+ private static String capitalize(String name) {
+ return name.substring(0, 1).toUpperCase() + name.substring(1);
+ }
+
+ public AccessLogComponent(AccessLogType logType,
+ String fileNamePattern,
+ String rotationInterval,
+ RotateScheme.Enum rotationScheme, String symlinkName) {
+ super(new ComponentModel(accessLogClass(logType), null, "container-core", null));
+ this.fileNamePattern = fileNamePattern;
+ this.rotationInterval = rotationInterval;
+ this.rotationScheme = rotationScheme;
+ this.symlinkName = symlinkName;
+
+ if (fileNamePattern == null)
+ throw new RuntimeException("File name pattern required when configuring access log.");
+ }
+
+ private static String accessLogClass(AccessLogType logType) {
+ switch (logType) {
+ case yApacheAccessLog:
+ return YApacheAccessLog.class.getName();
+ case queryAccessLog:
+ return VespaAccessLog.class.getName();
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
+ public void getConfig(AccessLogConfig.Builder builder) {
+ builder.fileHandler(fileHandlerConfig());
+ }
+
+ private AccessLogConfig.FileHandler.Builder fileHandlerConfig() {
+ AccessLogConfig.FileHandler.Builder builder = new AccessLogConfig.FileHandler.Builder();
+ if (fileNamePattern != null)
+ builder.pattern(fileNamePattern);
+ if (rotationInterval != null)
+ builder.rotation(rotationInterval);
+ if (rotationScheme != null)
+ builder.rotateScheme(rotationScheme);
+ if (symlinkName != null)
+ builder.symlink(symlinkName);
+
+ return builder;
+ }
+
+ public String getFileNamePattern() {
+ return fileNamePattern;
+ }
+
+ public static final RotateScheme.Enum rotateScheme(@Nullable String name) {
+ if (name == null)
+ return null;
+
+ switch (name) {
+ case "date":
+ return RotateScheme.Enum.DATE;
+ case "sequence":
+ return RotateScheme.Enum.SEQUENCE;
+ default:
+ throw new IllegalArgumentException("Invalid rotation scheme " + name);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java
new file mode 100644
index 00000000000..8f83978ce02
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.osgi.provider.model.ComponentModel;
+
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * @author gjoranv
+ * @author tonytv
+ */
+public class Component<CHILD extends AbstractConfigProducer<?>, MODEL extends ComponentModel>
+ extends AbstractConfigProducer<CHILD> implements Comparable<Component<?, ?>> {
+
+ public final MODEL model;
+ final Set<Pair<String, Component>> injectedComponents = new LinkedHashSet<>();
+
+ public Component(MODEL model) {
+ super(model.getComponentId().stringValue());
+ this.model = model;
+ }
+
+ public ComponentId getGlobalComponentId() {
+ return model.getComponentId();
+ }
+
+ public ComponentId getComponentId() {
+ return model.getComponentId();
+ }
+
+ public ComponentSpecification getClassId() {
+ return model.getClassId();
+ }
+
+ public void inject(Component component) {
+ injectForName("", component);
+ }
+
+ public void injectForName(String name, Component component) {
+ injectedComponents.add(new Pair<>(name, component));
+ }
+
+ public void addComponent(CHILD child) {
+ addChild(child);
+ }
+
+ /** For testing only */
+ public Set<String> getInjectedComponentIds() {
+ Set<String> injectedIds = new HashSet<>();
+ for (Pair<String, Component> injected : injectedComponents) {
+ injectedIds.add(injected.getSecond().getSubId());
+ }
+ return injectedIds;
+ }
+
+ @Override
+ public int compareTo(Component<?, ?> other) {
+ return getComponentId().compareTo(other.getComponentId());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java
new file mode 100644
index 00000000000..dda21bd80da
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * @author tonytv
+ */
+public class ComponentGroup <CHILD extends Component<?, ?>> extends ConfigProducerGroup<CHILD> {
+
+ public ComponentGroup(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ }
+
+ public void addComponent(CHILD producer) {
+ super.addComponent(producer.getComponentId(), producer);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java
new file mode 100644
index 00000000000..a800daedd38
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static com.yahoo.container.ComponentsConfig.Components;
+
+/**
+ * @author gjoranv
+ */
+public class ComponentsConfigGenerator {
+
+ public static List<Components.Builder> generate(Collection<? extends Component<?, ?>> components) {
+ List<Components.Builder> result = new ArrayList<>();
+
+ for (Component component : components) {
+ result.add(componentsConfig(component));
+ }
+ return result;
+ }
+
+ public static Components.Builder componentsConfig(Component<?, ?> component) {
+ Components.Builder builder = new Components.Builder();
+ builder.id(component.getGlobalComponentId().stringValue());
+ builder.configId(component.getConfigId());
+
+
+ bundleInstantiationSpecification(builder, component.model.bundleInstantiationSpec);
+ builder.inject.addAll(componentsToInject(component.injectedComponents));
+
+ return builder;
+ }
+
+ private static void bundleInstantiationSpecification(Components.Builder config, BundleInstantiationSpecification spec) {
+ config.classId(spec.classId.stringValue());
+ config.bundle(spec.bundle.stringValue());
+ }
+
+
+ private static List<Components.Inject.Builder> componentsToInject(
+ Collection<Pair<String, Component>> injectedComponents) {
+
+ List<Components.Inject.Builder> result = new ArrayList<>();
+
+ for (Pair<String, Component> injected : injectedComponents) {
+ Components.Inject.Builder builder = new Components.Inject.Builder();
+
+ builder.id(injected.getSecond().getGlobalComponentId().stringValue());
+ builder.name(injected.getFirst());
+ result.add(builder);
+ }
+ return result;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
new file mode 100644
index 00000000000..d3311806d69
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+import java.util.*;
+
+/**
+ * A group of config producers that have a component id.
+ *
+ * @author tonytv
+ */
+public class ConfigProducerGroup<CHILD extends AbstractConfigProducer<?>> extends AbstractConfigProducer<CHILD> {
+
+ private final Map<ComponentId, CHILD> producerById = new LinkedHashMap<>();
+
+ public ConfigProducerGroup(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ }
+
+ public void addComponent(ComponentId id, CHILD producer) {
+ boolean wasAdded = producerById.put(id, producer) == null;
+ if (!wasAdded) {
+ throw new IllegalArgumentException("Two entities have the same component id '" +
+ id + "' in the same scope.");
+ }
+ addChild(producer);
+ }
+
+ /**
+ * Removes a component by id
+ *
+ * @return the removed component, or null if it was not present
+ */
+ public CHILD removeComponent(ComponentId componentId) {
+ CHILD component = producerById.remove(componentId);
+ if (component == null) return null;
+ removeChild(component);
+ return component;
+ }
+
+ public Collection<CHILD> getComponents() {
+ return Collections.unmodifiableCollection(getChildren().values());
+ }
+
+ public <T extends CHILD> Collection<T> getComponents(Class<T> componentClass) {
+ Collection<T> result = new ArrayList<>();
+
+ for (CHILD child: getChildren().values()) {
+ if (componentClass.isInstance(child)) {
+ result.add(componentClass.cast(child));
+ }
+ }
+ return Collections.unmodifiableCollection(result);
+ }
+
+ /**
+ * @return A map of all components in this group, with (local) component ID as key.
+ */
+ public Map<ComponentId, CHILD> getComponentMap() {
+ return Collections.unmodifiableMap(producerById);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java
new file mode 100644
index 00000000000..9880aa49a95
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.chain.Chains;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Holder for components and options related to either processing/search/docproc
+ * for a container cluster.
+ *
+ * @author gjoranv
+ * @since 5.1.9
+ */
+public abstract class ContainerSubsystem<CHAINS extends Chains<?>> {
+
+ private final CHAINS chains;
+
+ public ContainerSubsystem(CHAINS chains) {
+ this.chains = chains;
+ }
+
+ @NonNull
+ public CHAINS getChains() {
+ if (chains == null)
+ throw new IllegalStateException("Null chains for " + this);
+ return chains;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java
new file mode 100644
index 00000000000..6c03c64ce02
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import java.util.*;
+
+import static com.yahoo.container.jdisc.JdiscBindingsConfig.Handlers;
+
+/**
+ * @author gjoranv
+ * @since 5.1.8
+ */
+public class DiscBindingsConfigGenerator {
+
+ public static Map<String, Handlers.Builder> generate(Collection<? extends Handler<?>> handlers) {
+ Map<String, Handlers.Builder> handlerBuilders = new LinkedHashMap<>();
+
+ for (Handler<?> handler : handlers) {
+ handlerBuilders.putAll(generate(handler));
+ }
+ return handlerBuilders;
+ }
+
+ public static <T extends Handler<?>> Map<String, Handlers.Builder> generate(T handler) {
+ if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty())
+ return Collections.emptyMap();
+
+ return Collections.singletonMap(handler.model.getComponentId().stringValue(),
+ new Handlers.Builder()
+ .serverBindings(handler.getServerBindings())
+ .clientBindings(handler.getClientBindings()));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
new file mode 100644
index 00000000000..758806395a6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.container.core.VipStatusConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+
+/**
+ * Sets up VipStatusHandler that answers OK when a certain file is present.
+ * @author tonytv
+ */
+public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer {
+ private final String fileName;
+
+ public FileStatusHandlerComponent(String id, String fileName, String... bindings) {
+ super(new ComponentModel(id, "com.yahoo.container.handler.VipStatusHandler", null, null));
+
+ this.fileName = fileName;
+ addServerBindings(bindings);
+ }
+
+ @Override
+ public void getConfig(VipStatusConfig.Builder builder) {
+ builder.accessdisk(true).
+ statusfile(fileName);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
new file mode 100644
index 00000000000..0a9657a2d9a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * <p>
+ * Models a jdisc RequestHandler (including ClientProvider).
+ * RequestHandlers always have at least one server binding,
+ * while ClientProviders have at least one client binding.
+ * </p>
+ * <p>
+ * Note that this is also used to model vespa handlers (which do not have any bindings)
+ * </p>
+ *
+ * @author gjoranv
+ * @since 5.1.6
+ */
+public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> {
+
+ private List<String> serverBindings = new ArrayList<>();
+ private List<String> clientBindings = new ArrayList<>();
+
+ public Handler(ComponentModel model) {
+ super(model);
+ }
+
+ public static Handler<AbstractConfigProducer<?>> fromClassName(String className) {
+ return new Handler<>(new ComponentModel(className, null, null, null));
+ }
+
+ public static Handler<AbstractConfigProducer<?>> getVespaHandlerFromClassName(String className) {
+ return new Handler<>(new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(className, null), null));
+ }
+
+ public void addServerBindings(String... bindings) {
+ serverBindings.addAll(Arrays.asList(bindings));
+ }
+
+ public void addClientBindings(String... bindings) {
+ clientBindings.addAll(Arrays.asList(bindings));
+ }
+
+ public final List<String> getServerBindings() {
+ return Collections.unmodifiableList(serverBindings);
+ }
+
+ public final List<String> getClientBindings() {
+ return Collections.unmodifiableList(clientBindings);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java
new file mode 100644
index 00000000000..b919d8bd70e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.container.FilterConfigProvider;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+
+/**
+ * This is only for the legacy yca filter setup, outside http.
+ *
+ * TODO: Remove when 'filter' directly under 'jdisc' can be removed from services.xml
+ *
+ * @author tonytv
+ */
+public class HttpFilter extends SimpleComponent {
+ private static final ComponentSpecification filterConfigProviderClass =
+ ComponentSpecification.fromString(FilterConfigProvider.class.getName());
+
+ public final SimpleComponent filterConfigProvider;
+
+ public HttpFilter(BundleInstantiationSpecification spec) {
+ super(new ComponentModel(spec));
+
+ filterConfigProvider = new SimpleComponent(new ComponentModel(
+ new BundleInstantiationSpecification(configProviderId(spec.id), filterConfigProviderClass, null)));
+
+ addChild(filterConfigProvider);
+ }
+
+ // public for testing
+ public static ComponentId configProviderId(ComponentId filterId) {
+ return ComponentId.fromString("filterConfig").nestInNamespace(filterId);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java
new file mode 100644
index 00000000000..68ba3436209
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.jdisc.http.ServletPathsConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+
+/**
+ * @author stiankri
+ */
+public class Servlet extends SimpleComponent {
+ private final String bindingPath;
+
+ public Servlet(ComponentModel componentModel, String bindingPath) {
+ super(componentModel);
+ this.bindingPath = bindingPath;
+ }
+
+ public ServletPathsConfig.Servlets.Builder toConfigBuilder() {
+ return new ServletPathsConfig.Servlets.Builder()
+ .path(bindingPath);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java
new file mode 100644
index 00000000000..7c6bc169878
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.servlet.ServletConfigConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+
+import java.util.Map;
+
+/**
+ * @author stiankri
+ */
+public class ServletProvider extends Servlet implements ServletConfigConfig.Producer {
+ public static final String BUNDLE = "container-core";
+ public static final String CLASS = "com.yahoo.container.servlet.ServletProvider";
+
+ private static final ComponentId SERVLET_PROVIDER_NAMESPACE = ComponentId.fromString("servlet-provider");
+ private final Map<String, String> servletConfig;
+
+ public ServletProvider(SimpleComponent servletToProvide, String bindingPath, Map<String, String> servletConfig) {
+ super(new ComponentModel(
+ new BundleInstantiationSpecification(servletToProvide.getComponentId().nestInNamespace(SERVLET_PROVIDER_NAMESPACE),
+ ComponentSpecification.fromString(CLASS),
+ ComponentSpecification.fromString(BUNDLE))),
+ bindingPath);
+
+ inject(servletToProvide);
+ addChild(servletToProvide);
+ this.servletConfig = servletConfig;
+ }
+
+ @Override
+ public void getConfig(ServletConfigConfig.Builder builder) {
+ builder.map(servletConfig);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
new file mode 100644
index 00000000000..a0835f3ce88
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * A component that only needs a simple ComponentModel.
+ *
+ * @author gjoranv
+ * @since 5.1.9
+ */
+public class SimpleComponent extends Component<AbstractConfigProducer<?>, ComponentModel> {
+
+ public SimpleComponent(ComponentModel model) {
+ super(model);
+ }
+
+ // @Convenience // For a component that uses the class name as id.
+ public SimpleComponent(String className) {
+ this(new ComponentModel(BundleInstantiationSpecification.getFromStrings(className, null, null)));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java
new file mode 100644
index 00000000000..b459acd63e9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.vespa.model.admin.MonitoringSystem;
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * @author tonytv
+ */
+public class StatisticsComponent extends SimpleComponent implements StatisticsConfig.Producer {
+
+ public StatisticsComponent() {
+ super("com.yahoo.statistics.StatisticsImpl");
+ }
+
+ @Override
+ public void getConfig(StatisticsConfig.Builder builder) {
+ MonitoringSystem monitoringSystem = getMonitoringService();
+ if (monitoringSystem != null) {
+ builder.
+ collectionintervalsec(monitoringSystem.getIntervalSeconds().doubleValue()).
+ loggingintervalsec(monitoringSystem.getIntervalSeconds().doubleValue());
+ }
+ builder.values(new StatisticsConfig.Values.Builder().
+ name("query_latency").
+ operations(new StatisticsConfig.Values.Operations.Builder().
+ name(StatisticsConfig.Values.Operations.Name.REGULAR).
+ arguments(new StatisticsConfig.Values.Operations.Arguments.Builder().
+ key("limits").
+ value("25,50,100,500"))));
+
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
new file mode 100644
index 00000000000..558fe368786
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.ComponentGroup;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import static com.yahoo.container.core.ChainsConfig.Chains.Type;
+
+/**
+ * Represents a component chain in the vespa model.
+ * The inner components are represented as children.
+ *
+ * @author tonytv
+ * @author gjoranv
+ */
+public class Chain<T extends ChainedComponent<?>> extends AbstractConfigProducer<AbstractConfigProducer<?>> {
+
+ private final ComponentId componentId;
+ private final ChainSpecification specWithoutInnerComponents;
+ private final ComponentGroup<T> innerComponentsGroup;
+ private static final Type.Enum TYPE = Type.SEARCH;
+
+
+ public Chain(ChainSpecification specWithoutInnerComponents) {
+ super(specWithoutInnerComponents.componentId.stringValue());
+
+ this.componentId = specWithoutInnerComponents.componentId;
+ this.specWithoutInnerComponents = specWithoutInnerComponents;
+ assertNoInnerComponents(specWithoutInnerComponents);
+
+ innerComponentsGroup = new ComponentGroup<>(this, "component");
+ }
+
+ private void assertNoInnerComponents(ChainSpecification specWithoutInnerComponents) {
+ for (ComponentSpecification component : specWithoutInnerComponents.componentReferences) {
+ assert (component.getNamespace() == null);
+ }
+ }
+
+ public void addInnerComponent(T component) {
+ innerComponentsGroup.addComponent(component);
+ }
+
+ public ChainSpecification getChainSpecification() {
+ Collection<ComponentSpecification> innerComponentSpecifications = new ArrayList<>();
+
+ for (ChainedComponent innerComponent : getInnerComponents()) {
+ innerComponentSpecifications.add(innerComponent.getGlobalComponentId().toSpecification());
+ }
+
+ return specWithoutInnerComponents.
+ addComponents(innerComponentSpecifications).
+ setComponentId(getGlobalComponentId());
+ }
+
+ public Collection<T> getInnerComponents() {
+ return innerComponentsGroup.getComponents();
+ }
+
+ public ComponentId getGlobalComponentId() {
+ return componentId;
+ }
+
+ public final ComponentId getId() { return getGlobalComponentId(); }
+
+ public final ComponentId getComponentId() {
+ return componentId;
+ }
+
+ // TODO: remove when DocumentProcessingHandler takes its own version of the chains config as ctor arg
+ public Type.Enum getType() {
+ return TYPE;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java
new file mode 100644
index 00000000000..90579e8b77c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.Component;
+
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ *
+ * Base class for all ChainedComponent config producers.
+ */
+public class ChainedComponent<T extends ChainedComponentModel> extends Component<AbstractConfigProducer<?>, T> {
+
+ public ChainedComponent(T model) {
+ super(model);
+ }
+
+ public void initialize() {}
+
+ @Override
+ public ComponentId getGlobalComponentId() {
+ return model.getComponentId().nestInNamespace(namespace());
+ }
+
+ private ComponentId namespace() {
+ AbstractConfigProducer owner = getParent().getParent();
+ return (owner instanceof Chain) ?
+ ((Chain) owner).getGlobalComponentId() :
+ null;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java
new file mode 100644
index 00000000000..34aee0e217e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.container.core.ChainsConfig;
+
+import java.util.Set;
+
+import static com.yahoo.container.core.ChainsConfig.Components;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ *
+ * Generates config for all the chained components.
+ */
+class ChainedComponentConfigGenerator {
+
+ public static void generate(ChainsConfig.Builder builder, Set<? extends ChainedComponent> components) {
+ for (ChainedComponent<ChainedComponentModel> component : components) {
+ builder.components(getComponent(component));
+ }
+ }
+
+ private static Components.Builder getComponent(ChainedComponent<ChainedComponentModel> component) {
+ return new Components.Builder()
+ .id(component.getGlobalComponentId().stringValue())
+ .dependencies(getDependencies(component));
+ }
+
+ private static Components.Dependencies.Builder getDependencies(ChainedComponent<ChainedComponentModel> component) {
+ Dependencies dependencies = component.model.dependencies;
+ return new Components.Dependencies.Builder()
+ .provides(dependencies.provides())
+ .before(dependencies.before())
+ .after(dependencies.after());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java
new file mode 100644
index 00000000000..bf8e611bfe8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.component.chain.model.ChainsModel;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.container.core.ChainsConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.ComponentGroup;
+import com.yahoo.vespa.model.container.component.ConfigProducerGroup;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Root config producer the whole chains model(contains chains and components).
+ * @author tonytv
+ * @author gjoranv
+ */
+public class Chains<CHAIN extends Chain<?>>
+ extends AbstractConfigProducer<AbstractConfigProducer<?>>
+ implements ChainsConfig.Producer {
+
+ private final ComponentGroup<ChainedComponent<?>> componentGroup;
+ private final ConfigProducerGroup<CHAIN> chainGroup;
+
+ public Chains(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ componentGroup = new ComponentGroup<>(this, "component");
+ chainGroup = new ConfigProducerGroup<>(this, "chain");
+ }
+
+ public void initializeComponents() {
+ for (ChainedComponent component : allComponents()) {
+ component.initialize();
+ }
+ }
+
+ public void validate() throws Exception {
+ ChainsModel chainsModel = new ChainsModel();
+
+ for (CHAIN chain : allChains().allComponents()) {
+ chainsModel.register(chain.getChainSpecification());
+ }
+ for (ChainedComponent<?> component : allComponents()) {
+ chainsModel.register(component.getGlobalComponentId(), component.model);
+ }
+ chainsModel.validate();
+
+ super.validate();
+ }
+
+ public Set<ChainedComponent<?>> allComponents() {
+ Set<ChainedComponent<?>> result = new LinkedHashSet<>();
+ result.addAll(componentGroup.getComponents());
+
+ for (CHAIN chain : allChains().allComponents()) {
+ result.addAll(chain.getInnerComponents());
+ }
+ return result;
+ }
+
+ public ComponentRegistry<ChainedComponent<?>> componentsRegistry() {
+ ComponentRegistry<ChainedComponent<?>> result = new ComponentRegistry<>();
+
+ for (ChainedComponent<?> component: componentGroup.getComponents())
+ result.register(component.getGlobalComponentId(), component);
+
+ for (CHAIN chain : allChains().allComponents()) {
+ for (ChainedComponent<?> component: chain.getInnerComponents()) {
+ result.register(component.getGlobalComponentId(), component);
+ }
+ }
+ return result;
+ }
+
+ public ComponentRegistry<CHAIN> allChains() {
+ ComponentRegistry<CHAIN> allChains = new ComponentRegistry<>();
+ for (CHAIN chain : chainGroup.getComponents()) {
+ allChains.register(chain.getId(), chain);
+ }
+ allChains.freeze();
+ return allChains;
+ }
+
+ public void add(CHAIN chain) {
+ chainGroup.addComponent(chain.getId(), chain);
+ }
+
+ public void add(ChainedComponent outerComponent) {
+ componentGroup.addComponent(outerComponent);
+ }
+
+ @Override
+ public void getConfig(ChainsConfig.Builder builder) {
+ ChainsConfigGenerator.generate(builder, allChains().allComponents());
+ ChainedComponentConfigGenerator.generate(builder, allComponents());
+ }
+
+ public ConfigProducerGroup<ChainedComponent<?>> getComponentGroup() {
+ return componentGroup;
+ }
+
+ protected ConfigProducerGroup<CHAIN> getChainGroup() {
+ return chainGroup;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java
new file mode 100644
index 00000000000..da3f8846974
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.container.core.ChainsConfig;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static com.yahoo.container.core.ChainsConfig.Chains.*;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ *
+ * Generates config for a all the chains.
+ */
+class ChainsConfigGenerator<T extends Chain> {
+
+ public static <T extends Chain> void generate(ChainsConfig.Builder builder, Collection<T> chains) {
+ for (T chain : chains) {
+ builder.chains(getChain(chain));
+ }
+ }
+
+ private static <T extends Chain> ChainsConfig.Chains.Builder getChain(T chain) {
+ ChainSpecification specification = chain.getChainSpecification();
+
+ return new ChainsConfig.Chains.Builder()
+ .type(chain.getType())
+ .id(specification.componentId.stringValue())
+ .components(getComponents(specification.componentReferences))
+ .inherits(getComponents(specification.inheritance.chainSpecifications))
+ .excludes(getComponents(specification.inheritance.excludedComponents))
+ .phases(getPhases(specification.phases()));
+ }
+
+ private static List<String> getComponents(Collection<ComponentSpecification> componentSpecs) {
+ List<String> components = new ArrayList<>();
+ for (ComponentSpecification spec : componentSpecs)
+ components.add(spec.stringValue());
+ return components;
+ }
+
+ private static List<Phases.Builder> getPhases(Collection<Phase> phases) {
+ List<Phases.Builder> builders = new ArrayList<>();
+ for (Phase phase : phases) {
+ builders.add(
+ new Phases.Builder()
+ .id(phase.getName())
+ .before(phase.before())
+ .after(phase.after()));
+ }
+ return builders;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java
new file mode 100644
index 00000000000..ecd1e45c7fa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.core.ChainsConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.Handler;
+
+
+/**
+ * Represents a handler for processing chains.
+ *
+ * @author gjoranv
+ * @since 5.1.7
+ */
+public class ProcessingHandler<CHAINS extends Chains<?>>
+ extends Handler<AbstractConfigProducer<?>>
+ implements ChainsConfig.Producer {
+
+ protected final CHAINS chains;
+
+ public ProcessingHandler(CHAINS chains, String handlerClass) {
+ super(new ComponentModel(BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null), null));
+ this.chains = chains;
+
+ }
+
+ @Override
+ public void getConfig(ChainsConfig.Builder builder) {
+ chains.getConfig(builder);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java
new file mode 100644
index 00000000000..c35c50bfde9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.container.component.chain;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java
new file mode 100644
index 00000000000..4471383c95e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.container.component;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
new file mode 100644
index 00000000000..c52a7375528
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
@@ -0,0 +1,181 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.configserver;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.container.StatisticsConfig;
+import com.yahoo.container.jdisc.config.HealthMonitorConfig;
+import com.yahoo.jdisc.metrics.yamasconsumer.cloud.ScoreBoardConfig;
+import com.yahoo.net.HostName;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions;
+import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer;
+
+import java.util.Optional;
+
+/**
+ * Represents a config server cluster.
+ *
+ * @author lulf
+ * @since 5.15
+ */
+public class ConfigserverCluster extends AbstractConfigProducer
+ implements
+ ZookeeperServerConfig.Producer,
+ ConfigserverConfig.Producer,
+ ScoreBoardConfig.Producer,
+ StatisticsConfig.Producer,
+ HealthMonitorConfig.Producer {
+ private final CloudConfigOptions options;
+ private ContainerCluster containerCluster;
+
+ public ConfigserverCluster(AbstractConfigProducer parent, String subId, CloudConfigOptions options) {
+ super(parent, subId);
+ this.options = options;
+ }
+
+ public void setContainerCluster(ContainerCluster containerCluster) {
+ this.containerCluster = containerCluster;
+
+ // If we are in a config server cluster the correct zone is propagated through cloud config options,
+ // not through config to deployment options (see StandaloneContainerApplication.scala),
+ // so we need to propagate the zone options into the container from here
+ Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment();
+ RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName();
+ containerCluster.setZone(new Zone(environment, region));
+ }
+
+ @Override
+ public void getConfig(ZookeeperServerConfig.Builder builder) {
+ String myhostname = HostName.getLocalhost();
+ int myid = 0;
+ int i = 0;
+ for (ConfigServer server : getConfigServers()) {
+ if (server.hostName.equals(myhostname)) {
+ myid = i;
+ }
+ builder.server(getZkServer(server, i));
+ i++;
+ }
+ builder.myid(myid);
+ if (options.zookeeperClientPort().isPresent()) {
+ builder.clientPort(options.zookeeperClientPort().get());
+ }
+ }
+
+ @Override
+ public void getConfig(ConfigserverConfig.Builder builder) {
+ for (String pluginDir : getConfigModelPluginDirs()) {
+ builder.configModelPluginDir(pluginDir);
+ }
+ if (options.sessionLifeTimeSecs().isPresent()) {
+ builder.sessionLifetime(options.sessionLifeTimeSecs().get());
+ }
+ if (options.zookeeperBarrierTimeout().isPresent()) {
+ builder.zookeeper(new ConfigserverConfig.Zookeeper.Builder().barrierTimeout(options.zookeeperBarrierTimeout().get()));
+ }
+ if (options.rpcPort().isPresent()) {
+ builder.rpcport(options.rpcPort().get());
+ }
+ if (options.multiTenant().isPresent()) {
+ builder.multitenant(options.multiTenant().get());
+ }
+ if (options.payloadCompressionType().isPresent()) {
+ builder.payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.valueOf(options.payloadCompressionType().get()));
+ }
+ for (ConfigServer server : getConfigServers()) {
+ ConfigserverConfig.Zookeeperserver.Builder zkBuilder = new ConfigserverConfig.Zookeeperserver.Builder();
+ zkBuilder.hostname(server.hostName);
+ if (options.zookeeperClientPort().isPresent()) {
+ zkBuilder.port(options.zookeeperClientPort().get());
+ }
+ builder.zookeeperserver(zkBuilder);
+ }
+ if (options.environment().isPresent()) {
+ builder.environment(options.environment().get());
+ }
+ if (options.region().isPresent()) {
+ builder.region(options.region().get());
+ }
+ if (options.defaultFlavor().isPresent()) {
+ builder.defaultFlavor(options.defaultFlavor().get());
+ }
+ if (options.defaultAdminFlavor().isPresent()) {
+ builder.defaultAdminFlavor(options.defaultAdminFlavor().get());
+ }
+ if (options.defaultContainerFlavor().isPresent()) {
+ builder.defaultContainerFlavor(options.defaultContainerFlavor().get());
+ }
+ if (options.defaultContentFlavor().isPresent()) {
+ builder.defaultContentFlavor(options.defaultContentFlavor().get());
+ }
+
+ builder.serverId(HostName.getLocalhost());
+ if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) {
+ builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort());
+ }
+ if (options.useVespaVersionInRequest().isPresent()) {
+ builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get());
+ } else if (options.multiTenant().isPresent()) {
+ builder.useVespaVersionInRequest(options.multiTenant().get());
+ }
+ if (options.hostedVespa().isPresent()) {
+ builder.hostedVespa(options.hostedVespa().get());
+ }
+ if (options.numParallelTenantLoaders().isPresent()) {
+ builder.numParallelTenantLoaders(options.numParallelTenantLoaders().get());
+ }
+ }
+
+ private String[] getConfigModelPluginDirs() {
+ if (options.configModelPluginDirs().length > 0) {
+ return options.configModelPluginDirs();
+ } else {
+ return new String[]{Defaults.getDefaults().vespaHome() + "lib/jars/config-models"};
+ }
+ }
+
+ private ConfigServer[] getConfigServers() {
+ if (options.allConfigServers().length > 0) {
+ return options.allConfigServers();
+ } else {
+ return new ConfigServer[]{new ConfigServer(HostName.getLocalhost(), Optional.<Integer>empty()) };
+ }
+ }
+
+ private ZookeeperServerConfig.Server.Builder getZkServer(ConfigServer server, int id) {
+ ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder();
+ if (options.zookeeperElectionPort().isPresent()) {
+ builder.electionPort(options.zookeeperElectionPort().get());
+ }
+ if (options.zookeeperQuorumPort().isPresent()) {
+ builder.quorumPort(options.zookeeperQuorumPort().get());
+ }
+ builder.hostname(server.hostName);
+ builder.id(id);
+ return builder;
+ }
+
+ @Override
+ public void getConfig(ScoreBoardConfig.Builder builder) {
+ builder.applicationName("configserver");
+ builder.flushTime(60);
+ builder.step(60);
+ }
+
+ @Override
+ public void getConfig(StatisticsConfig.Builder builder) {
+ builder.collectionintervalsec(60.0);
+ builder.loggingintervalsec(60.0);
+ }
+
+ @Override
+ public void getConfig(HealthMonitorConfig.Builder builder) {
+ builder.snapshot_interval(60.0);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
new file mode 100644
index 00000000000..3bb4f9d09f5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.configserver.option;
+
+import java.util.Optional;
+
+/**
+ * @author tonytv
+ */
+public interface CloudConfigOptions {
+
+ class ConfigServer {
+ public final String hostName;
+ public final Optional<Integer> port;
+
+ public ConfigServer(String hostName, Optional<Integer> port) {
+ this.hostName = hostName;
+ this.port = port;
+ }
+ }
+
+
+ Optional<Integer> rpcPort();
+ Optional<Boolean> multiTenant();
+ Optional<Boolean> hostedVespa();
+
+ ConfigServer[] allConfigServers();
+ Optional<Integer> zookeeperClientPort();
+ String[] configModelPluginDirs();
+ Optional<Long> sessionLifeTimeSecs();
+
+ //TODO: which unit?
+ Optional<Long> zookeeperBarrierTimeout();
+ Optional<Integer> zookeeperElectionPort();
+ Optional<Integer> zookeeperQuorumPort();
+ Optional<String> payloadCompressionType();
+ Optional<String> environment();
+ Optional<String> region();
+ Optional<String> defaultFlavor();
+ Optional<String> defaultAdminFlavor();
+ Optional<String> defaultContainerFlavor();
+ Optional<String> defaultContentFlavor();
+ Optional<Boolean> useVespaVersionInRequest();
+ Optional<Integer> numParallelTenantLoaders();
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java
new file mode 100644
index 00000000000..a6f5bfa895a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author tonytv
+ */
+@ExportPackage
+package com.yahoo.vespa.model.container.configserver.option;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java
new file mode 100644
index 00000000000..a1c98396ffc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java
@@ -0,0 +1,178 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.docproc;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.config.docproc.DocprocConfig;
+import com.yahoo.container.jdisc.config.SessionConfig;
+import com.yahoo.container.jdisc.ContainerMbusConfig;
+import com.yahoo.config.docproc.SchemamappingConfig;
+import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.ContainerSubsystem;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author einarmr
+ * @author gjoranv
+ * @since 5.1.9
+ */
+public class ContainerDocproc extends ContainerSubsystem<DocprocChains>
+ implements
+ ContainerMbusConfig.Producer,
+ SchemamappingConfig.Producer,
+ DocprocConfig.Producer
+{
+ public final Options options;
+
+ // Whether or not to prefer sending to a local node.
+ private boolean preferLocalNode = false;
+
+ // The number of nodes to use per client.
+ private int numNodesPerClient = 0;
+
+ private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>();
+
+ public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) {
+ this(cluster, chains, new Options(false, null, null, null, null, null, null));
+ }
+
+ public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) {
+ this(cluster, chains, options, true);
+ }
+
+ private void addSource(
+ final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) {
+ final MbusClient mbusClient = new MbusClient(name, type);
+ mbusClient.addClientBindings("mbus://*/" + mbusClient.getSessionName());
+ cluster.addComponent(mbusClient);
+ }
+
+ public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) {
+ super(chains);
+ assert (options != null) : "Null Options for " + this + " under cluster " + cluster.getName();
+ this.options = options;
+
+ if (addSourceClientProvider) {
+ addSource(cluster, "source", SessionConfig.Type.SOURCE);
+ addSource(cluster, MbusRequestContext.internalNoThrottledSource, SessionConfig.Type.INTERNAL);
+ }
+ }
+
+ public boolean isCompressDocuments() {
+ return options.compressDocuments;
+ }
+
+ public boolean isPreferLocalNode() {
+ return preferLocalNode;
+ }
+
+ public void setPreferLocalNode(boolean preferLocalNode) {
+ this.preferLocalNode = preferLocalNode;
+ }
+
+ public int getNumNodesPerClient() {
+ return numNodesPerClient;
+ }
+
+ public void setNumNodesPerClient(int numNodesPerClient) {
+ this.numNodesPerClient = numNodesPerClient;
+ }
+
+ @Override
+ public void getConfig(ContainerMbusConfig.Builder builder) {
+ builder.maxpendingcount(getMaxMessagesInQueue());
+ if (getMaxQueueMbSize() != null)
+ builder.maxpendingsize(getMaxQueueMbSize()); //yes, this shall be set in megabytes.
+ }
+
+ private int getMaxMessagesInQueue() {
+ if (options.maxMessagesInQueue != null) {
+ return options.maxMessagesInQueue;
+ }
+
+ //maxmessagesinqueue has not been set for this node. let's try to give a good value anyway:
+ return 2048 * getChains().allChains().allComponents().size();
+ //intentionally high, getMaxQueueMbSize() will probably kick in before this one!
+ }
+
+ @Nullable
+ private Integer getMaxQueueMbSize() {
+ return options.maxQueueMbSize;
+ }
+
+
+ private Integer getMaxQueueTimeMs() {
+ return options.maxQueueTimeMs;
+ }
+
+ @Override
+ public void getConfig(DocprocConfig.Builder builder) {
+ if (getMaxQueueTimeMs() != null) {
+ builder.maxqueuetimems(getMaxQueueTimeMs());
+ }
+ }
+
+ public ProcessingHandler<DocprocChains> getDocprocHandler() {
+ return ((DocprocChains) getChains()).getDocprocHandler();
+ }
+
+ @Override
+ public void getConfig(SchemamappingConfig.Builder builder) {
+ Map<Pair<String, String>, String> allMappings = new HashMap<>();
+ for (DocprocChain chain : getChains().allChains().allComponents()) {
+ for (DocumentProcessor processor : chain.getInnerComponents()) {
+ allMappings.putAll(fieldNameSchemaMap());
+ allMappings.putAll(chain.fieldNameSchemaMap());
+ allMappings.putAll(processor.fieldNameSchemaMap());
+ for (Map.Entry<Pair<String,String>, String> e : allMappings.entrySet()) {
+ String doctype = e.getKey().getFirst();
+ String from = e.getKey().getSecond();
+ String to = e.getValue();
+ builder.fieldmapping(new SchemamappingConfig.Fieldmapping.Builder().
+ chain(chain.getId().stringValue()).
+ docproc(processor.getGlobalComponentId().stringValue()).
+ indocument(from).
+ inprocessor(to).
+ doctype(doctype!=null?doctype:""));
+ }
+ allMappings.clear();
+ }
+ }
+ }
+
+ /**
+ * The field name schema map that applies to this whole chain
+ * @return doctype,from → to
+ */
+ public Map<Pair<String,String>,String> fieldNameSchemaMap() {
+ return fieldNameSchemaMap;
+ }
+
+ public static class Options {
+ // Whether or not to compress documents after processing them.
+ public final boolean compressDocuments;
+
+ public final Integer maxMessagesInQueue;
+ public final Integer maxQueueMbSize;
+ public final Integer maxQueueTimeMs;
+
+ public final Double maxConcurrentFactor;
+ public final Double documentExpansionFactor;
+ public final Integer containerCoreMemory;
+
+ public Options(boolean compressDocuments, Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) {
+ this.compressDocuments = compressDocuments;
+ this.maxMessagesInQueue = maxMessagesInQueue;
+ this.maxQueueMbSize = maxQueueMbSize;
+ this.maxQueueTimeMs = maxQueueTimeMs;
+ this.maxConcurrentFactor = maxConcurrentFactor;
+ this.documentExpansionFactor = documentExpansionFactor;
+ this.containerCoreMemory = containerCoreMemory;
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java
new file mode 100644
index 00000000000..28d0b298f64
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.docproc;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+
+import java.util.Map;
+
+import static com.yahoo.container.core.ChainsConfig.Chains.Type;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocprocChain extends Chain<DocumentProcessor> {
+ private Map<Pair<String, String>, String> fieldNameSchemaMap;
+ private static final Type.Enum TYPE = Type.Enum.DOCPROC;
+
+ public DocprocChain(ChainSpecification specWithoutInnerComponents, Map<Pair<String,String>, String> fieldNameSchemaMap) {
+ super(specWithoutInnerComponents);
+ this.fieldNameSchemaMap = fieldNameSchemaMap;
+ }
+
+ /**
+ * The field name schema map that applies to this whole chain
+ * @return doctype,from → to
+ */
+ public Map<Pair<String,String>,String> fieldNameSchemaMap() {
+ return fieldNameSchemaMap;
+ }
+
+ public String getServiceName() {
+ return getParent().getParent().getParent().getConfigId() + "/" + getSessionName();
+ }
+
+ public String getSessionName() {
+ return "chain." + getComponentId().stringValue();
+ }
+
+ public Type.Enum getType() {
+ return TYPE;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java
new file mode 100644
index 00000000000..e1dbbf185c7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.docproc;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.jdisc.config.SessionConfig;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.chain.Chains;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+
+/**
+ * @author einarmr
+ * @since 5.1.9
+ */
+public class DocprocChains extends Chains<DocprocChain> {
+ private final ProcessingHandler<DocprocChains> docprocHandler;
+
+ public DocprocChains(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ docprocHandler = new ProcessingHandler<>(this, "com.yahoo.docproc.jdisc.DocumentProcessingHandler");
+ addComponent(docprocHandler);
+ }
+
+ public ProcessingHandler<DocprocChains> getDocprocHandler() {
+ return docprocHandler;
+ }
+
+ private void addComponent(Component component) {
+ if (!(getParent() instanceof ContainerCluster)) {
+ return;
+ }
+ ((ContainerCluster) getParent()).addComponent(component);
+ }
+
+
+ public void addServersAndClientsForChains() {
+ if (getParent() instanceof ContainerCluster) {
+ for (DocprocChain chain: getChainGroup().getComponents())
+ addServerAndClientForChain((ContainerCluster) getParent(), chain);
+ }
+ }
+
+ private void addServerAndClientForChain(ContainerCluster cluster, DocprocChain docprocChain) {
+ docprocHandler.addServerBindings("mbus://*/" + docprocChain.getSessionName());
+
+ cluster.addMbusServer(ComponentId.fromString(docprocChain.getSessionName()));
+
+ MbusClient client = new MbusClient(docprocChain.getSessionName(), SessionConfig.Type.INTERMEDIATE);
+ client.addClientBindings("mbus://*/" + client.getSessionName());
+ addComponent(client);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java
new file mode 100644
index 00000000000..789b578eb13
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.docproc;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+import com.yahoo.vespa.model.container.docproc.model.DocumentProcessorModel;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentProcessor extends ChainedComponent<DocumentProcessorModel> {
+
+ public static final String INDEXER = "com.yahoo.docprocs.indexing.IndexingProcessor";
+
+ private final Map<Pair<String, String>, String> fieldNameSchemaMap;
+
+ public DocumentProcessor(DocumentProcessorModel model) {
+ super(model);
+ this.fieldNameSchemaMap = model.fieldNameSchemaMap();
+ }
+
+ /**
+ * The field name schema map that applies to this docproc
+ * @return doctype,from → to
+ */
+ public Map<Pair<String,String>,String> fieldNameSchemaMap() {
+ return fieldNameSchemaMap;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java
new file mode 100644
index 00000000000..87038115e0c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.docproc;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.container.jdisc.config.SessionConfig;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.Handler;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class MbusClient extends Handler<AbstractConfigProducer<?>> implements SessionConfig.Producer {
+ private static final ComponentSpecification CLASSNAME =
+ ComponentSpecification.fromString("com.yahoo.container.jdisc.messagebus.MbusClientProvider");
+
+ private final String sessionName;
+ private final SessionConfig.Type.Enum type;
+
+ public MbusClient(String sessionName, SessionConfig.Type.Enum type) {
+ super(new ComponentModel(new BundleInstantiationSpecification(createId(sessionName), CLASSNAME, null)));
+ this.sessionName = sessionName;
+ this.type = type;
+ }
+
+ private static ComponentId createId(String sessionName) {
+ return ComponentId.fromString(sessionName).nestInNamespace(
+ ComponentId.fromString("MbusClient"));
+ }
+
+ @Override
+ public void getConfig(SessionConfig.Builder sb) {
+ sb.
+ name(sessionName).
+ type(type);
+ }
+
+ public String getSessionName() {
+ return sessionName;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java
new file mode 100644
index 00000000000..1f166540b94
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.docproc.model;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import net.jcip.annotations.Immutable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+@Immutable
+public class DocumentProcessorModel extends ChainedComponentModel {
+ private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>();
+
+ public DocumentProcessorModel(BundleInstantiationSpecification bundleInstantiationSpec, Dependencies dependencies, Map<Pair<String, String>, String> fieldNameSchemaMap) {
+ super(bundleInstantiationSpec, dependencies);
+ this.fieldNameSchemaMap.putAll(fieldNameSchemaMap);
+ }
+
+ /**
+ * The field name schema map that applies to this docproc
+ * @return doctype,from → to
+ */
+ public Map<Pair<String,String>,String> fieldNameSchemaMap() {
+ return fieldNameSchemaMap;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
new file mode 100644
index 00000000000..24aeda9ed84
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.jdisc.http.ConnectorConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import org.w3c.dom.Element;
+
+import static com.yahoo.component.ComponentSpecification.fromString;
+import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.21.0
+ */
+public class ConnectorFactory extends SimpleComponent implements ConnectorConfig.Producer {
+
+ private final String name;
+ private volatile int listenPort;
+ private final Element legacyConfig;
+
+ public ConnectorFactory(final String name, final int listenPort, final Element legacyConfig) {
+ super(new ComponentModel(
+ new BundleInstantiationSpecification(new ComponentId(name),
+ fromString("com.yahoo.jdisc.http.server.jetty.ConnectorFactory"),
+ fromString("jdisc_http_service"))
+
+ ));
+
+
+ this.name = name;
+ this.listenPort = listenPort;
+ this.legacyConfig = legacyConfig;
+ }
+
+ @Override
+ public void getConfig(ConnectorConfig.Builder connectorBuilder) {
+ if (legacyConfig != null) {
+ {
+ Element tcpKeepAliveEnabled = XML.getChild(legacyConfig, "tcpKeepAliveEnabled");
+ if (tcpKeepAliveEnabled != null) {
+ connectorBuilder.tcpKeepAliveEnabled(Boolean.valueOf(XML.getValue(tcpKeepAliveEnabled).trim()));
+ }
+ }
+ {
+ Element tcpNoDelayEnabled = XML.getChild(legacyConfig, "tcpNoDelayEnabled");
+ if (tcpNoDelayEnabled != null) {
+ connectorBuilder.tcpNoDelay(Boolean.valueOf(XML.getValue(tcpNoDelayEnabled).trim()));
+ }
+ }
+ {
+ Element tcpListenBacklogLength = XML.getChild(legacyConfig, "tcpListenBacklogLength");
+ if (tcpListenBacklogLength != null) {
+ connectorBuilder.acceptQueueSize(Integer.parseInt(XML.getValue(tcpListenBacklogLength).trim()));
+ }
+ }
+ {
+ Element idleConnectionTimeout = XML.getChild(legacyConfig, "idleConnectionTimeout");
+ if (idleConnectionTimeout != null) {
+ connectorBuilder.idleTimeout(Double.parseDouble(XML.getValue(idleConnectionTimeout).trim()));
+ }
+ }
+ {
+ Element soLinger = XML.getChild(legacyConfig, "soLinger");
+ if (soLinger != null) {
+
+ connectorBuilder.soLingerTime((int) Double.parseDouble(XML.getValue(soLinger).trim()));
+ }
+ }
+ {
+ Element sendBufferSize = XML.getChild(legacyConfig, "sendBufferSize");
+ if (sendBufferSize != null) {
+ connectorBuilder.outputBufferSize(Integer.parseInt(XML.getValue(sendBufferSize).trim()));
+ }
+ }
+ {
+ Element maxHeaderSize = XML.getChild(legacyConfig, "maxHeaderSize");
+ if (maxHeaderSize != null) {
+ connectorBuilder.headerCacheSize(Integer.parseInt(XML.getValue(maxHeaderSize).trim()));
+ }
+ }
+
+ Element ssl = XML.getChild(legacyConfig, "ssl");
+ Element sslEnabled = XML.getChild(ssl, "enabled");
+ if (ssl != null &&
+ sslEnabled != null &&
+ Boolean.parseBoolean(XML.getValue(sslEnabled).trim())) {
+ ConnectorConfig.Ssl.Builder sslBuilder = new ConnectorConfig.Ssl.Builder();
+ sslBuilder.enabled(true);
+ {
+ Element keyStoreType = XML.getChild(ssl, "keyStoreType");
+ if (keyStoreType != null) {
+ sslBuilder.keyStoreType(KeyStoreType.Enum.valueOf(XML.getValue(keyStoreType).trim()));
+ }
+ }
+ {
+ Element keyStorePath = XML.getChild(ssl, "keyStorePath");
+ if (keyStorePath != null) {
+ sslBuilder.keyStorePath(XML.getValue(keyStorePath).trim());
+ }
+ }
+ {
+ Element trustStorePath = XML.getChild(ssl, "trustStorePath");
+ if (trustStorePath != null) {
+ sslBuilder.trustStorePath(XML.getValue(trustStorePath).trim());
+ }
+ }
+ {
+ Element keyDBKey = XML.getChild(ssl, "keyDBKey");
+ if (keyDBKey != null) {
+ sslBuilder.keyDbKey(XML.getValue(keyDBKey).trim());
+ }
+ }
+ {
+ Element algorithm = XML.getChild(ssl, "algorithm");
+ if (algorithm != null) {
+ sslBuilder.sslKeyManagerFactoryAlgorithm(XML.getValue(algorithm).trim());
+ }
+ }
+ {
+ Element protocol = XML.getChild(ssl, "protocol");
+ if (protocol != null) {
+ sslBuilder.protocol(XML.getValue(protocol).trim());
+ }
+ }
+ connectorBuilder.ssl(sslBuilder);
+ }
+ }
+
+ connectorBuilder.listenPort(listenPort);
+ connectorBuilder.name(name);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getListenPort() {
+ return listenPort;
+ }
+
+ public void setListenPort(int httpPort) {
+ this.listenPort = httpPort;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java
new file mode 100644
index 00000000000..f9a416a6fd6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ */
+public class Filter extends ChainedComponent<ChainedComponentModel> {
+
+ public Filter(ChainedComponentModel model) {
+ super(model);
+ }
+
+ public FilterConfigProvider addAndInjectConfigProvider() {
+ FilterConfigProvider filterConfigProvider = new FilterConfigProvider(model);
+ addComponent(filterConfigProvider);
+ inject(filterConfigProvider);
+ return filterConfigProvider;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java
new file mode 100644
index 00000000000..f38c99bd631
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.component.chain.Chains;
+
+/**
+ * @author tonytv
+ */
+public class FilterChains extends Chains<Chain<Filter>> {
+
+ public FilterChains(AbstractConfigProducer parent) {
+ super(parent, "filters");
+
+ addChild(new SimpleComponent("com.yahoo.container.http.filter.FilterChainRepository"));
+ }
+
+ public boolean hasChain(ComponentId filterChain) {
+ for (Chain<Filter> chain : allChains().allComponents()) {
+ if (chain.getId().equals(filterChain))
+ return true;
+ }
+ return false;
+ }
+
+ public boolean hasChainThatInherits(ComponentId filterChain) {
+ for (Chain<Filter> chain : allChains().allComponents()) {
+ for (ComponentSpecification spec : chain.getChainSpecification().inheritance.chainSpecifications) {
+ if(spec.toId().equals(filterChain))
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java
new file mode 100644
index 00000000000..5c6ce3454a8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.core.http.HttpFilterConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static com.yahoo.container.core.http.HttpFilterConfig.Param;
+
+/**
+ * @author gjoranv
+ * @since 5.1.23
+ */
+public class FilterConfigProvider extends SimpleComponent implements HttpFilterConfig.Producer {
+
+ private static final ComponentSpecification filterConfigProviderClass =
+ ComponentSpecification.fromString(com.yahoo.container.FilterConfigProvider.class.getName());
+
+ private final ChainedComponentModel filterModel;
+ private HashMap<String, String> configMap = new LinkedHashMap<>();
+
+ public FilterConfigProvider(ChainedComponentModel filterModel) {
+ super(new ComponentModel(
+ new BundleInstantiationSpecification(
+ configProviderId(filterModel.getComponentId()),
+ filterConfigProviderClass,
+ null)));
+
+ this.filterModel = filterModel;
+ }
+
+ @Override
+ public void getConfig(HttpFilterConfig.Builder builder) {
+ builder.filterName(filterModel.getComponentId().stringValue())
+ .filterClass(filterModel.getClassId().stringValue());
+
+ for (Map.Entry<String, String> param : configMap.entrySet()) {
+ builder.param(
+ new Param.Builder()
+ .name(param.getKey())
+ .value(param.getValue()));
+ }
+ }
+
+ public String putConfig(String key, String value) {
+ return configMap.put(key, value);
+ }
+
+ static ComponentId configProviderId(ComponentId filterId) {
+ return ComponentId.fromString("filterConfig").nestInNamespace(filterId);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
new file mode 100644
index 00000000000..7c44ce8e44f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.jdisc.config.HttpServerConfig;
+import com.yahoo.jdisc.http.ServerConfig;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+// This can be collapsed into JettyHttpServer now
+/**
+ * @author tonytv
+ */
+public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>>
+ implements HttpServerConfig.Producer, ServerConfig.Producer {
+
+ public static class Binding {
+ public final ComponentSpecification filterId;
+ public final String binding;
+
+ public Binding(ComponentSpecification filterId, String binding) {
+ this.filterId = filterId;
+ this.binding = binding;
+ }
+ }
+
+ private FilterChains filterChains;
+ private JettyHttpServer httpServer;
+ public final List<Binding> bindings;
+
+ public Http(List<Binding> bindings) {
+ super( "http");
+ this.bindings = Collections.unmodifiableList(bindings);
+ }
+
+ public void setFilterChains(FilterChains filterChains) {
+ this.filterChains = filterChains;
+ }
+
+ public FilterChains getFilterChains() {
+ return filterChains;
+ }
+
+ public JettyHttpServer getHttpServer() {
+ return httpServer;
+ }
+
+ public void setHttpServer(JettyHttpServer newServer) {
+ JettyHttpServer oldServer = this.httpServer;
+ this.httpServer = newServer;
+
+ if (oldServer == null && newServer != null) {
+ addChild(newServer);
+ } else if (newServer == null && oldServer != null) {
+ removeChild(oldServer);
+ } else if (newServer == null && oldServer == null) {
+ //do nothing
+ } else {
+ //none of them are null
+ removeChild(oldServer);
+ addChild(newServer);
+ }
+ }
+
+ public void removeAllServers() {
+ setHttpServer(null);
+ }
+
+ public List<Binding> getBindings() {
+ return bindings;
+ }
+
+ @Override
+ public void getConfig(HttpServerConfig.Builder builder) {
+ for (Binding binding: bindings)
+ builder.filter(filterBindings(binding));
+ }
+
+ @Override
+ public void getConfig(ServerConfig.Builder builder) {
+ for (final Binding binding : bindings) {
+ builder.filter(
+ new ServerConfig.Filter.Builder()
+ .id(binding.filterId.stringValue())
+ .binding(binding.binding));
+ }
+ }
+
+ static HttpServerConfig.Filter.Builder filterBindings(Binding binding) {
+ HttpServerConfig.Filter.Builder builder = new HttpServerConfig.Filter.Builder();
+ builder.id(binding.filterId.stringValue()).
+ binding(binding.binding);
+ return builder;
+ }
+
+
+ @Override
+ public void validate() throws Exception {
+ validate(bindings);
+ }
+
+ void validate(Collection<Binding> bindings) {
+ if (!bindings.isEmpty()) {
+ if (filterChains == null)
+ throw new IllegalArgumentException("Null FilterChains is not allowed when there are filter bindings!");
+
+ ComponentRegistry<ChainedComponent<?>> filters = filterChains.componentsRegistry();
+ ComponentRegistry<Chain<Filter>> chains = filterChains.allChains();
+
+ for (Binding binding: bindings) {
+ if (filters.getComponent(binding.filterId) == null && chains.getComponent(binding.filterId) == null)
+ throw new RuntimeException("Can't find filter " + binding.filterId + " for binding " + binding.binding);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
new file mode 100644
index 00000000000..30a9e12bf92
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.jdisc.http.ServerConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static com.yahoo.component.ComponentSpecification.fromString;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.16.0
+ */
+public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer {
+
+ private List<ConnectorFactory> connectorFactories = new ArrayList<>();
+
+ public JettyHttpServer(ComponentId id) {
+ super(new ComponentModel(
+ new BundleInstantiationSpecification(id,
+ fromString("com.yahoo.jdisc.http.server.jetty.JettyHttpServer"),
+ fromString("jdisc_http_service"))
+ ));
+ final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(id);
+ addChild(filterBindingsProviderComponent);
+ inject(filterBindingsProviderComponent);
+ }
+
+ public void addConnector(ConnectorFactory connectorFactory) {
+ connectorFactories.add(connectorFactory);
+ addChild(connectorFactory);
+ }
+
+ public void removeConnector(ConnectorFactory connectorFactory) {
+ if (connectorFactory == null) {
+ return;
+ }
+ removeChild(connectorFactory);
+ connectorFactories.remove(connectorFactory);
+ }
+
+ public List<ConnectorFactory> getConnectorFactories() {
+ return Collections.unmodifiableList(connectorFactories);
+ }
+
+ @Override
+ public void getConfig(ServerConfig.Builder builder) {
+ }
+
+ static ComponentModel providerComponentModel(final ComponentId parentId, String className) {
+ final ComponentSpecification classNameSpec = new ComponentSpecification(
+ className);
+ return new ComponentModel(new BundleInstantiationSpecification(
+ classNameSpec.nestInNamespace(parentId),
+ classNameSpec,
+ null));
+ }
+
+ public static final class FilterBindingsProviderComponent extends SimpleComponent {
+ public FilterBindingsProviderComponent(final ComponentId parentId) {
+ super(providerComponentModel(parentId, "com.yahoo.container.jdisc.FilterBindingsProvider"));
+ }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java
new file mode 100644
index 00000000000..304a6cd5052
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.container.http;
+
+import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java
new file mode 100644
index 00000000000..9f033160f6d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http.xml;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder;
+import com.yahoo.vespa.model.container.http.Filter;
+import com.yahoo.vespa.model.container.http.FilterConfigProvider;
+import org.w3c.dom.Element;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ */
+public class FilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Filter> {
+
+ protected Filter doBuild(AbstractConfigProducer ancestor, Element filterElement) {
+ ChainedComponentModelBuilder modelBuilder = new ChainedComponentModelBuilder(filterElement);
+ Filter filter = new Filter(modelBuilder.build());
+ DomComponentBuilder.addChildren(ancestor, filterElement, filter);
+ addFilterConfig(filterElement, filter);
+
+ return filter;
+ }
+
+ private static void addFilterConfig(Element filterElement, Filter filter) {
+ Element filterConfigElement = XML.getChild(filterElement, "filter-config");
+ if (filterConfigElement == null)
+ return;
+
+ FilterConfigProvider filterConfigProvider = filter.addAndInjectConfigProvider();
+ putFilterConfig(filterConfigElement, filterConfigProvider);
+ }
+
+ private static void putFilterConfig(Element filterConfigElement, FilterConfigProvider filterConfigProvider) {
+ for (Element e : XML.getChildren(filterConfigElement)) {
+ filterConfigProvider.putConfig(e.getTagName(), XML.getValue(e));
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java
new file mode 100644
index 00000000000..65f12cf4e3f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http.xml;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.http.Filter;
+import org.w3c.dom.Element;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType;
+
+/**
+ * @author tonytv
+ */
+public class FilterChainBuilder extends DomChainBuilderBase<Filter, Chain<Filter>> {
+ private static Collection<ComponentType<Filter>> allowedComponentTypes = Collections.singleton(ComponentType.filter);
+
+
+ public FilterChainBuilder(Map<String, ComponentType> outerFilterTypeByComponentName) {
+ super(allowedComponentTypes, outerFilterTypeByComponentName);
+ }
+
+ @Override
+ protected Chain<Filter> buildChain(AbstractConfigProducer ancestor, Element producerSpec, ChainSpecification specWithoutInnerComponents) {
+ return new Chain<>(specWithoutInnerComponents);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java
new file mode 100644
index 00000000000..df38402be91
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http.xml;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase;
+import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+import com.yahoo.vespa.model.container.http.Filter;
+import com.yahoo.vespa.model.container.http.FilterChains;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * @author tonytv
+ */
+public class FilterChainsBuilder extends DomChainsBuilder<Filter, Chain<Filter>, FilterChains> {
+ private static final Collection<ComponentType<Filter>> allowedComponentTypes =
+ Collections.singleton(ComponentType.filter);
+
+ //TODO: simplify
+ private static final Map<String, Class<? extends DomChainBuilderBase<? extends Filter, ? extends Chain<Filter>>>> chainType2BuilderClass =
+ Collections.unmodifiableMap(
+ new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Filter, ? extends Chain<Filter>>>>() {{
+ put("chain", FilterChainBuilder.class); // TODO: remove when 'chain' under 'http' is removed from xml schema
+ put("request-chain", FilterChainBuilder.class);
+ put("response-chain", FilterChainBuilder.class);
+ }});
+
+ public FilterChainsBuilder() {
+ super(null, allowedComponentTypes, null);
+ }
+
+ @Override
+ protected FilterChains newChainsInstance(AbstractConfigProducer parent) {
+ return new FilterChains(parent);
+ }
+
+ @Override
+ protected ChainsBuilder<Filter, Chain<Filter>> readChains(
+ AbstractConfigProducer ancestor,
+ List<Element> allChainsElems, Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) {
+
+ return new ChainsBuilder<>(ancestor, allChainsElems, outerComponentTypeByComponentName, chainType2BuilderClass);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
new file mode 100644
index 00000000000..d3ccefc3e26
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http.xml;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.http.FilterChains;
+import com.yahoo.vespa.model.container.http.Http;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author tonytv
+ * @author gjoranv
+ */
+public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> {
+
+ @Override
+ protected Http doBuild(AbstractConfigProducer ancestor, Element spec) {
+ FilterChains filterChains;
+ List<Http.Binding> bindings = new ArrayList<>();
+
+ Element filteringElem = XML.getChild(spec, "filtering");
+ if (filteringElem != null) {
+ filterChains = new FilterChainsBuilder().build(ancestor, filteringElem);
+ bindings = readFilterBindings(filteringElem);
+ } else {
+ filterChains = new FilterChainsBuilder().newChainsInstance(ancestor);
+ }
+
+ Http http = new Http(bindings);
+ http.setFilterChains(filterChains);
+
+ buildHttpServers(ancestor, http, spec);
+
+ return http;
+ }
+
+ private List<Http.Binding> readFilterBindings(Element filteringSpec) {
+ List<Http.Binding> result = new ArrayList<>();
+
+ for (Element child: XML.getChildren(filteringSpec)) {
+ String tagName = child.getTagName();
+ if ((tagName.equals("request-chain") || tagName.equals("response-chain"))) {
+ ComponentSpecification chainId = XmlHelper.getIdRef(child);
+
+ for (Element bindingSpec: XML.getChildren(child, "binding")) {
+ String binding = XML.getValue(bindingSpec);
+ result.add(new Http.Binding(chainId, binding));
+ }
+ }
+ }
+ return result;
+ }
+
+ private void buildHttpServers(AbstractConfigProducer ancestor, Http http, Element spec) {
+ http.setHttpServer(new JettyHttpServerBuilder().build(ancestor, spec));
+ }
+
+ static int readPort(Element spec, DeployState deployState) {
+ String portString = spec.getAttribute("port");
+
+ int port = Integer.parseInt(portString);
+ if (port < 0)
+ throw new IllegalArgumentException(String.format("Invalid port %d.", port));
+
+ int legalPortInHostedVespa = Container.BASEPORT;
+ if (deployState.isHostedVespa() && port != legalPortInHostedVespa) {
+ deployState.getDeployLogger().log(LogLevel.WARNING,
+ String.format("Trying to set port to %d for http server with id %s. You cannot set port to anything else than %s",
+ port, spec.getAttribute("id"), legalPortInHostedVespa));
+ }
+
+ return port;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
new file mode 100644
index 00000000000..be92c00652f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http.xml;
+
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.http.ConnectorFactory;
+import org.w3c.dom.Element;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.21.0
+ */
+public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ConnectorFactory> {
+ @Override
+ protected ConnectorFactory doBuild(AbstractConfigProducer ancestor, Element serverSpec) {
+ String name = XmlHelper.getIdString(serverSpec);
+ int port = HttpBuilder.readPort(serverSpec, ancestor.getRoot().getDeployState());
+
+ Element legacyServerConfig = XML.getChild(serverSpec, "config");
+ if (legacyServerConfig != null) {
+ String configName = legacyServerConfig.getAttribute("name");
+ if (!configName.equals("container.jdisc.config.http-server")) {
+ legacyServerConfig = null;
+ }
+ }
+ return new ConnectorFactory(name, port, legacyServerConfig);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java
new file mode 100644
index 00000000000..c28849dba3f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.http.xml;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.http.ConnectorFactory;
+import com.yahoo.vespa.model.container.http.JettyHttpServer;
+import org.w3c.dom.Element;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.17.0
+ */
+public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<JettyHttpServer> {
+ @Override
+ protected JettyHttpServer doBuild(AbstractConfigProducer ancestor, Element http) {
+ JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("jdisc-jetty"));
+ for (Element serverSpec: XML.getChildren(http, "server")) {
+ ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(ancestor, serverSpec);
+ jettyHttpServer.addConnector(connectorFactory);
+ }
+
+ return jettyHttpServer;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java
new file mode 100644
index 00000000000..def9442f9e4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.jersey;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.VersionSpecification;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.Servlet;
+
+/**
+ * @author tonytv
+ */
+public class Jersey2Servlet extends Servlet {
+ public static final String BUNDLE = "container-jersey2";
+ public static final String CLASS = "com.yahoo.container.servlet.jersey.JerseyServletProvider";
+
+ private static final ComponentId REST_API_NAMESPACE = ComponentId.fromString("rest-api");
+
+ public Jersey2Servlet(String bindingPath) {
+ super(new ComponentModel(
+ new BundleInstantiationSpecification(idSpecFromPath(bindingPath),
+ ComponentSpecification.fromString(CLASS),
+ ComponentSpecification.fromString(BUNDLE))),
+ bindingPath + "/*");
+ }
+
+ private static ComponentSpecification idSpecFromPath(String path) {
+ return new ComponentSpecification(
+ RestApi.idFromPath(path),
+ VersionSpecification.emptyVersionSpecification,
+ REST_API_NAMESPACE);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java
new file mode 100644
index 00000000000..3c7c9dde86a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.jersey;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.Handler;
+
+/**
+ * @author gjoranv
+ * @since 5.6
+ */
+public class JerseyHandler extends Handler<AbstractConfigProducer<?>> {
+
+ public static final String BUNDLE = "container-jersey";
+ public static final String CLASS = "com.yahoo.container.jdisc.jersey.JerseyHandler";
+
+ public JerseyHandler(String bindingPath) {
+ super(new ComponentModel(bundleSpec(CLASS, BUNDLE, bindingPath)));
+ }
+
+ public static BundleInstantiationSpecification bundleSpec(String className, String bundle, String bindingPath) {
+ return BundleInstantiationSpecification.getFromStrings(
+ className + "-" + RestApi.idFromPath(bindingPath),
+ className,
+ bundle);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java
new file mode 100644
index 00000000000..f546e43318f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.jersey;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.config.jersey.JerseyInitConfig;
+import com.yahoo.vespa.model.container.component.Component;
+
+import java.util.Optional;
+
+/**
+ * @author gjoranv
+ * @since 5.6
+ */
+public class RestApi extends AbstractConfigProducer<AbstractConfigProducer<?>> implements
+ JerseyInitConfig.Producer
+{
+ public final boolean isJersey2;
+ private final String bindingPath;
+ private final Component<?, ?> jerseyHandler;
+ private RestApiContext restApiContext;
+
+ public RestApi(String bindingPath, boolean isJersey2) {
+ super(idFromPath(bindingPath));
+ this.bindingPath = bindingPath;
+ this.isJersey2 = isJersey2;
+
+ jerseyHandler = isJersey2 ?
+ createJersey2Servlet(this.bindingPath):
+ createJersey1Handler(this.bindingPath);
+ addChild(jerseyHandler);
+ }
+
+ public static String idFromPath(String path) {
+ return path.replaceAll("/", "|");
+ }
+
+ private Jersey2Servlet createJersey2Servlet(String bindingPath) {
+ return new Jersey2Servlet(bindingPath);
+ }
+
+ private static JerseyHandler createJersey1Handler(String bindingPath) {
+ JerseyHandler jerseyHandler = new JerseyHandler(bindingPath);
+ jerseyHandler.addServerBindings(getBindings(bindingPath));
+ return jerseyHandler;
+ }
+
+ public String getBindingPath() {
+ return bindingPath;
+ }
+
+ @Override
+ public void getConfig(JerseyInitConfig.Builder builder) {
+ builder.jerseyMapping(bindingPath);
+ }
+
+ public void setRestApiContext(RestApiContext restApiContext) {
+ this.restApiContext = restApiContext;
+ addChild(restApiContext);
+ jerseyHandler.inject(restApiContext);
+ }
+
+ public RestApiContext getContext() { return restApiContext; }
+
+ public Optional<JerseyHandler> getJersey1Handler() {
+ return isJersey2 ?
+ Optional.empty():
+ Optional.of((JerseyHandler)jerseyHandler);
+ }
+
+ public Optional<Jersey2Servlet> getJersey2Servlet() {
+ return isJersey2 ?
+ Optional.of((Jersey2Servlet)jerseyHandler) :
+ Optional.empty();
+ }
+
+ private static String[] getBindings(String bindingPath) {
+ String bindingWithoutScheme = "://*/" + bindingPath + "/*";
+ return new String[] {"http" + bindingWithoutScheme, "https" + bindingWithoutScheme};
+ }
+
+ public void prepare() {
+ restApiContext.prepare();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java
new file mode 100644
index 00000000000..e18a40e1c6a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.jersey;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.di.config.JerseyBundlesConfig;
+import com.yahoo.container.di.config.JerseyInjectionConfig;
+import com.yahoo.container.di.config.JerseyInjectionConfig.Inject;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.SimpleComponent;
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+
+/**
+ * @author gjoranv
+ * @since 5.16
+ */
+public class RestApiContext extends SimpleComponent implements
+ JerseyBundlesConfig.Producer,
+ JerseyInjectionConfig.Producer
+{
+ private static final Logger log = Logger.getLogger(RestApi.class.getName());
+ public static final String CONTAINER_CLASS = "com.yahoo.container.di.config.RestApiContext";
+
+ private final List<BundleInfo> bundles = new ArrayList<>();
+
+ // class name -> componentId
+ private final Map<String, String> injectComponentForClass = new LinkedHashMap<>();
+
+ private final String bindingPath;
+
+ @Nullable
+ private ContainerCluster containerCluster;
+
+ public RestApiContext(AbstractConfigProducer<?> ancestor, String bindingPath) {
+ super(componentModel(bindingPath));
+ this.bindingPath = bindingPath;
+
+ if (ancestor instanceof ContainerCluster)
+ containerCluster = (ContainerCluster)ancestor;
+
+ }
+
+ private static ComponentModel componentModel(String bindingPath) {
+ return new ComponentModel(BundleInstantiationSpecification.getFromStrings(
+ CONTAINER_CLASS + "-" + RestApi.idFromPath(bindingPath),
+ CONTAINER_CLASS,
+ null));
+ }
+
+ @Override
+ public void getConfig(JerseyBundlesConfig.Builder builder) {
+ builder.bundles(createBundlesConfig(bundles));
+ }
+
+ private List<JerseyBundlesConfig.Bundles.Builder> createBundlesConfig(List<BundleInfo> bundles) {
+ List<JerseyBundlesConfig.Bundles.Builder> builders = new ArrayList<>();
+ for (BundleInfo b : bundles) {
+ builders.add(
+ new JerseyBundlesConfig.Bundles.Builder()
+ .spec(b.spec)
+ .packages(b.getPackagesToScan())
+ );
+ }
+ return builders;
+ }
+
+ public void addBundles(Collection<BundleInfo> newBundles) {
+ bundles.addAll(newBundles);
+ }
+
+ @Override
+ public void getConfig(JerseyInjectionConfig.Builder builder) {
+ for (Map.Entry<String, String> i : injectComponentForClass.entrySet()) {
+ builder.inject(new Inject.Builder()
+ .forClass(i.getKey())
+ .instance(i.getValue()));
+ }
+ }
+
+ public void addInjections(Map<String, String> injections) {
+ injectComponentForClass.putAll(injections);
+ }
+
+ @Override
+ public void validate() throws Exception {
+ super.validate();
+
+ if (bundles.isEmpty())
+ log.warning("No bundles in rest-api '" + bindingPath +
+ "' - components will only be loaded from classpath.");
+ }
+
+ public void prepare() {
+ if (containerCluster == null) return;
+
+ containerCluster.getAllComponents().stream().
+ filter(isCycleGeneratingComponent.negate()).
+ forEach(this::inject);
+ }
+
+
+ /*
+ * Example problem
+ *
+ * RestApiContext -> ApplicationStatusHandler -> ComponentRegistry<HttpServer> -> JettyHttpServer -> ComponentRegistry<Jersey2Servlet> -> RestApiContext
+ */
+ private Predicate<Component> isCycleGeneratingComponent = component -> {
+ switch (component.getClassId().getName()) {
+ case CONTAINER_CLASS:
+ case JerseyHandler.CLASS:
+ case Jersey2Servlet.CLASS:
+ case "com.yahoo.jdisc.http.server.jetty.JettyHttpServer":
+ case "com.yahoo.container.handler.observability.ApplicationStatusHandler":
+ return true;
+ default:
+ return false;
+ }
+ };
+
+ public static class BundleInfo {
+ // SymbolicName[:Version]
+ public final String spec;
+
+ private final List<String> packagesToScan = new ArrayList<>();
+
+ public BundleInfo(String spec) {
+ this.spec = spec;
+ }
+
+ public List<String> getPackagesToScan() {
+ return packagesToScan;
+ }
+
+ public void addPackageToScan(String pkg) {
+ packagesToScan.add(pkg);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java
new file mode 100644
index 00000000000..ed7239e0bbf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.jersey.xml;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.jersey.RestApi;
+import com.yahoo.vespa.model.container.jersey.RestApiContext;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute;
+
+/**
+ * @author gjoranv
+ * @since 5.6
+ */
+public class RestApiBuilder extends VespaDomBuilder.DomConfigProducerBuilder<RestApi> {
+
+ @Override
+ protected RestApi doBuild(AbstractConfigProducer ancestor, Element spec) {
+ String bindingPath = spec.getAttribute("path");
+ boolean jersey2 = Boolean.parseBoolean(getOptionalAttribute(spec, "jersey2").orElse("false"));
+ RestApi restApi = new RestApi(bindingPath, jersey2);
+
+ restApi.setRestApiContext(
+ createRestApiContext(ancestor, spec, bindingPath));
+ return restApi;
+ }
+
+ private RestApiContext createRestApiContext(AbstractConfigProducer ancestor, Element spec, String bindingPath) {
+ RestApiContext restApiContext = new RestApiContext(ancestor, bindingPath);
+
+ restApiContext.addBundles(getBundles(spec));
+
+ return restApiContext;
+ }
+
+ private List<RestApiContext.BundleInfo> getBundles(Element spec) {
+ List<RestApiContext.BundleInfo> bundles = new ArrayList<>();
+ for (Element bundleElement : XML.getChildren(spec, "components")) {
+ bundles.add(getBundle(bundleElement));
+ }
+ return bundles;
+ }
+
+ private RestApiContext.BundleInfo getBundle(Element bundleElement) {
+ RestApiContext.BundleInfo bundle = new RestApiContext.BundleInfo(
+ bundleElement.getAttribute("bundle"));
+
+ for (Element packageElement : XML.getChildren(bundleElement, "package"))
+ bundle.addPackageToScan(XML.getValue(packageElement));
+
+ return bundle;
+ }
+
+ // TODO: use for naming injected components instead
+ private Map<String, String> getInjections(Element spec) {
+ Map<String, String> injectForClass = new LinkedHashMap<>();
+ for (Element injectElement : XML.getChildren(spec, "inject")) {
+ injectForClass.put(injectElement.getAttribute("for-class"),
+ injectElement.getAttribute("component"));
+ }
+ return injectForClass;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java
new file mode 100644
index 00000000000..74acf6dcc7f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.container;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java
new file mode 100644
index 00000000000..ae7bcb64f66
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.processing;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+
+/**
+ * Represents a processing chain in the config model
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class ProcessingChain extends Chain<Processor> {
+
+ public ProcessingChain(ChainSpecification specWithoutInnerProcessors) {
+ super(specWithoutInnerProcessors);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java
new file mode 100644
index 00000000000..032aacccca8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.processing;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.chain.Chains;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Root config producer for processing
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class ProcessingChains extends Chains<ProcessingChain> {
+ public static final String[] defaultBindings = new String[]
+ {"http://*/processing/*", "https://*/processing/*"};
+
+
+ public ProcessingChains(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java
new file mode 100644
index 00000000000..b16243576f9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.processing;
+
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+
+/**
+ * Representation of a Processor in the configuration model
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @since 5.1.6
+ */
+public class Processor extends ChainedComponent<ChainedComponentModel> {
+
+ public Processor(ChainedComponentModel model) {
+ super(model);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java
new file mode 100644
index 00000000000..88d4a0c8599
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+
+/**
+ * @author gjoranv
+ */
+public class ContainerHttpGateway extends Container {
+
+ public ContainerHttpGateway(ContainerCluster parent, String name, int wantedPort) {
+ super(parent, name);
+
+ // TODO: when this class is removed, all ports for the gateway will map to standard container ports
+ // this is just a tjuvtriks to keep the old gateway port allocation for now.
+ setBasePort(wantedPort);
+ }
+
+ @Override
+ public String getServiceType() { return "container-httpgateway"; }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
new file mode 100644
index 00000000000..0caa84390f8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -0,0 +1,204 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.binaryprefix.BinaryPrefix;
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.prelude.fastsearch.FS4ResourcePool;
+import com.yahoo.prelude.semantics.SemanticRulesConfig;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.ContainerSubsystem;
+import com.yahoo.vespa.model.container.search.searchchain.HttpProvider;
+import com.yahoo.vespa.model.container.search.searchchain.LocalProvider;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import com.yahoo.vespa.model.search.*;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.search.config.QrStartConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.search.query.profile.config.QueryProfilesConfig;
+import com.yahoo.search.pagetemplates.PageTemplatesConfig;
+
+import java.util.*;
+
+/**
+ * @author gjoranv
+ * @author tonytv
+ * @since 5.1.9
+ */
+public class ContainerSearch extends ContainerSubsystem<SearchChains>
+ implements
+ IndexInfoConfig.Producer,
+ IlscriptsConfig.Producer,
+ QrSearchersConfig.Producer,
+ QrStartConfig.Producer,
+ QueryProfilesConfig.Producer,
+ SemanticRulesConfig.Producer,
+ PageTemplatesConfig.Producer
+{
+ private final List<AbstractSearchCluster> systems = new LinkedList<>();
+ private Options options = null;
+
+ //For legacy qrs clusters only.
+ private BinaryScaledAmount totalCacheSize = new BinaryScaledAmount();
+
+ private QueryProfiles queryProfiles;
+ private SemanticRules semanticRules;
+ private PageTemplates pageTemplates;
+ private boolean hostedVespa = false;
+
+ public ContainerSearch(ContainerCluster cluster, SearchChains chains, Options options) {
+ super(chains);
+ this.options = options;
+ this.hostedVespa = cluster.isHostedVespa();
+ cluster.addComponent(getFS4ResourcePool());
+ }
+
+ private Component<?, ComponentModel> getFS4ResourcePool() {
+ BundleInstantiationSpecification spec = BundleInstantiationSpecification.
+ getInternalSearcherSpecificationFromStrings(FS4ResourcePool.class.getName(), null);
+ return new Component<>(new ComponentModel(spec));
+ }
+
+ public void connectSearchClusters(Map<String, AbstractSearchCluster> searchClusters) {
+ systems.addAll(searchClusters.values());
+ initializeSearchChains(searchClusters);
+ }
+
+ // public for testing
+ public void initializeSearchChains(Map<String, ? extends AbstractSearchCluster> searchClusters) {
+ getChains().initialize(searchClusters, totalCacheSize);
+
+ QrsCache defaultCacheOptions = getOptions().cacheSettings.get("");
+ if (defaultCacheOptions != null) {
+ for (LocalProvider localProvider: getChains().localProviders()) {
+ localProvider.setCacheSize(defaultCacheOptions.size);
+ }
+ }
+
+ for (LocalProvider localProvider: getChains().localProviders()) {
+ QrsCache cacheOptions = getOptions().cacheSettings.get(localProvider.getClusterName());
+ if (cacheOptions != null) {
+ localProvider.setCacheSize(cacheOptions.size);
+ }
+ }
+ }
+
+ public void setTotalCacheSize(BinaryScaledAmount totalCacheSize) {
+ this.totalCacheSize = totalCacheSize;
+ }
+
+ public void setQueryProfiles(QueryProfiles queryProfiles) {
+ this.queryProfiles = queryProfiles;
+ }
+
+ public void setSemanticRules(SemanticRules semanticRules) {
+ this.semanticRules = semanticRules;
+ }
+
+ public void setPageTemplates(PageTemplates pageTemplates) {
+ this.pageTemplates = pageTemplates;
+ }
+
+ @Override
+ public void getConfig(QueryProfilesConfig.Builder builder) {
+ if (queryProfiles!=null) queryProfiles.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(SemanticRulesConfig.Builder builder) {
+ if (semanticRules!=null) semanticRules.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(PageTemplatesConfig.Builder builder) {
+ if (pageTemplates!=null) pageTemplates.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(QrStartConfig.Builder qsB) {
+ final QrStartConfig.Jvm.Builder internalBuilder = new QrStartConfig.Jvm.Builder();
+ if (hostedVespa) {
+ internalBuilder.heapSizeAsPercentageOfPhysicalMemory(33);
+ }
+ qsB.jvm(internalBuilder.directMemorySizeCache(totalCacheSizeMb()));
+ }
+
+ private int totalCacheSizeMb() {
+ if (!totalCacheSize.equals(new BinaryScaledAmount())) {
+ return (int) totalCacheSize.as(BinaryPrefix.mega);
+ } else {
+ return totalHttpProviderCacheSize();
+ }
+ }
+
+ private int totalHttpProviderCacheSize() {
+ int totalCacheSizeMb = 0;
+ for (HttpProvider provider: getChains().httpProviders())
+ totalCacheSizeMb += provider.cacheSizeMB();
+
+ return totalCacheSizeMb;
+ }
+
+ @Override
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ for (AbstractSearchCluster sc : systems) {
+ sc.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ for (AbstractSearchCluster sc : systems) {
+ sc.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(QrSearchersConfig.Builder builder) {
+ for (int i = 0; i < systems.size(); i++) {
+ AbstractSearchCluster sys = findClusterWithId(systems, i);
+ QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder().
+ name(sys.getClusterName());
+ for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) {
+ scB.searchdef(spec.getSearchDefinition().getName());
+ }
+ scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId()));
+ scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName()));
+ if (sys instanceof IndexedSearchCluster) {
+ scB.rowbits(sys.getRowBits());
+ for (Dispatch tld: ((IndexedSearchCluster)sys).getTLDs()) {
+ scB.dispatcher(new QrSearchersConfig.Searchcluster.Dispatcher.Builder().
+ host(tld.getHostname()).
+ port(tld.getDispatchPort()));
+ }
+ } else {
+ scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().
+ routespec(((StreamingSearchCluster)sys).getStorageRouteSpec()));
+ }
+ builder.searchcluster(scB);
+ }
+ }
+
+ private static AbstractSearchCluster findClusterWithId(List<AbstractSearchCluster> clusters, int index) {
+ for (AbstractSearchCluster sys : clusters) {
+ if (sys.getClusterIndex() == index) {
+ return sys;
+ }
+ }
+ throw new IllegalArgumentException("No search cluster with index " + index + " exists");
+ }
+
+ public Options getOptions() {
+ return options;
+ }
+
+ /**
+ * Struct that encapsulates qrserver options.
+ */
+ public static class Options {
+ public Map<String, QrsCache> cacheSettings = new LinkedHashMap<>();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java
new file mode 100644
index 00000000000..11903b55d67
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.search.query.profile.OverridableQueryProfile;
+import com.yahoo.search.query.profile.QueryProfile;
+import com.yahoo.search.query.profile.QueryProfileVariants;
+
+import java.util.*;
+
+/**
+ * Represents a set of query profile variants (more or less) as they were declared -
+ * a helper to produce config, which is also using the "declared" representation
+ *
+ * @author bratseth
+ */
+public class DeclaredQueryProfileVariants {
+
+ private Map<String, VariantQueryProfile> variantQueryProfiles =new LinkedHashMap<>();
+
+ public DeclaredQueryProfileVariants(QueryProfile profile) {
+ // Recreates the declared view (settings per set of dimensions)
+ // from the runtime view (dimension-value pairs per variable)
+ // yes, this is a little backwards, but the complexity of two representations
+ // is contained right here...
+ // TODO: This has become unnecessary as the variants now retains the original structure
+ for (Map.Entry<String,QueryProfileVariants.FieldValues> fieldValueEntry : profile.getVariants().getFieldValues().entrySet()) {
+ for (QueryProfileVariants.FieldValue fieldValue : fieldValueEntry.getValue().asList()) {
+ addVariant(fieldValueEntry.getKey(),fieldValue.getValue(),fieldValue.getDimensionValues().getValues());
+ }
+ }
+
+ for (QueryProfileVariants.FieldValue fieldValue : profile.getVariants().getInherited().asList()) {
+ for (QueryProfile inherited : (List<QueryProfile>)fieldValue.getValue())
+ addVariantInherited(inherited,fieldValue.getDimensionValues().getValues());
+ }
+
+ dereferenceCompoundedVariants(profile,"");
+ }
+
+ private void addVariant(String name,Object value,String[] dimensionValues) {
+ String dimensionString=toCanonicalString(dimensionValues);
+ VariantQueryProfile variant=variantQueryProfiles.get(dimensionString);
+ if (variant==null) {
+ variant=new VariantQueryProfile(dimensionValues);
+ variantQueryProfiles.put(dimensionString,variant);
+ }
+ variant.getValues().put(name,value);
+ }
+
+ private void addVariantInherited(QueryProfile inherited,String[] dimensionValues) {
+ String dimensionString=toCanonicalString(dimensionValues);
+ VariantQueryProfile variant=variantQueryProfiles.get(dimensionString);
+ if (variant==null) {
+ variant=new VariantQueryProfile(dimensionValues);
+ variantQueryProfiles.put(dimensionString,variant);
+ }
+ variant.inherit(inherited);
+ }
+
+ private void dereferenceCompoundedVariants(QueryProfile profile,String prefix) {
+ // A variant of a.b is represented as the value a pointing to an anonymous profile a
+ // having the variants
+ for (Map.Entry<String,Object> entry : profile.declaredContent().entrySet()) {
+ if ( ! (entry.getValue() instanceof QueryProfile)) continue;
+ QueryProfile subProfile=(QueryProfile)entry.getValue();
+ // Export if defined implicitly in this, or if this contains overrides
+ if (!subProfile.isExplicit() || subProfile instanceof OverridableQueryProfile) {
+ String entryPrefix=prefix + entry.getKey() + ".";
+ dereferenceCompoundedVariants(subProfile.getVariants(),entryPrefix);
+ dereferenceCompoundedVariants(subProfile,entryPrefix);
+ }
+ }
+
+ if (profile.getVariants()==null) return;
+ // We need to do the same dereferencing to overridables pointed to by variants of this
+ for (Map.Entry<String,QueryProfileVariants.FieldValues> fieldValueEntry : profile.getVariants().getFieldValues().entrySet()) {
+ for (QueryProfileVariants.FieldValue fieldValue : fieldValueEntry.getValue().asList()) {
+ if ( ! (fieldValue.getValue() instanceof QueryProfile)) continue;
+ QueryProfile subProfile=(QueryProfile)fieldValue.getValue();
+ // Export if defined implicitly in this, or if this contains overrides
+ if (!subProfile.isExplicit() || subProfile instanceof OverridableQueryProfile) {
+ String entryPrefix=prefix + fieldValueEntry.getKey() + ".";
+ dereferenceCompoundedVariants(subProfile.getVariants(),entryPrefix);
+ dereferenceCompoundedVariants(subProfile,entryPrefix);
+ }
+ }
+ }
+ }
+
+ private void dereferenceCompoundedVariants(QueryProfileVariants profileVariants,String prefix) {
+ if (profileVariants==null) return;
+ for (Map.Entry<String,QueryProfileVariants.FieldValues> fieldVariant : profileVariants.getFieldValues().entrySet()) {
+ for (QueryProfileVariants.FieldValue variantValue : fieldVariant.getValue().asList()) {
+ addVariant(prefix + fieldVariant.getKey(),variantValue.getValue(),variantValue.getDimensionValues().getValues());
+ }
+ }
+ }
+
+ public String toCanonicalString(String[] dimensionValues) {
+ StringBuilder b=new StringBuilder();
+ for (String dimensionValue : dimensionValues) {
+ if (dimensionValue!=null)
+ b.append(dimensionValue);
+ else
+ b.append("*");
+ b.append(",");
+ }
+ b.deleteCharAt(b.length()-1); // Remove last,
+ return b.toString();
+ }
+
+ public Map<String, VariantQueryProfile> getVariantQueryProfiles() { return variantQueryProfiles; }
+
+ public class VariantQueryProfile {
+
+ private Map<String,Object> values=new LinkedHashMap<>();
+
+ private List<QueryProfile> inherited=new ArrayList<>();
+
+ private String[] dimensionValues;
+
+ public VariantQueryProfile(String[] dimensionValues) {
+ this.dimensionValues=dimensionValues;
+ }
+
+ public String[] getDimensionValues() { return dimensionValues; }
+
+ public void inherit(QueryProfile inheritedProfile) {
+ inherited.add(inheritedProfile);
+ }
+
+ public List<QueryProfile> inherited() { return inherited; }
+
+ public Map<String,Object> getValues() { return values; }
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java
new file mode 100644
index 00000000000..c38001d4a83
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.search.pagetemplates.PageTemplatesConfig;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Owns the page templates to be handed to the qrs nodes.
+ * Owned by a container cluster.
+ *
+ * @author bratseth
+ */
+public class PageTemplates implements Serializable, PageTemplatesConfig.Producer {
+
+ private List<String> pages = new ArrayList<>();
+
+ /** The number of pages in this, for reporting */
+ //private int pages=0;
+
+ /** Validates page templates in an application package. The passed readers will be closed. */
+ public static void validate(ApplicationPackage applicationPackage) {
+ List<NamedReader> pageTemplateFiles=null;
+ try {
+ pageTemplateFiles=applicationPackage.getPageTemplateFiles();
+ new PageTemplateXMLReader().read(pageTemplateFiles,true); // Parse XML for validation only
+ }
+ finally {
+ NamedReader.closeAll(pageTemplateFiles);
+ }
+ }
+
+ /** Creates from an application package. The passed readers will be closed. */
+ public static PageTemplates create(ApplicationPackage applicationPackage) {
+ List<NamedReader> pageTemplateFiles=null;
+ try {
+ pageTemplateFiles=applicationPackage.getPageTemplateFiles();
+ return new PageTemplates(pageTemplateFiles);
+ }
+ finally {
+ NamedReader.closeAll(pageTemplateFiles);
+ }
+ }
+
+ // We are representing these as XML rather than a structured config type because the structure
+ // is not easily representable by config (arbitrary nesting of many types of elements within each other)
+ // and config<->xml generation will not pull its weight in work and possible bugs.
+ // The XML content is already validated when we get here.
+ public PageTemplates(List<NamedReader> readers) {
+ for (NamedReader pageReader : readers) {
+ try {
+ pages.add(contentAsString(pageReader));
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not read page template '" + pageReader.getName() + "'",e);
+ }
+ }
+ }
+
+ @Override
+ public void getConfig(PageTemplatesConfig.Builder builder) {
+ for (String page : pages) {
+ builder.page(page);
+ }
+ }
+
+ private String contentAsString(Reader pageReader) throws IOException {
+ BufferedReader bufferedReader=new BufferedReader(pageReader);
+ StringBuilder b=new StringBuilder();
+ String line;
+ while (null!=(line=bufferedReader.readLine())) {
+ b.append(line);
+ b.append("\n");
+ }
+ return b.toString();
+ }
+
+ @Override
+ public String toString() {
+ return pages.toString();
+ }
+
+ /**
+ * The config produced by this
+ *
+ * @return page templates config
+ */
+ public PageTemplatesConfig getConfig() {
+ PageTemplatesConfig.Builder ptB = new PageTemplatesConfig.Builder();
+ getConfig(ptB);
+ return new PageTemplatesConfig(ptB);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java
new file mode 100644
index 00000000000..72dc2379dbf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+/**
+ * A helper class to wrap a set of QRS cache settings.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class QrsCache {
+ public final Integer size;
+
+ public QrsCache(Integer size) {
+ this.size = size;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
new file mode 100644
index 00000000000..cc57a58ed47
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
@@ -0,0 +1,276 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.search.query.profile.BackedOverridableQueryProfile;
+import com.yahoo.search.query.profile.QueryProfile;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.SubstituteString;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.config.QueryProfilesConfig;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.Map.Entry;
+
+/**
+ * Owns the query profiles and query profile types to be handed to the qrs nodes.
+ * Owned by a container cluster
+ *
+ * @author bratseth
+ */
+public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer {
+
+ private QueryProfileRegistry registry;
+
+ /**
+ * Creates a new set of query profiles for which the config can be returned at request
+ *
+ * @param registry the registry containing the query profiles and types of this.
+ * The given registry cannot be frozen on calling this.
+ */
+ public QueryProfiles(QueryProfileRegistry registry) {
+ this.registry = registry;
+ }
+
+ public QueryProfiles() {
+ this.registry = new QueryProfileRegistry();
+ }
+
+ public QueryProfileRegistry getRegistry() {
+ return registry;
+ }
+
+ @Override
+ public void getConfig(QueryProfilesConfig.Builder builder) {
+ for (QueryProfile profile : registry.allComponents()) {
+ builder.queryprofile(createConfig(profile));
+ }
+ for (QueryProfileType profileType : registry.getTypeRegistry().allComponents()) {
+ if ( ! profileType.isBuiltin())
+ builder.queryprofiletype(createConfig(profileType));
+ }
+ }
+
+ private QueryProfilesConfig.Queryprofile.Builder createConfig(QueryProfile profile) {
+ QueryProfilesConfig.Queryprofile.Builder qB = new QueryProfilesConfig.Queryprofile.Builder();
+ qB.id(profile.getId().stringValue());
+ if (profile.getType() != null)
+ qB.type(profile.getType().getId().stringValue());
+ for (QueryProfile inherited : profile.inherited())
+ qB.inherit(inherited.getId().stringValue());
+
+ if (profile.getVariants()!=null) {
+ for (String dimension : profile.getVariants().getDimensions())
+ qB.dimensions(dimension);
+ }
+ addFieldChildren(qB, profile, "");
+ addVariants(qB, profile);
+ return qB;
+ }
+
+ private void addFieldChildren(QueryProfilesConfig.Queryprofile.Builder qpB, QueryProfile profile, String namePrefix) {
+ List<Map.Entry<String,Object>> content=new ArrayList<>(profile.declaredContent().entrySet());
+ Collections.sort(content,new MapEntryKeyComparator());
+ if (profile.getValue()!=null) { // Add "prefix with dot removed"=value:
+ QueryProfilesConfig.Queryprofile.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Property.Builder();
+ String fullName = namePrefix.substring(0, namePrefix.length() - 1);
+ Object value = profile.getValue();
+ if (value instanceof SubstituteString)
+ value=value.toString(); // Send only types understood by configBuilder downwards
+ propB.name(fullName);
+ if (value!=null) propB.value(value.toString());
+ qpB.property(propB);
+ }
+ for (Map.Entry<String,Object> field : content) {
+ addField(qpB, profile, field, namePrefix);
+ }
+ }
+
+ private void addVariantFieldChildren(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder qpB,
+ QueryProfile profile, String namePrefix) {
+ List<Map.Entry<String,Object>> content=new ArrayList<>(profile.declaredContent().entrySet());
+ Collections.sort(content,new MapEntryKeyComparator());
+ if (profile.getValue()!=null) { // Add "prefix with dot removed"=value:
+ QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder();
+ String fullName = namePrefix.substring(0, namePrefix.length() - 1);
+ Object value = profile.getValue();
+ if (value instanceof SubstituteString)
+ value=value.toString(); // Send only types understood by configBuilder downwards
+ propB.name(fullName);
+ if (value!=null) propB.value(value.toString());
+ qpB.property(propB);
+ }
+ for (Map.Entry<String,Object> field : content) {
+ addVariantField(qpB, field, namePrefix);
+ }
+ }
+
+ private void addField(QueryProfilesConfig.Queryprofile.Builder qpB,
+ QueryProfile profile, Entry<String, Object> field, String namePrefix) {
+ String fullName=namePrefix + field.getKey();
+ if (field.getValue() instanceof QueryProfile) {
+ QueryProfile subProfile=(QueryProfile)field.getValue();
+ if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content
+ addFieldChildren(qpB, subProfile,fullName + ".");
+ }
+ else { // Reference to an id'ed profile - output reference plus any local overrides
+ QueryProfilesConfig.Queryprofile.Reference.Builder refB = new QueryProfilesConfig.Queryprofile.Reference.Builder();
+ createReferenceFieldConfig(refB, profile, fullName, field.getKey(), ((BackedOverridableQueryProfile) subProfile).getBacking().getId().stringValue());
+ qpB.reference(refB);
+ addFieldChildren(qpB, subProfile,fullName + ".");
+ }
+ }
+ else { // a primitive
+ qpB.property(createPropertyFieldConfig(profile, fullName, field.getKey(), field.getValue()));
+ }
+ }
+
+ private void addVariantField(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder qpB,
+ Entry<String, Object> field, String namePrefix) {
+ String fullName=namePrefix + field.getKey();
+ if (field.getValue() instanceof QueryProfile) {
+ QueryProfile subProfile=(QueryProfile)field.getValue();
+ if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content
+ addVariantFieldChildren(qpB, subProfile,fullName + ".");
+ }
+ else { // Reference to an id'ed profile - output reference plus any local overrides
+ QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference.Builder refB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference.Builder();
+ createVariantReferenceFieldConfig(refB, fullName, ((BackedOverridableQueryProfile) subProfile).getBacking().getId().stringValue());
+ qpB.reference(refB);
+ addVariantFieldChildren(qpB, subProfile,fullName + ".");
+ }
+ }
+ else { // a primitive
+ qpB.property(createVariantPropertyFieldConfig(fullName, field.getValue()));
+ }
+ }
+
+ private void addVariants(QueryProfilesConfig.Queryprofile.Builder qB, QueryProfile profile) {
+ if (profile.getVariants()==null) return;
+ DeclaredQueryProfileVariants declaredVariants=new DeclaredQueryProfileVariants(profile);
+ for (DeclaredQueryProfileVariants.VariantQueryProfile variant : declaredVariants.getVariantQueryProfiles().values()) {
+ QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder varB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder();
+ for (String dimensionValue : variant.getDimensionValues()) {
+ if (dimensionValue==null)
+ dimensionValue="*";
+ varB.fordimensionvalues(dimensionValue);
+ }
+ for (QueryProfile inherited : variant.inherited())
+ varB.inherit(inherited.getId().stringValue());
+
+ List<Map.Entry<String,Object>> content=new ArrayList<>(variant.getValues().entrySet());
+ Collections.sort(content,new MapEntryKeyComparator());
+ for (Map.Entry<String,Object> value : content) {
+ addVariantField(varB, value,"");
+ }
+ qB.queryprofilevariant(varB);
+ }
+ }
+
+ private void createReferenceFieldConfig(QueryProfilesConfig.Queryprofile.Reference.Builder refB, QueryProfile profile,
+ String fullName, String localName, String stringValue) {
+ refB.name(fullName);
+ if (stringValue!=null) refB.value(stringValue);
+ Boolean overridable=null;
+ if (profile!=null)
+ overridable=profile.isDeclaredOverridable(localName, null);
+ if (overridable!=null)
+ refB.overridable(""+overridable);
+ }
+
+ private void createVariantReferenceFieldConfig(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference.Builder refB,
+ String fullName, String stringValue) {
+ refB.name(fullName);
+ if (stringValue!=null) refB.value(stringValue);
+ }
+
+ private QueryProfilesConfig.Queryprofile.Property.Builder createPropertyFieldConfig(
+ QueryProfile profile, String fullName, String localName, Object value) {
+ QueryProfilesConfig.Queryprofile.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Property.Builder();
+ Boolean overridable=null;
+ if (value instanceof SubstituteString)
+ value=value.toString(); // Send only types understood by configBuilder downwards
+ propB.name(fullName);
+ if (value!=null) propB.value(value.toString());
+ if (profile!=null)
+ overridable=profile.isDeclaredOverridable(localName, null);
+ if (overridable!=null)
+ propB.overridable(""+overridable);
+ return propB;
+ }
+
+ private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(
+ String fullName, Object value) {
+ QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder();
+ if (value instanceof SubstituteString)
+ value=value.toString(); // Send only types understood by configBuilder downwards
+ propB.name(fullName);
+ if (value!=null) propB.value(value.toString());
+ return propB;
+ }
+
+ private QueryProfilesConfig.Queryprofiletype.Builder createConfig(QueryProfileType profileType) {
+ QueryProfilesConfig.Queryprofiletype.Builder qtB = new QueryProfilesConfig.Queryprofiletype.Builder();
+ qtB.id(profileType.getId().stringValue());
+ if (profileType.isDeclaredStrict())
+ qtB.strict(true);
+ if (profileType.getDeclaredMatchAsPath())
+ qtB.matchaspath(true);
+ for (QueryProfileType inherited : profileType.inherited())
+ qtB.inherit(inherited.getId().stringValue());
+ List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values());
+ Collections.sort(fields);
+ for (FieldDescription field : fields)
+ qtB.field(createConfig(field));
+ return qtB;
+ }
+
+ private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) {
+ QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder();
+ fB.
+ name(field.getName()).
+ type(field.getType().stringValue());
+ if ( ! field.isOverridable())
+ fB.overridable(false);
+ if (field.isMandatory())
+ fB.mandatory(true);
+ String aliases=toSpaceSeparatedString(field.getAliases());
+ if (!aliases.isEmpty())
+ fB.alias(aliases);
+ return fB;
+ }
+
+ public String toSpaceSeparatedString(List<String> list) {
+ StringBuilder b=new StringBuilder();
+ for (Iterator<String> i=list.iterator(); i.hasNext(); ) {
+ b.append(i.next());
+ if (i.hasNext())
+ b.append(" ");
+ }
+ return b.toString();
+ }
+
+ private static class MapEntryKeyComparator implements Comparator<Map.Entry<String,Object>> {
+
+ public int compare(Map.Entry<String,Object> e1,Map.Entry<String,Object> e2) {
+ return e1.getKey().compareTo(e2.getKey());
+ }
+
+ public boolean equals(Object other) {
+ return other instanceof MapEntryKeyComparator;
+ }
+
+ }
+
+ /**
+ * The config produced by this
+ * @return query profiles config
+ */
+ public QueryProfilesConfig getConfig() {
+ QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder();
+ getConfig(qB);
+ return new QueryProfilesConfig(qB);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java
new file mode 100644
index 00000000000..26e6ae75999
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.query.profile.config.QueryProfileXMLReader;
+import com.yahoo.config.application.api.ApplicationPackage;
+
+import java.util.List;
+
+/**
+ * Reads the query profile and query profile types from an application package. The actual reading
+ * is delegated to a {@link com.yahoo.search.query.profile.config.QueryProfileXMLReader}.
+ *
+ * @author bratseth
+ */
+// TODO: Move into QueryProfiles
+public class QueryProfilesBuilder {
+
+ /** Build the set of query profiles for an application package */
+ public QueryProfiles build(ApplicationPackage applicationPackage) {
+ List<NamedReader> queryProfileTypeFiles=null;
+ List<NamedReader> queryProfileFiles=null;
+ try {
+ queryProfileTypeFiles=applicationPackage.getQueryProfileTypeFiles();
+ queryProfileFiles=applicationPackage.getQueryProfileFiles();
+ return new QueryProfiles(new QueryProfileXMLReader().read(queryProfileTypeFiles,queryProfileFiles));
+ }
+ finally {
+ NamedReader.closeAll(queryProfileTypeFiles);
+ NamedReader.closeAll(queryProfileFiles);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java
new file mode 100644
index 00000000000..3969054c1d1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.io.IOUtils;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.prelude.semantics.RuleBase;
+import com.yahoo.prelude.semantics.RuleImporter;
+import com.yahoo.prelude.semantics.parser.ParseException;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Reads the semantic rules from the application package by delegating to RuleConfigDeriver.
+ *
+ * @author bratseth
+ */
+// TODO: Move into SemanticRules
+public class SemanticRuleBuilder {
+
+ /** Build the set of semantic rules for an application package */
+ public SemanticRules build(ApplicationPackage applicationPackage) {
+ List<NamedReader> ruleBaseFiles = null;
+ try {
+ ruleBaseFiles = applicationPackage.getFiles(ApplicationPackage.RULES_DIR, "sr");
+ return new SemanticRules(ruleBaseFiles.stream().map(this::toRuleBaseConfigView).collect(Collectors.toList()));
+ }
+ finally {
+ NamedReader.closeAll(ruleBaseFiles);
+ }
+ }
+
+ private SemanticRules.RuleBase toRuleBaseConfigView(NamedReader reader) {
+ try {
+ String ruleBaseString = IOUtils.readAll(reader.getReader());
+ boolean isDefault = ruleBaseString.contains("@default");
+ return new SemanticRules.RuleBase(toName(reader.getName()), isDefault, ruleBaseString);
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("Could not load rules bases", e);
+ }
+ }
+
+ private String toName(String fileName) {
+ String shortName = new File(fileName).getName();
+ return shortName.substring(0, shortName.length()-".sr".length());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java
new file mode 100644
index 00000000000..4e24ffd8b19
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.prelude.semantics.SemanticRulesConfig;
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Returns the semantic rules config form a set of rule bases.
+ * Owned by a container cluster
+ *
+ * @author bratseth
+ */
+public class SemanticRules implements Serializable, SemanticRulesConfig.Producer {
+
+ private List<RuleBase> ruleBases;
+
+ public SemanticRules(List<RuleBase> ruleBases) {
+ this.ruleBases = ruleBases;
+ }
+
+ @Override
+ public void getConfig(SemanticRulesConfig.Builder builder) {
+ for (RuleBase ruleBase : ruleBases)
+ builder.rulebase(ruleBase.getConfig());
+ }
+
+ /** A config view of a rule base */
+ public static class RuleBase {
+
+ private final String name;
+ private final boolean isDefault;
+ private final String rules;
+
+ public RuleBase(String name, boolean isDefault, String rules) {
+ this.name = name;
+ this.isDefault = isDefault;
+ this.rules = rules;
+ }
+
+ private SemanticRulesConfig.Rulebase.Builder getConfig() {
+ SemanticRulesConfig.Rulebase.Builder ruleBaseBuilder = new SemanticRulesConfig.Rulebase.Builder();
+ ruleBaseBuilder.name(name);
+ ruleBaseBuilder.isdefault(isDefault);
+ ruleBaseBuilder.rules(rules);
+ return ruleBaseBuilder;
+ }
+
+
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java
new file mode 100644
index 00000000000..0dbc7368954
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java
@@ -0,0 +1,269 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.collections.CollectionUtil;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.search.searchchain.model.federation.FederationSearcherModel;
+import com.yahoo.search.federation.FederationConfig;
+import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec;
+import com.yahoo.vespa.model.container.component.Component;
+
+import java.util.*;
+
+/**
+ * Config producer for the FederationSearcher.
+ * @author tonytv
+ */
+public class FederationSearcher extends Searcher<FederationSearcherModel> implements FederationConfig.Producer {
+
+ private final Optional<Component> targetSelector;
+
+ /**
+ * Generates config for a single search chain contained in a target.
+ */
+ private static final class SearchChainConfig {
+ private final SearchChain searchChain;
+ //Zero if not applicable
+ final ComponentId providerId;
+ final FederationOptions targetOptions;
+ final List<String> documentTypes;
+
+ SearchChainConfig(SearchChain searchChain, ComponentId providerId,
+ FederationOptions targetOptions, List<String> documentTypes) {
+ this.searchChain = searchChain;
+ this.providerId = providerId;
+ this.targetOptions = targetOptions;
+ this.documentTypes = documentTypes;
+ }
+
+ public FederationConfig.Target.SearchChain.Builder getSearchChainConfig() {
+ FederationConfig.Target.SearchChain.Builder sB = new FederationConfig.Target.SearchChain.Builder();
+ FederationOptions resolvedOptions = targetOptions.inherit(searchChain.federationOptions());
+ sB.
+ searchChainId(searchChain.getGlobalComponentId().stringValue()).
+ timeoutMillis(resolvedOptions.getTimeoutInMilliseconds()).
+ requestTimeoutMillis(resolvedOptions.getRequestTimeoutInMilliseconds()).
+ optional(resolvedOptions.getOptional()).
+ useByDefault(resolvedOptions.getUseByDefault()).
+ documentTypes(documentTypes);
+ if (providerId != null)
+ sB.providerId(providerId.stringValue());
+ return sB;
+ }
+ }
+
+ /**
+ * One or more search chains that are handled as a single group,
+ * which can be federated to as a single entity.
+ */
+ private static abstract class Target {
+ final ComponentId id;
+ final FederationOptions targetOptions;
+
+ public Target(ComponentId id, FederationOptions targetOptions) {
+ this.id = id;
+ this.targetOptions = targetOptions;
+ }
+
+ public FederationConfig.Target.Builder getTargetConfig() {
+ FederationConfig.Target.Builder tb = new FederationConfig.Target.Builder();
+ tb.
+ id(id.stringValue()).
+ useByDefault(targetOptions.getUseByDefault());
+ getSearchChainsConfig(tb);
+ return tb;
+ }
+
+ protected abstract void getSearchChainsConfig(FederationConfig.Target.Builder tb);
+ }
+
+ private static class SearchChainTarget extends Target {
+ private final SearchChainConfig searchChainConfig;
+
+ public SearchChainTarget(SearchChain searchChain,
+ FederationOptions targetOptions) {
+ super(searchChain.getComponentId(), targetOptions);
+ searchChainConfig = new SearchChainConfig(
+ searchChain,
+ null,
+ targetOptions,
+ searchChain.getDocumentTypes());
+ }
+
+ @Override
+ protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) {
+ tB.searchChain(searchChainConfig.getSearchChainConfig());
+ }
+ }
+
+ private static class SourceGroupTarget extends Target {
+ private final SearchChainConfig leaderConfig;
+ private final List<SearchChainConfig> participantsConfig =
+ new ArrayList<>();
+
+ public SourceGroupTarget(SourceGroup group,
+ FederationOptions targetOptions) {
+ super(group.getComponentId(), applyDefaultSourceGroupOptions(targetOptions));
+
+ leaderConfig = createConfig(group.leader(), targetOptions);
+ for (Source participant : group.participants()) {
+ participantsConfig.add(
+ createConfig(participant, targetOptions));
+ }
+ }
+
+ private static FederationOptions applyDefaultSourceGroupOptions(FederationOptions targetOptions) {
+ FederationOptions defaultSourceGroupOption = new FederationOptions().setUseByDefault(true);
+ return targetOptions.inherit(defaultSourceGroupOption);
+ }
+
+ private SearchChainConfig createConfig(Source source,
+ FederationOptions targetOptions) {
+ return new SearchChainConfig(
+ source,
+ source.getParentProvider().getComponentId(),
+ targetOptions,
+ source.getDocumentTypes());
+ }
+
+ @Override
+ protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) {
+ tB.searchChain(leaderConfig.getSearchChainConfig());
+ for (SearchChainConfig participant : participantsConfig) {
+ tB.searchChain(participant.getSearchChainConfig());
+ }
+ }
+ }
+
+ private static class TargetResolver {
+ final ComponentRegistry<SearchChain> searchChainRegistry;
+ final SourceGroupRegistry sourceGroupRegistry;
+
+ /**
+ * @return true if searchChain.id newer than sourceGroup.id
+ */
+ private boolean newerVersion(SearchChain searchChain,
+ SourceGroup sourceGroup) {
+ if (searchChain == null || sourceGroup == null) {
+ return false;
+ } else {
+ return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId());
+ }
+ }
+
+ /**
+ * @return true if a newer than b
+ */
+ private boolean newerVersion(ComponentId a, ComponentId b) {
+ return a.compareTo(b) > 0;
+ }
+
+
+ TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry,
+ SourceGroupRegistry sourceGroupRegistry) {
+ this.searchChainRegistry = searchChainRegistry;
+ this.sourceGroupRegistry = sourceGroupRegistry;
+ }
+
+ Target resolve(FederationSearcherModel.TargetSpec specification) {
+ SearchChain searchChain = searchChainRegistry.getComponent(
+ specification.sourceSpec);
+ SourceGroup sourceGroup = sourceGroupRegistry.getComponent(
+ specification.sourceSpec);
+
+ if (searchChain == null && sourceGroup == null) {
+ return null;
+ } else if (sourceGroup == null ||
+ newerVersion(searchChain, sourceGroup)) {
+ return new SearchChainTarget(searchChain, specification.federationOptions);
+ } else {
+ return new SourceGroupTarget(sourceGroup, specification.federationOptions);
+ }
+ }
+ }
+
+ private final Map<ComponentId, Target> resolvedTargets =
+ new LinkedHashMap<>();
+
+ public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) {
+ super(searcherModel);
+ this.targetSelector = targetSelector;
+
+ if (targetSelector.isPresent())
+ addChild(targetSelector.get());
+ }
+
+ @Override
+ public void getConfig(FederationConfig.Builder builder) {
+ for (Target target : resolvedTargets.values()) {
+ builder.target(target.getTargetConfig());
+ }
+
+ if (targetSelector.isPresent()) {
+ builder.targetSelector(targetSelector.get().getGlobalComponentId().stringValue());
+ }
+ }
+
+ @Override
+ public void initialize() {
+ initialize(getSearchChains().allChains(), getSearchChains().allSourceGroups());
+ }
+
+ void initialize(ComponentRegistry<SearchChain> searchChainRegistry,
+ SourceGroupRegistry sourceGroupRegistry) {
+ TargetResolver targetResolver = new TargetResolver(
+ searchChainRegistry, sourceGroupRegistry);
+
+ addSourceTargets(targetResolver, model.targets);
+
+ if (model.inheritDefaultSources)
+ addDefaultTargets(targetResolver, searchChainRegistry);
+ }
+
+ private void addSourceTargets(TargetResolver targetResolver, List<TargetSpec> targets) {
+ for (TargetSpec targetSpec : targets) {
+
+ Target target = targetResolver.resolve(targetSpec);
+ if (target == null) {
+ throw new RuntimeException("Can't find source " +
+ targetSpec.sourceSpec +
+ " used as a source for federation '" +
+ getComponentId() + "'");
+ }
+
+ Target duplicate = resolvedTargets.put(target.id, target);
+ if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) {
+ throw new RuntimeException("Search chain " + target.id + " added twice with different federation options"
+ + " to the federation searcher " + getComponentId());
+ }
+ }
+ }
+
+
+ private void addDefaultTargets(TargetResolver targetResolver, ComponentRegistry<SearchChain> searchChainRegistry) {
+ for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) {
+ ComponentSpecification specification = genericTarget.getComponentId().toSpecification();
+
+ //Can't use genericTarget directly, as it might be part of a source group.
+ Target federationTarget = targetResolver.resolve(new TargetSpec(specification, new FederationOptions()));
+ //Do not replace manually added sources, as they might have manually configured federation options
+ if (!resolvedTargets.containsKey(federationTarget.id))
+ resolvedTargets.put(federationTarget.id, federationTarget);
+ }
+ }
+
+
+ private static List<GenericTarget> defaultTargets(Collection<SearchChain> allSearchChains) {
+ Collection<Provider> providers =
+ CollectionUtil.filter(allSearchChains, Provider.class);
+
+ List<GenericTarget> defaultTargets = new ArrayList<>();
+ for (Provider provider : providers) {
+ defaultTargets.addAll(provider.defaultFederationTargets());
+ }
+ return defaultTargets;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
new file mode 100644
index 00000000000..9ed62f15244
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+
+/**
+ * A search chain that is intended to be used for federation (i.e. providers, sources)
+ * @author tonytv
+ */
+abstract public class GenericTarget extends SearchChain {
+
+ private final FederationOptions federationOptions;
+
+ public GenericTarget(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) {
+ super(specWithoutInnerSearchers);
+ this.federationOptions = federationOptions;
+ }
+
+ @Override
+ public FederationOptions federationOptions() {
+ FederationOptions defaultOptions = new FederationOptions().setUseByDefault(useByDefault());
+ return federationOptions.inherit(defaultOptions);
+ }
+
+ /** The value for useByDefault in case the user have not specified any **/
+ abstract protected boolean useByDefault();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java
new file mode 100644
index 00000000000..ef6c6ef4df6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.binaryprefix.BinaryPrefix;
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.cache.QrBinaryCacheConfig;
+import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
+import com.yahoo.search.federation.ProviderConfig;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.search.searchchain.model.federation.HttpProviderSpec;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.search.federation.ProviderConfig.Node;
+import static com.yahoo.search.federation.ProviderConfig.Yca;
+
+
+/**
+ * A provider containing a http searcher.
+ * @author tonytv
+ */
+public class HttpProvider extends Provider implements ProviderConfig.Producer,
+ QrBinaryCacheConfig.Producer,
+ QrBinaryCacheRegionConfig.Producer {
+
+ private final HttpProviderSpec providerSpec;
+
+ //TODO: For backward compatibility only, eliminate this later
+ private BinaryScaledAmount cacheSize;
+
+ public double getCacheWeight() {
+ return providerSpec.cacheWeight;
+ }
+
+ /**
+ * TODO: remove, for backward compatibility only.
+ */
+ public void setCacheSize(BinaryScaledAmount cacheSize) {
+ this.cacheSize = cacheSize;
+ }
+
+ /*
+ * Config producer for the contained http searcher..
+ */
+
+ public HttpProvider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions, HttpProviderSpec providerSpec) {
+ super(specWithoutInnerSearchers, federationOptions);
+ this.providerSpec = providerSpec;
+ }
+
+ @Override
+ public void getConfig(ProviderConfig.Builder builder) {
+ if (providerSpec.path != null)
+ builder.path(providerSpec.path);
+ if (providerSpec.connectionParameters.readTimeout != null)
+ builder.readTimeout(providerSpec.connectionParameters.readTimeout );
+ if (providerSpec.connectionParameters.connectionTimeout != null)
+ builder.connectionTimeout(providerSpec.connectionParameters.connectionTimeout);
+ if (providerSpec.connectionParameters.connectionPoolTimeout != null)
+ builder.connectionPoolTimeout(providerSpec.connectionParameters.connectionPoolTimeout);
+ if (providerSpec.connectionParameters.retries != null)
+ builder.retries(providerSpec.connectionParameters.retries);
+
+ builder.node(getNodes(providerSpec.nodes));
+
+ if (providerSpec.ycaApplicationId != null) {
+ builder.yca(getYca(providerSpec));
+ }
+ }
+
+ private static Yca.Builder getYca(HttpProviderSpec providerSpec) {
+ Yca.Builder yca = new Yca.Builder()
+ .applicationId(providerSpec.ycaApplicationId);
+
+ if (providerSpec.ycaProxy != null) {
+ yca.useProxy(true);
+ if (providerSpec.ycaProxy.host != null) {
+ yca.host(providerSpec.ycaProxy.host)
+ .port(providerSpec.ycaProxy.port);
+ }
+ }
+ if (providerSpec.ycaCertificateTtl != null) yca.ttl(providerSpec.ycaCertificateTtl);
+ if (providerSpec.ycaRetryWait != null) yca.ttl(providerSpec.ycaRetryWait);
+ return yca;
+ }
+
+ private static List<Node.Builder> getNodes(List<HttpProviderSpec.Node> nodeSpecs) {
+ ArrayList<Node.Builder> nodes = new ArrayList<>();
+ for (HttpProviderSpec.Node node : nodeSpecs) {
+ nodes.add(
+ new Node.Builder()
+ .host(node.host)
+ .port(node.port));
+ }
+ return nodes;
+ }
+
+ public int cacheSizeMB() {
+ return providerSpec.cacheSizeMB != null ?
+ providerSpec.cacheSizeMB :
+ (int) cacheSize.as(BinaryPrefix.mega);
+ }
+
+ @Override
+ public void getConfig(QrBinaryCacheConfig.Builder builder) {
+ builder.cache_size(cacheSizeMB());
+ }
+
+ @Override
+ public void getConfig(QrBinaryCacheRegionConfig.Builder builder) {
+ builder.region_size(cacheSizeMB());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java
new file mode 100644
index 00000000000..44f9879230d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.binaryprefix.BinaryPrefix;
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.search.searchchain.model.federation.HttpProviderSpec;
+import java.util.List;
+
+/**
+
+* @author tonytv
+*/
+public class HttpProviderSearcher extends Searcher<ChainedComponentModel> {
+
+
+ public HttpProviderSearcher(ChainedComponentModel model) {
+ super(model);
+ }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
new file mode 100644
index 00000000000..1dd4fb478ec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
@@ -0,0 +1,188 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.prelude.cluster.QrMonitorConfig;
+import com.yahoo.search.config.dispatchprototype.SearchNodesConfig;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.search.config.ClusterConfig;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.search.searchchain.model.federation.LocalProviderSpec;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+import com.yahoo.vespa.model.search.SearchNode;
+
+import java.util.*;
+
+/**
+ * Config producer for search chain responsible for sending queries to a local cluster.
+ *
+ * @author tonytv
+ */
+public class LocalProvider extends Provider implements
+ DocumentdbInfoConfig.Producer,
+ ClusterConfig.Producer,
+ AttributesConfig.Producer,
+ QrMonitorConfig.Producer,
+ RankProfilesConfig.Producer,
+ SearchNodesConfig.Producer,
+ DispatchConfig.Producer {
+
+ private final LocalProviderSpec providerSpec;
+ private volatile AbstractSearchCluster searchCluster;
+
+
+ @Override
+ public void getConfig(ClusterConfig.Builder builder) {
+ assert (searchCluster != null) : "Null search cluster!";
+ builder.clusterId(searchCluster.getClusterIndex());
+ builder.clusterName(searchCluster.getClusterName());
+
+ if (providerSpec.cacheSize != null)
+ builder.cacheSize(providerSpec.cacheSize);
+
+ if (searchCluster.getVisibilityDelay() != null)
+ builder.cacheTimeout(convertVisibilityDelay(searchCluster.getVisibilityDelay()));
+ }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ searchCluster.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ searchCluster.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(QrMonitorConfig.Builder builder) {
+ int requestTimeout = federationOptions().getTimeoutInMilliseconds();
+ if (requestTimeout != -1) {
+ builder.requesttimeout(requestTimeout);
+ }
+ }
+
+ @Override
+ public void getConfig(final SearchNodesConfig.Builder builder) {
+ if (!(searchCluster instanceof IndexedSearchCluster)) {
+ log.warning("Could not build SearchNodesConfig: Only supported for IndexedSearchCluster, got "
+ + searchCluster.getClass().getCanonicalName());
+ return;
+ }
+ final IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster) searchCluster;
+ for (final SearchNode searchNode : indexedSearchCluster.getSearchNodes()) {
+ builder.search_node(
+ new SearchNodesConfig.Search_node.Builder()
+ .host(searchNode.getHostName())
+ .port(searchNode.getDispatchPort()));
+ }
+ }
+
+ private void addProviderSearchers(LocalProviderSpec providerSpec) {
+ for (ChainedComponentModel searcherModel : providerSpec.searcherModels) {
+ addInnerComponent(new Searcher<>(searcherModel));
+ }
+ }
+
+ @Override
+ public ChainSpecification getChainSpecification() {
+ ChainSpecification spec =
+ super.getChainSpecification();
+ return new ChainSpecification(spec.componentId, spec.inheritance, spec.phases(),
+ disableStemmingIfStreaming(spec.componentReferences));
+ }
+
+ //TODO: ugly, restructure this
+ private Set<ComponentSpecification> disableStemmingIfStreaming(Set<ComponentSpecification> searcherReferences) {
+ if (!searchCluster.isStreaming()) {
+ return searcherReferences;
+ } else {
+ Set<ComponentSpecification> filteredSearcherReferences = new LinkedHashSet<>(searcherReferences);
+ filteredSearcherReferences.remove(
+ toGlobalComponentId(
+ new ComponentId("com.yahoo.prelude.querytransform.StemmingSearcher")).
+ toSpecification());
+ return filteredSearcherReferences;
+ }
+ }
+
+ private ComponentId toGlobalComponentId(ComponentId searcherId) {
+ return searcherId.nestInNamespace(getComponentId());
+ }
+
+ public String getClusterName() {
+ return providerSpec.clusterName;
+ }
+
+ public void setSearchCluster(AbstractSearchCluster searchCluster) {
+ assert (this.searchCluster == null);
+ this.searchCluster = searchCluster;
+ }
+
+ public LocalProvider(ChainSpecification specWithoutInnerSearchers,
+ FederationOptions federationOptions,
+ LocalProviderSpec providerSpec) {
+ super(specWithoutInnerSearchers, federationOptions);
+ addProviderSearchers(providerSpec);
+ this.providerSpec = providerSpec;
+ }
+
+ @Override
+ public List<String> getDocumentTypes() {
+ List<String> documentTypes = new ArrayList<>();
+
+ for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) {
+ documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName());
+ }
+
+ return documentTypes;
+ }
+
+ @Override
+ public FederationOptions federationOptions() {
+ Double queryTimeoutInSeconds = searchCluster.getQueryTimeout();
+
+ return queryTimeoutInSeconds == null ?
+ super.federationOptions() :
+ super.federationOptions().inherit(
+ new FederationOptions().setTimeoutInMilliseconds((int) (queryTimeoutInSeconds * 1000)));
+ }
+
+ @Override
+ public void getConfig(DocumentdbInfoConfig.Builder builder) {
+ searchCluster.getConfig(builder);
+ }
+
+ /**
+ * For backward compatibility only, do not use.
+ */
+ public void setCacheSize(Integer cacheSize) {
+ providerSpec.cacheSize = cacheSize;
+ }
+
+ // The semantics of visibility delay in search is deactivating caches if the
+ // delay is less than 1.0, in qrs the cache is deactivated if the delay is 0
+ // (or less). 1.0 seems a little arbitrary, so just doing the conversion
+ // here instead of having two totally independent implementations having to
+ // follow each other down in the modules.
+ private static Double convertVisibilityDelay(Double visibilityDelay) {
+ return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay;
+ }
+
+ @Override
+ public void getConfig(DispatchConfig.Builder builder) {
+ if (!(searchCluster instanceof IndexedSearchCluster)) {
+ log.warning("Could not build DispatchConfig: Only supported for IndexedSearchCluster, got "
+ + searchCluster.getClass().getCanonicalName());
+ return;
+ }
+ ((IndexedSearchCluster) searchCluster).getConfig(builder);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java
new file mode 100644
index 00000000000..8dba2e8b589
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.vespa.model.container.component.ConfigProducerGroup;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Base config producer for search chains that communicate with backends.
+ *
+ * @author tonytv
+ */
+public class Provider extends GenericTarget {
+
+ private ConfigProducerGroup<Source> sources;
+
+ public Provider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) {
+ super(specWithoutInnerSearchers, federationOptions);
+ sources = new ConfigProducerGroup<>(this, "source");
+ }
+
+ public void addSource(Source source) {
+ sources.addComponent(source.getComponentId(), source);
+ }
+
+ public Collection<Source> getSources() {
+ return sources.getComponents();
+ }
+
+ @Override
+ protected boolean useByDefault() {
+ return sources.getComponents().isEmpty();
+ }
+
+ public Collection<? extends GenericTarget> defaultFederationTargets() {
+ if (sources.getComponents().isEmpty()) {
+ return Arrays.asList(this);
+ } else {
+ return sources.getComponents();
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java
new file mode 100644
index 00000000000..148bab3f84a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.vespa.model.container.component.chain.Chain;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents a search chain in the vespa model.
+ *
+ * @author tonytv
+ */
+public class SearchChain extends Chain<Searcher<?>> {
+
+ public SearchChain(ChainSpecification specWithoutInnerSearchers) {
+ super(specWithoutInnerSearchers);
+ }
+
+ public FederationOptions federationOptions() {
+ return new FederationOptions().setUseByDefault(true);
+ }
+
+ //A list of documents types that this search chain provides results for, empty if unknown
+ public List<String> getDocumentTypes() {
+ return Collections.emptyList();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
new file mode 100644
index 00000000000..d13e13b232f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.binaryprefix.BinaryScaledAmount;
+import com.yahoo.collections.CollectionUtil;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.chain.Chains;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.LocalClustersCreator;
+import com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.VespaSearchChainsCreator;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Root config producer of the whole search chains model (contains searchchains and searchers).
+ *
+ * @author tonytv
+ */
+public class SearchChains extends Chains<SearchChain> {
+
+ private final SourceGroupRegistry sourceGroups = new SourceGroupRegistry();
+
+ public SearchChains(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ }
+
+ public void initialize(Map<String, ? extends AbstractSearchCluster> searchClustersByName, BinaryScaledAmount totalProviderCacheSize) {
+ LocalClustersCreator.addDefaultLocalProviders(this, searchClustersByName.keySet());
+ VespaSearchChainsCreator.addVespaSearchChains(this);
+
+ validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers.
+ initializeComponents(searchClustersByName, totalProviderCacheSize);
+ }
+
+ private void initializeComponents(Map<String, ? extends AbstractSearchCluster> searchClustersByName,
+ BinaryScaledAmount totalProviderCacheSize) {
+ setSearchClusterForLocalProvider(searchClustersByName);
+ setCacheSizeForHttpProviders(totalProviderCacheSize);
+ initializeComponents();
+ }
+
+ private void setCacheSizeForHttpProviders(BinaryScaledAmount totalProviderCacheSize) {
+ double totalCacheWeight = 0;
+ for (HttpProvider provider : httpProviders()) {
+ totalCacheWeight += provider.getCacheWeight();
+ }
+
+ final BinaryScaledAmount cacheUnit = totalProviderCacheSize.divide(totalCacheWeight);
+ for (HttpProvider provider : httpProviders()) {
+ provider.setCacheSize(cacheUnit.multiply(provider.getCacheWeight()));
+ }
+ }
+
+ private void setSearchClusterForLocalProvider(Map<String, ? extends AbstractSearchCluster> clusterIndexByName) {
+ for (LocalProvider provider : localProviders()) {
+ AbstractSearchCluster cluster = clusterIndexByName.get(provider.getClusterName());
+ if (cluster == null) {
+ throw new RuntimeException("No searchable content cluster with id '" + provider.getClusterName() + "'");
+ }
+ provider.setSearchCluster(cluster);
+ }
+ }
+
+ private void validateSourceGroups() {
+ for (SourceGroup sourceGroup : sourceGroups.groups()) {
+ sourceGroup.validate();
+
+ if (getChainGroup().getComponentMap().containsKey(sourceGroup.getComponentId())) {
+ throw new RuntimeException(
+ String.format("Same id used for a source and another search chain/provider: '%s'",
+ sourceGroup.getComponentId()));
+ }
+ }
+ }
+
+ @Override
+ public void validate() throws Exception {
+ validateSourceGroups();
+ super.validate();
+ }
+
+ public SourceGroupRegistry allSourceGroups() {
+ return sourceGroups;
+ }
+
+ public Collection<LocalProvider> localProviders() {
+ return CollectionUtil.filter(allChains().allComponents(), LocalProvider.class);
+ }
+
+
+ public Collection<HttpProvider> httpProviders() {
+ return CollectionUtil.filter(allChains().allComponents(), HttpProvider.class);
+ }
+
+ /*
+ * If searchChain is a provider, its sources must already have been attached.
+ */
+ @Override
+ public void add(SearchChain searchChain) {
+ assert !(searchChain instanceof Source);
+
+ super.add(searchChain);
+
+ if (searchChain instanceof Provider) {
+ sourceGroups.addSources((Provider)searchChain);
+ }
+ }
+
+ @Override
+ public ComponentRegistry<SearchChain> allChains() {
+ ComponentRegistry<SearchChain> allChains = new ComponentRegistry<>();
+ for (SearchChain chain : getChainGroup().getComponents()) {
+ allChains.register(chain.getId(), chain);
+ if (chain instanceof Provider)
+ addSources(allChains, (Provider)chain);
+ }
+ allChains.freeze();
+ return allChains;
+ }
+
+ private void addSources(ComponentRegistry<SearchChain> chains, Provider provider) {
+ for (Source source : provider.getSources()) {
+ chains.register(source.getId(), source);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java
new file mode 100644
index 00000000000..653be591be3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
+
+/**
+ * @author gjoranv
+ * @author tonytv
+ */
+public class Searcher<T extends ChainedComponentModel> extends ChainedComponent<T> {
+
+ public Searcher(T model) {
+ super(model);
+ }
+
+ protected SearchChains getSearchChains() {
+ AbstractConfigProducer ancestor = getParent();
+ while (!(ancestor instanceof SearchChains)) {
+ ancestor = ancestor.getParent();
+ }
+ return (SearchChains)ancestor;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java
new file mode 100644
index 00000000000..fe839b904d2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+import java.util.Arrays;
+
+
+/**
+ * Config producer for source, which is contained in a provider.
+ *
+ * @author tonytv
+ */
+public class Source extends GenericTarget {
+
+ //Each source group must have exactly one leader, and an arbitrary number of participants
+ public enum GroupOption {
+ leader,
+ participant
+ }
+
+ public final GroupOption groupOption;
+
+ public Source(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions,
+ GroupOption groupOption) {
+ super(specWithoutInnerSearchers, federationOptions);
+ this.groupOption = groupOption;
+ }
+
+ @Override
+ public FederationOptions federationOptions() {
+ return super.federationOptions().inherit(getParentProvider().federationOptions());
+ }
+
+ @Override
+ protected boolean useByDefault() {
+ return false;
+ }
+
+ public Provider getParentProvider() {
+ AbstractConfigProducer parent = getParent();
+ while (!(parent instanceof Provider)) {
+ parent = parent.getParent();
+ }
+ return (Provider)parent;
+ }
+
+ @Override
+ public ChainSpecification getChainSpecification() {
+ return super.getChainSpecification().addInherits(
+ Arrays.asList(getParentProvider().getComponentId().toSpecification()));
+ }
+
+ public ComponentId getGlobalComponentId() {
+ return getComponentId().nestInNamespace(
+ getParentProvider().getComponentId());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java
new file mode 100644
index 00000000000..799bba45b04
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.ComponentId;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * A set of sources with the same name,
+ * each associated with a different provider,
+ * that fills the same role.
+ * @author tonytv
+ */
+final class SourceGroup {
+ private final ComponentId id;
+ private Source leader;
+ private final Set<Source> participants =
+ new LinkedHashSet<>();
+
+ private void setLeader(Source leader) {
+ assert (validMember(leader));
+
+ if (this.leader != null) {
+ throw new IllegalStateException(
+ "There can not be two default providers for the source "
+ + id);
+ }
+
+ this.leader = leader;
+ }
+
+ private void addParticipant(Source source) {
+ assert (validMember(source));
+ assert (!source.equals(leader));
+
+ if (!participants.add(source)) {
+ throw new RuntimeException("Source added twice to the same group "
+ + source);
+ }
+ }
+
+ private boolean validMember(Source leader) {
+ return leader.getComponentId().equals(id);
+ }
+
+ public ComponentId getComponentId() {
+ return id;
+ }
+
+ public SourceGroup(ComponentId id) {
+ this.id = id;
+ }
+
+ public void add(Source source) {
+ assert source.getComponentId().equals(getComponentId()):
+ "Ids differ: " + source.getComponentId() + " -- " + getComponentId();
+
+ if (Source.GroupOption.leader == source.groupOption) {
+ setLeader(source);
+ } else {
+ addParticipant(source);
+ }
+ }
+
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Source id: ").append(id).append("\n").
+ append("Leader provider: ").append(
+ leader.getParentProvider().getComponentId()).append("\n").
+ append("Participants:");
+
+ for (Source participant : participants) {
+ builder.append("\n").append(" Provider: ").append(
+ participant.getParentProvider().getComponentId());
+ }
+ return builder.toString();
+ }
+
+ public Source leader() {
+ return leader;
+ }
+
+ public Collection<Source> participants() {
+ return Collections.unmodifiableCollection(participants);
+ }
+
+ public void validate() {
+ if (leader == null)
+ throw new IllegalStateException("Missing leader for the source " + getComponentId() +
+ ". One of the sources must use the attribute id instead of idref.");
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java
new file mode 100644
index 00000000000..fda962402d1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ComponentAdaptor;
+import com.yahoo.component.provider.ComponentRegistry;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+
+/**
+ * Owns all the source groups in the search chains model.
+ * @author tonytv
+ */
+class SourceGroupRegistry {
+ private final ComponentRegistry<ComponentAdaptor<SourceGroup>> sourceGroups
+ = new ComponentRegistry<>();
+
+ private void add(Source source) {
+ getGroup(source.getComponentId()).add(source);
+ }
+
+ private SourceGroup getGroup(ComponentId sourceId) {
+ ComponentAdaptor<SourceGroup> group =
+ sourceGroups.getComponent(sourceId);
+ if (group == null) {
+ group = new ComponentAdaptor<>(sourceId,
+ new SourceGroup(sourceId));
+ sourceGroups.register(group.getId(), group);
+ }
+ return group.model;
+ }
+
+ void addSources(Provider provider) {
+ for (Source source : provider.getSources()) {
+ add(source);
+ }
+ }
+
+ public Collection<SourceGroup> groups() {
+ List<SourceGroup> result = new ArrayList<>();
+ for (ComponentAdaptor<SourceGroup> group :
+ sourceGroups.allComponents()) {
+ result.add(group.model);
+ }
+ return result;
+ }
+
+ public SourceGroup getComponent(ComponentSpecification spec) {
+ ComponentAdaptor<SourceGroup> result = sourceGroups.getComponent(spec);
+ return (result != null)?
+ result.model :
+ null;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java
new file mode 100644
index 00000000000..43f4a28ff30
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.search.searchchain.model.federation.LocalProviderSpec;
+import com.yahoo.vespa.model.container.search.searchchain.LocalProvider;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Adds default search chains for all local clusters not mentioned explicitly
+ * @author tonytv
+ */
+public class LocalClustersCreator {
+ static ChainSpecification emptySearchChainSpecification(String componentName) {
+ return new ChainSpecification(
+ new ComponentId(componentName),
+ VespaSearchChainsCreator.inheritsVespaPhases(), //TODO: refactor
+ Collections.<Phase>emptyList(),
+ Collections.<ComponentSpecification>emptySet());
+ }
+
+ static LocalProvider createDefaultLocalProvider(String clusterName) {
+ return new LocalProvider(emptySearchChainSpecification(clusterName), new FederationOptions(),
+ new LocalProviderSpec(clusterName, null));
+ }
+
+ static Set<String> presentClusters(SearchChains searchChains) {
+ Set<String> presentClusters = new LinkedHashSet<>();
+ for (LocalProvider provider : searchChains.localProviders()) {
+ presentClusters.add(provider.getClusterName());
+ }
+ return presentClusters;
+ }
+
+ public static void addDefaultLocalProviders(SearchChains searchChains, Set<String> clusterNames) {
+ Set<String> missingClusters = new LinkedHashSet<>(clusterNames);
+ missingClusters.removeAll(presentClusters(searchChains));
+
+ for (String clusterName : missingClusters) {
+ searchChains.add(createDefaultLocalProvider(clusterName));
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java
new file mode 100644
index 00000000000..ed5fd3b759e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.search.searchchain.PhaseNames;
+import com.yahoo.search.searchchain.model.VespaSearchers;
+import com.yahoo.search.searchchain.model.federation.FederationSearcherModel;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.search.searchchain.*;
+
+import java.util.*;
+
+/**
+ * Creates the search chains vespaPhases, vespa and native.
+ *
+ * <p>TODO: refactor</p>
+ * @author tonytv
+ */
+public class VespaSearchChainsCreator {
+ private static class PhasesCreator {
+ private static Set<String> set(String successor) {
+ return successor == null ? null : new LinkedHashSet<>(Arrays.asList(successor));
+ }
+
+ private static String lastElement(String[] phases) {
+ return phases[phases.length - 1];
+ }
+
+ private static Phase createPhase(String phase, String before) {
+ return new Phase(phase, set(before), null);
+ }
+
+ public static Collection<Phase> linearPhases(String... phases) {
+ List<Phase> result = new ArrayList<>();
+
+ for (int i=0; i < phases.length - 1; ++i) {
+ result.add(
+ createPhase(phases[i], phases[i+1]));
+ }
+
+ if (phases.length > 0) {
+ result.add(
+ createPhase(lastElement(phases), null));
+ }
+
+ return result;
+ }
+ }
+
+ private static Set<ComponentSpecification> noSearcherReferences() {
+ return Collections.emptySet();
+ }
+
+ private static Collection<Phase> noPhases() {
+ return Collections.emptySet();
+ }
+
+ private static ChainSpecification.Inheritance inherits(ComponentId chainId) {
+ Set<ComponentSpecification> inheritsSet = new LinkedHashSet<>();
+ inheritsSet.add(chainId.toSpecification());
+ return new ChainSpecification.Inheritance(inheritsSet, null);
+ }
+
+ static ChainSpecification.Inheritance inheritsVespaPhases() {
+ return inherits(vespaPhasesSpecification().componentId);
+ }
+
+ private static void addInnerSearchers(SearchChain searchChain, Collection<ChainedComponentModel> searcherModels) {
+ for (ChainedComponentModel searcherModel : searcherModels) {
+ searchChain.addInnerComponent(createSearcher(searcherModel));
+ }
+ }
+
+ private static Searcher<? extends ChainedComponentModel> createSearcher(ChainedComponentModel searcherModel) {
+ if (searcherModel instanceof FederationSearcherModel) {
+ return new FederationSearcher((FederationSearcherModel) searcherModel, Optional.<Component>empty());
+ } else {
+ return new Searcher<>(searcherModel);
+ }
+ }
+
+
+ private static ChainSpecification nativeSearchChainSpecification() {
+ return new ChainSpecification(
+ new ComponentId("native"),
+ inheritsVespaPhases(),
+ noPhases(),
+ noSearcherReferences());
+ }
+
+ private static ChainSpecification vespaSearchChainSpecification() {
+ return new ChainSpecification(
+ new ComponentId("vespa"),
+ inherits(nativeSearchChainSpecification().componentId),
+ noPhases(),
+ noSearcherReferences());
+ }
+
+
+ private static ChainSpecification vespaPhasesSpecification() {
+ return new ChainSpecification(
+ new ComponentId("vespaPhases"),
+ new ChainSpecification.Inheritance(null, null),
+ PhasesCreator.linearPhases(
+ PhaseNames.RAW_QUERY,
+ PhaseNames.TRANSFORMED_QUERY,
+ PhaseNames.BLENDED_RESULT,
+ PhaseNames.UNBLENDED_RESULT,
+ PhaseNames.BACKEND),
+ noSearcherReferences());
+ }
+
+ private static SearchChain createVespaPhases() {
+ return new SearchChain(vespaPhasesSpecification());
+ }
+
+ private static SearchChain createNative() {
+ SearchChain nativeChain = new SearchChain(nativeSearchChainSpecification());
+ addInnerSearchers(nativeChain, VespaSearchers.nativeSearcherModels);
+ return nativeChain;
+ }
+
+ private static SearchChain createVespa() {
+ SearchChain vespaChain = new SearchChain(vespaSearchChainSpecification());
+ addInnerSearchers(vespaChain, VespaSearchers.vespaSearcherModels);
+ return vespaChain;
+ }
+
+ public static void addVespaSearchChains(SearchChains searchChains) {
+ searchChains.add(
+ createVespaPhases());
+ searchChains.add(
+ createNative());
+ searchChains.add(
+ createVespa());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
new file mode 100644
index 00000000000..f6352e9ac35
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.core.AccessLogConfig;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.AccessLogComponent;
+import com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType;
+import org.w3c.dom.Element;
+
+import java.util.Optional;
+
+import static com.yahoo.collections.CollectionUtil.firstMatching;
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute;
+
+import static com.yahoo.config.model.builder.xml.XmlHelper.nullIfEmpty;
+
+/**
+ * @author tonytv
+ */
+public class AccessLogBuilder {
+ /*
+ * Valid values for the type attribute in services.xml
+ * Must be kept in sync with containercluster.rnc:AccessLog
+ */
+ private enum AccessLogTypeLiteral {
+ VESPA("vespa"),
+ YAPACHE("yapache"),
+ DISABLED("disabled");
+
+ final String attributeValue;
+
+ AccessLogTypeLiteral(String attributeValue) {
+ this.attributeValue = attributeValue;
+ }
+
+ static AccessLogTypeLiteral fromAttributeValue(String value) {
+ return firstMatching(
+ AccessLogTypeLiteral.values(),
+ typeLiteral -> typeLiteral.attributeValue.equals(value)).get();
+ }
+ }
+
+ private static class DomBuilder extends VespaDomBuilder.DomConfigProducerBuilder<AccessLogComponent> {
+ private final AccessLogType accessLogType;
+
+ public DomBuilder(AccessLogType accessLogType) {
+ this.accessLogType = accessLogType;
+ }
+
+ @Override
+ protected AccessLogComponent doBuild(AbstractConfigProducer ancestor, Element spec) {
+ return new AccessLogComponent(
+ accessLogType,
+ fileNamePattern(spec),
+ rotationInterval(spec),
+ rotationScheme(spec),
+ symlinkName(spec));
+ }
+
+ private String symlinkName(Element spec) {
+ return nullIfEmpty(spec.getAttribute("symlinkName"));
+ }
+
+ private AccessLogConfig.FileHandler.RotateScheme.Enum rotationScheme(Element spec) {
+ return AccessLogComponent.rotateScheme(nullIfEmpty(spec.getAttribute("rotationScheme")));
+ }
+
+ private String rotationInterval(Element spec) {
+ return nullIfEmpty(spec.getAttribute("rotationInterval"));
+ }
+
+ private String fileNamePattern(Element spec) {
+ return nullIfEmpty(spec.getAttribute("fileNamePattern"));
+ }
+ }
+
+ public static Optional<AccessLogComponent> buildIfNotDisabled(ContainerCluster cluster, Element accessLogSpec) {
+ AccessLogTypeLiteral typeLiteral =
+ getOptionalAttribute(accessLogSpec, "type").
+ map(AccessLogTypeLiteral::fromAttributeValue).
+ orElse(AccessLogTypeLiteral.VESPA);
+
+ switch (typeLiteral) {
+ case DISABLED:
+ return Optional.empty();
+ case VESPA:
+ return Optional.of(new DomBuilder(AccessLogType.queryAccessLog).build(cluster, accessLogSpec));
+ case YAPACHE:
+ return Optional.of(new DomBuilder(AccessLogType.yApacheAccessLog).build(cluster, accessLogSpec));
+ default:
+ throw new InconsistentSchemaAndCodeError();
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
new file mode 100644
index 00000000000..544f4f93c69
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.processing.handler.ProcessingHandler;
+import com.yahoo.search.handler.SearchHandler;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * This object builds a bundle instantiation spec from an XML element.
+ *
+ * @author gjoranv
+ */
+public class BundleInstantiationSpecificationBuilder {
+
+ public static BundleInstantiationSpecification build(Element spec, boolean legacyMode) {
+ ComponentSpecification id = XmlHelper.getIdRef(spec);
+ ComponentSpecification classId = getComponentSpecification(spec, "class");
+ ComponentSpecification bundle = getComponentSpecification(spec, "bundle");
+
+ BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle);
+ if ( ! legacyMode) // TODO: Remove?
+ validate(instSpec);
+
+ return bundle == null ? setBundleForKnownClass(instSpec) : instSpec;
+ }
+
+ private static BundleInstantiationSpecification setBundleForKnownClass(BundleInstantiationSpecification spec) {
+ return BundleMapper.getBundle(spec.getClassName()).
+ map(spec::inBundle).
+ orElse(spec);
+ }
+
+
+ private static void validate(BundleInstantiationSpecification instSpec) {
+ List<String> forbiddenClasses = Arrays.asList(
+ "com.yahoo.search.handler.SearchHandler",
+ "com.yahoo.processing.handler.ProcessingHandler");
+
+ for (String forbiddenClass: forbiddenClasses) {
+ if (forbiddenClass.equals(instSpec.getClassName())) {
+ throw new RuntimeException("Setting up " + forbiddenClass + " manually is not supported.");
+ }
+ }
+ }
+
+ //null if missing
+ private static ComponentSpecification getComponentSpecification(Element spec, String attributeName) {
+ return (spec.hasAttribute(attributeName)) ?
+ new ComponentSpecification(spec.getAttribute(attributeName)) :
+ null;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
new file mode 100644
index 00000000000..e09d47e7bc1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
@@ -0,0 +1,132 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.vespa.defaults.Defaults;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author gjoranv
+ * @author lulf
+ * @since 5.45
+ */
+public class BundleMapper {
+
+ public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().vespaHome() + "lib/jars");
+ public static final String searchAndDocprocBundle = "container-search-and-docproc";
+
+ private static final Map<String, String> bundleFromClass;
+ private static final Map<String, Path> bundleFileFromClass;
+
+ public static Optional<String> getBundle(String className) {
+ return Optional.ofNullable(bundleFromClass.get(className));
+ }
+
+ public static Optional<Path> getBundlePath(String className) {
+ return Optional.ofNullable(absoluteBundlePath(bundleFileFromClass.get(className)));
+ }
+
+ public static Path absoluteBundlePath(Path fileName) {
+ if (fileName == null) return null;
+ return LIBRARY_PATH.resolve(fileName);
+ }
+
+ /**
+ * TODO: This is a temporary hack to ensure that users can use our internal components without
+ * specifying the bundle in which the components reside. Ideally, this information
+ * should be generated during vespamodel build time.
+ *
+ * The container_maven_plugin has much of the logic in place, but needs to be extended.
+ */
+ static {
+ bundleFromClass = new HashMap<>();
+ bundleFileFromClass = new HashMap<>();
+
+ bundleFromClass.put("com.yahoo.docproc.AbstractConcreteDocumentFactory", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.docproc.DocumentProcessor", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.docproc.SimpleDocumentProcessor", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.docproc.util.JoinerDocumentProcessor", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.docproc.util.SplitterDocumentProcessor", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.example.TimingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.language.simple.SimpleLinguistics", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.cluster.ClusterSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.fastsearch.FastSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.fastsearch.VespaBackEndSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.CJKSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.CollapsePhraseSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.IndexCombinatorSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.LiteralBoostSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.NoRankingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.NonPhrasingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.NormalizingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.PhrasingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.RecallSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.querytransform.StemmingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.BlendingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.CachingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.DocumentSourceSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.FieldCollapsingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.KeyValueSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.QuerySnapshotSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.QueryValidatingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.QuotingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.searcher.ValidateSortingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.semantics.SemanticSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.statistics.StatisticsSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.prelude.templates.SearchRendererAdaptor", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.Searcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.cluster.ClusterSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.cluster.PingableSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.FederationSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.ForwardingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.http.HTTPClientSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.http.HTTPProviderSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.http.HTTPSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.news.NewsSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.federation.vespa.VespaSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.grouping.GroupingQueryParser", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.grouping.GroupingValidator", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.grouping.vespa.GroupingExecutor", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.handler.SearchWithRendererHandler", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplate", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplateSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.engine.Resolver", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.query.rewrite.QueryRewriteSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.NameRewriter", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.AllLowercasingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.DefaultPositionSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.LegacyCombinator", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.LowercasingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.NGramSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.QueryCombinator", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.querytransform.VespaLowercasingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.rendering.Renderer", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.rendering.SectionedRenderer", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.searchchain.ForkingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.searchchain.example.ExampleSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.searchers.CacheControlSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.statistics.PeakQpsSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.search.statistics.TimingSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.vespa.streamingvisitors.MetricsSearcher", searchAndDocprocBundle);
+ bundleFromClass.put("com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher", searchAndDocprocBundle);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
new file mode 100644
index 00000000000..a82291cd159
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.config.application.Xml;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.application.api.ApplicationPackage;
+
+import com.yahoo.path.Path;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions;
+import com.yahoo.vespa.model.container.configserver.ConfigserverCluster;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+import java.util.List;
+
+/**
+ * Builds the config model for the standalone config server.
+ *
+ * @author lulf
+ * @since 5.16
+ */
+public class ConfigServerContainerModelBuilder extends ContainerModelBuilder {
+
+ private final CloudConfigOptions options;
+ private final static String HOSTED_VESPA_INCLUDE_DIR = "hosted-vespa";
+
+ public ConfigServerContainerModelBuilder(CloudConfigOptions options) {
+ super(true, Networking.enable);
+ this.options = options;
+ }
+
+
+ @Override
+ public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) {
+ ApplicationPackage app = modelContext.getDeployState().getApplicationPackage();
+ if (!app.getFiles(Path.fromString(HOSTED_VESPA_INCLUDE_DIR), ".xml").isEmpty()) {
+ app.validateIncludeDir(HOSTED_VESPA_INCLUDE_DIR);
+ List<Element> configModelElements = Xml.allElemsFromPath(app, HOSTED_VESPA_INCLUDE_DIR);
+ mergeInto(spec, configModelElements);
+ }
+
+ ConfigserverCluster cluster = new ConfigserverCluster(modelContext.getParentProducer(), "configserver", options);
+ super.doBuild(model, spec, modelContext.modifyParent(cluster));
+ cluster.setContainerCluster(model.getCluster());
+ }
+
+ private void mergeInto(Element destination, List<Element> configModelElements) {
+ for (Element jdiscElement: configModelElements) {
+ for (Node child = jdiscElement.getFirstChild(); child != null; child = child.getNextSibling()) {
+ Node copiedNode = destination.getOwnerDocument().importNode(child, true);
+ destination.appendChild(copiedNode);
+ }
+ }
+ }
+
+ @Override
+ protected void addDefaultComponents(ContainerCluster containerCluster) {
+ // To avoid search specific stuff.
+ }
+
+ @Override
+ protected void addDefaultHandlers(ContainerCluster containerCluster) {
+ addDefaultHandlersExceptStatus(containerCluster);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
new file mode 100644
index 00000000000..a8055afeea8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -0,0 +1,576 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.component.Version;
+import com.yahoo.config.application.Xml;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.application.provider.IncludeDirs;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.container.jdisc.config.MetricDefaultsConfig;
+import com.yahoo.search.rendering.RendererRegistry;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.builder.xml.dom.DomClientProviderBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.DomFilterBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.DomHandlerBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.ServletBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessingBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder;
+import com.yahoo.vespa.model.clients.ContainerDocumentApi;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
+import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.http.Http;
+import com.yahoo.vespa.model.container.http.xml.HttpBuilder;
+import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder;
+import com.yahoo.vespa.model.container.processing.ProcessingChains;
+import com.yahoo.vespa.model.container.search.ContainerSearch;
+import com.yahoo.vespa.model.container.search.PageTemplates;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+import com.yahoo.vespa.model.container.search.SemanticRules;
+import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
+import com.yahoo.vespa.model.container.xml.document.DocumentFactoryBuilder;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+/**
+ * @author tonytv
+ */
+public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
+
+ /**
+ * Default path to vip status file for container in Hosted Vespa.
+ */
+ static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().vespaHome() + "var/mediasearch/oor/status.html";
+ /**
+ * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE
+ */
+ static final String HOSTED_VESPA_STATUS_FILE_YINST_SETTING = "cloudconfig_server__tenant_vip_status_file";
+
+ public enum Networking { disable, enable }
+
+ private ApplicationPackage app;
+ private final boolean standaloneBuilder;
+ private final Networking networking;
+ protected DeployLogger log;
+
+ public static final List<ConfigModelId> configModelIds =
+ ImmutableList.of(ConfigModelId.fromName("container"), ConfigModelId.fromName("jdisc"));
+
+ private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName();
+ private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName();
+
+ public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) {
+ super(ContainerModel.class);
+ this.standaloneBuilder = standaloneBuilder;
+ this.networking = networking;
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() {
+ return configModelIds;
+ }
+
+ @Override
+ public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) {
+ app = modelContext.getApplicationPackage();
+ checkVersion(spec);
+
+ this.log = modelContext.getDeployLogger();
+
+ ContainerCluster cluster = createContainerCluster(spec, modelContext);
+ addClusterContent(cluster, spec, modelContext);
+ addBundlesForPlatformComponents(cluster);
+
+ model.setCluster(cluster);
+ }
+
+ protected void addBundlesForPlatformComponents(ContainerCluster cluster) {
+ for (Component<?, ?> component : cluster.getAllComponents()) {
+ String componentClass = component.model.bundleInstantiationSpec.getClassName();
+ BundleMapper.getBundlePath(componentClass).
+ ifPresent(cluster::addPlatformBundle);
+ }
+ }
+
+ private ContainerCluster createContainerCluster(Element spec, final ConfigModelContext modelContext) {
+ return new VespaDomBuilder.DomConfigProducerBuilder<ContainerCluster>() {
+ @Override
+ protected ContainerCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ return new ContainerCluster(ancestor, modelContext.getProducerId(), modelContext.getProducerId());
+ }
+ }.build(modelContext.getParentProducer(), spec);
+ }
+
+ private void addClusterContent(ContainerCluster cluster, Element spec, ConfigModelContext modelContext) {
+ DocumentFactoryBuilder.buildDocumentFactories(cluster, spec);
+
+ addConfiguredComponents(cluster, spec);
+ addHandlers(cluster, spec);
+
+ addHttp(spec, cluster);
+ addRestApis(spec, cluster);
+ addServlets(spec, cluster);
+ addProcessing(spec, cluster);
+ addSearch(spec, cluster, modelContext.getDeployState().getQueryProfiles(), modelContext.getDeployState().getSemanticRules());
+ addDocproc(spec, cluster);
+ addDocumentApi(spec, cluster); //NOTE: Document API must be set up _after_ search!
+
+ addAccessLogs(cluster, spec);
+ addRoutingAliases(cluster, spec);
+ addNodes(cluster, spec);
+
+ addClientProviders(spec, cluster);
+ addServerProviders(spec, cluster);
+ addLegacyFilters(spec, cluster);
+
+ addDefaultHandlers(cluster);
+ addStatusHandlers(cluster, modelContext);
+ addDefaultComponents(cluster);
+ setDefaultMetricConsumerFactory(cluster);
+
+ //TODO: overview handler, see DomQrserverClusterBuilder
+ //TODO: cache options.
+ }
+
+ private void addRoutingAliases(ContainerCluster cluster, Element spec) {
+ Element aliases = XML.getChild(spec, "aliases");
+ for (Element alias : XML.getChildren(aliases, "service-alias")) {
+ cluster.serviceAliases().add(XML.getValue(alias));
+ }
+ for (Element alias : XML.getChildren(aliases, "endpoint-alias")) {
+ cluster.endpointAliases().add(XML.getValue(alias));
+ }
+ }
+
+ private void addConfiguredComponents(ContainerCluster cluster, Element spec) {
+ for (Element components : XML.getChildren(spec, "components")) {
+ addIncludes(components);
+ addConfiguredComponents(cluster, components, "component");
+ }
+ addConfiguredComponents(cluster, spec, "component");
+ }
+
+ protected void addDefaultComponents(ContainerCluster cluster) {
+ }
+
+ protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) {
+ cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.STATE_MONITOR);
+ }
+
+ protected void addDefaultHandlers(ContainerCluster cluster) {
+ addDefaultHandlersExceptStatus(cluster);
+ }
+
+ protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) {
+ if (configModelContext.getDeployState().isHostedVespa()) {
+ String name = "status.html";
+ Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_YINST_SETTING));
+ cluster.addComponent(
+ new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE),
+ "http://*/" + name, "https://*/" + name));
+ } else {
+ cluster.addVipHandler();
+ }
+ }
+
+ /**
+ * Intended for use by legacy builders only.
+ * Will be called during building when using ContainerModelBuilder.
+ */
+ public static void addDefaultHandler_legacyBuilder(ContainerCluster cluster) {
+ addDefaultHandlersExceptStatus(cluster);
+ cluster.addVipHandler();
+ }
+
+ protected static void addDefaultHandlersExceptStatus(ContainerCluster cluster) {
+ cluster.addDefaultRootHandler();
+ cluster.addMetricStateHandler();
+ cluster.addApplicationStatusHandler();
+ cluster.addStatisticsHandler();
+ }
+
+ private void addClientProviders(Element spec, ContainerCluster cluster) {
+ for (Element clientSpec: XML.getChildren(spec, "client")) {
+ cluster.addComponent(new DomClientProviderBuilder().build(cluster, clientSpec));
+ }
+ }
+
+ private void addServerProviders(Element spec, ContainerCluster cluster) {
+ addConfiguredComponents(cluster, spec, "server");
+ }
+
+ private void addLegacyFilters(Element spec, ContainerCluster cluster) {
+ for (Component component : buildLegacyFilters(cluster, spec)) {
+ cluster.addComponent(component);
+ }
+ }
+
+ private List<Component> buildLegacyFilters(AbstractConfigProducer ancestor,
+ Element spec) {
+ List<Component> components = new ArrayList<>();
+
+ for (Element node : XML.getChildren(spec, "filter")) {
+ components.add(new DomFilterBuilder().build(ancestor, node));
+ }
+ return components;
+ }
+
+ protected void addAccessLogs(ContainerCluster cluster, Element spec) {
+ List<Element> accessLogElements = getAccessLogElements(spec);
+
+ for (Element accessLog : accessLogElements) {
+ AccessLogBuilder.buildIfNotDisabled(cluster, accessLog).ifPresent(cluster::addComponent);
+ }
+
+ if (accessLogElements.isEmpty() && cluster.getSearch() != null)
+ cluster.addDefaultSearchAccessLog();
+ }
+
+ protected final List<Element> getAccessLogElements(Element spec) {
+ return XML.getChildren(spec, "accesslog");
+ }
+
+
+ protected void addHttp(Element spec, ContainerCluster cluster) {
+ Element httpElement = XML.getChild(spec, "http");
+ if (httpElement != null) {
+ cluster.setHttp(buildHttp(cluster, httpElement));
+ }
+ }
+
+ private Http buildHttp(ContainerCluster cluster, Element httpElement) {
+ Http http = new HttpBuilder().build(cluster, httpElement);
+
+ if (networking == Networking.disable)
+ http.removeAllServers();
+
+ return http;
+ }
+
+ protected void addRestApis(Element spec, ContainerCluster cluster) {
+ for (Element restApiElem : XML.getChildren(spec, "rest-api")) {
+ cluster.addRestApi(
+ new RestApiBuilder().build(cluster, restApiElem));
+ }
+ }
+
+ private void addServlets(Element spec, ContainerCluster cluster) {
+ for (Element servletElem : XML.getChildren(spec, "servlet")) {
+ cluster.addServlet(
+ new ServletBuilder().build(cluster, servletElem));
+ }
+ }
+
+ private void addDocumentApi(Element spec, ContainerCluster cluster) {
+ ContainerDocumentApi containerDocumentApi = buildDocumentApi(cluster, spec);
+ if (containerDocumentApi != null) {
+ cluster.setDocumentApi(containerDocumentApi);
+ }
+ }
+
+ private void addDocproc(Element spec, ContainerCluster cluster) {
+ ContainerDocproc containerDocproc = buildDocproc(cluster, spec);
+ if (containerDocproc != null) {
+ cluster.setDocproc(containerDocproc);
+
+ ContainerDocproc.Options docprocOptions = containerDocproc.options;
+ cluster.setMbusParams(new ContainerCluster.MbusParams(
+ docprocOptions.maxConcurrentFactor, docprocOptions.documentExpansionFactor, docprocOptions.containerCoreMemory));
+ }
+ }
+
+ private void addSearch(Element spec, ContainerCluster cluster, QueryProfiles queryProfiles, SemanticRules semanticRules) {
+ Element searchElement = XML.getChild(spec, "search");
+ if (searchElement != null) {
+ addIncludes(searchElement);
+ cluster.setSearch(buildSearch(cluster, searchElement, queryProfiles, semanticRules));
+
+ addSearchHandler(cluster, searchElement);
+ validateAndAddConfiguredComponents(cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement);
+ }
+ }
+
+ private void addProcessing(Element spec, ContainerCluster cluster) {
+ Element processingElement = XML.getChild(spec, "processing");
+ if (processingElement != null) {
+ addIncludes(processingElement);
+ cluster.setProcessingChains(new DomProcessingBuilder(null).build(cluster, processingElement),
+ serverBindings(processingElement, ProcessingChains.defaultBindings));
+ validateAndAddConfiguredComponents(cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement);
+ }
+ }
+
+ private ContainerSearch buildSearch(ContainerCluster containerCluster, Element producerSpec,
+ QueryProfiles queryProfiles, SemanticRules semanticRules) {
+ SearchChains searchChains = new DomSearchChainsBuilder(null, false).build(containerCluster, producerSpec);
+
+ ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains, new ContainerSearch.Options());
+
+ applyApplicationPackageDirectoryConfigs(containerCluster.getRoot().getDeployState().getApplicationPackage(), containerSearch);
+ containerSearch.setQueryProfiles(queryProfiles);
+ containerSearch.setSemanticRules(semanticRules);
+
+ return containerSearch;
+ }
+
+ private void applyApplicationPackageDirectoryConfigs(ApplicationPackage applicationPackage,ContainerSearch containerSearch) {
+ PageTemplates.validate(applicationPackage);
+ containerSearch.setPageTemplates(PageTemplates.create(applicationPackage));
+ }
+
+ private void addHandlers(ContainerCluster cluster, Element spec) {
+ for (Element component: XML.getChildren(spec, "handler")) {
+ cluster.addComponent(
+ new DomHandlerBuilder().build(cluster, component));
+ }
+ }
+
+ private void checkVersion(Element spec) {
+ String version = spec.getAttribute("version");
+
+ if (!Version.fromString(version).equals(new Version(1))) {
+ throw new RuntimeException("Expected container version to be 1.0, but got " + version);
+ }
+ }
+
+ private void addNodes(ContainerCluster cluster, Element spec) {
+ if (standaloneBuilder) {
+ addStandaloneNode(cluster);
+ } else {
+ addNodesFromXml(cluster, spec);
+ }
+ }
+
+ private void addStandaloneNode(ContainerCluster cluster) {
+ Container container = new Container(cluster, "standalone");
+ cluster.addContainers(Collections.singleton(container));
+ }
+
+ private void addNodesFromXml(ContainerCluster cluster, Element spec) {
+ Element nodesElement = XML.getChild(spec, "nodes");
+ if (nodesElement == null) { // default single node on localhost
+ Container container = new Container(cluster, "container.0");
+ HostResource host = allocateSingleNodeHost(cluster, log);
+ container.setHostResource(host);
+ if ( ! container.isInitialized() ) // TODO: Fold this into initService
+ container.initService();
+ cluster.addContainers(Collections.singleton(container));
+ }
+ else {
+ String defaultJvmArgs = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME);
+ String defaultPreLoad = null;
+ if (nodesElement.hasAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)) {
+ defaultPreLoad = nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME);
+ }
+ boolean useCpuSocketAffinity = false;
+ if (nodesElement.hasAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)) {
+ useCpuSocketAffinity = Boolean.parseBoolean(nodesElement.getAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME));
+ }
+ List<Container> result = new ArrayList<>();
+ result.addAll(createNodesFromXmlNodeCount(cluster, nodesElement));
+ addNodesFromXmlNodeList(cluster, spec, nodesElement, result);
+ applyDefaultJvmArgs(result, defaultJvmArgs);
+ applyRoutingAliasProperties(result, cluster);
+ if (defaultPreLoad != null) {
+ applyDefaultPreload(result, defaultPreLoad);
+ }
+ if (useCpuSocketAffinity) {
+ AbstractService.distributeCpuSocketAffinity(result);
+ }
+
+ cluster.addContainers(result);
+ }
+ }
+
+ private void applyRoutingAliasProperties(List<Container> result, ContainerCluster cluster) {
+ if (!cluster.serviceAliases().isEmpty()) {
+ result.forEach(container -> {
+ container.setProp("servicealiases", cluster.serviceAliases().stream().collect(Collectors.joining(",")));
+ });
+ }
+ if (!cluster.endpointAliases().isEmpty()) {
+ result.forEach(container -> {
+ container.setProp("endpointaliases", cluster.endpointAliases().stream().collect(Collectors.joining(",")));
+ });
+ }
+ }
+
+ private HostResource allocateSingleNodeHost(ContainerCluster cluster, DeployLogger logger) {
+ if (cluster.isHostedVespa()) {
+ ClusterSpec clusterSpec = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty());
+ return cluster.getHostSystem().allocateHosts(clusterSpec, Capacity.fromNodeCount(1), 1, logger).keySet().iterator().next();
+ } else {
+ return cluster.getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC);
+ }
+ }
+
+ private void addNodesFromXmlNodeList(ContainerCluster cluster,
+ Element spec, Element nodesElement, List<Container> result) {
+ int nodeCount = 0;
+ for (Element nodeElem: XML.getChildren(nodesElement, "node")) {
+ Container container = new ContainerServiceBuilder("container." + nodeCount).build(cluster, nodeElem);
+ result.add(container);
+ ++nodeCount;
+ }
+ }
+
+ private List<Container> createNodesFromXmlNodeCount(ContainerCluster cluster, Element nodesElement) {
+ List<Container> result = new ArrayList<>();
+ if (nodesElement.hasAttribute("count")) {
+ NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement));
+ Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().getHostSystem(), ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty(), log);
+ for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) {
+ String id = "container." + entry.getValue().index();
+ Container container = new Container(cluster, id, entry.getValue().retired());
+ container.setHostResource(entry.getKey());
+ container.initService();
+ result.add(container);
+ }
+ }
+ return result;
+ }
+
+ private void applyDefaultJvmArgs(List<Container> containers, String defaultJvmArgs) {
+ for (Container container: containers) {
+ if (container.getJvmArgs().isEmpty())
+ container.prependJvmArgs(defaultJvmArgs);
+ }
+ }
+
+ private void applyDefaultPreload(List<Container> containers, String defaultPreLoad) {
+ for (Container container: containers) {
+ container.setPreLoad(defaultPreLoad);
+ }
+ }
+
+ private void addSearchHandler(ContainerCluster cluster, Element searchElement) {
+ ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(
+ cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler");
+
+ String[] defaultBindings = {"http://*/search/*", "https://*/search/*"};
+ for (String binding: serverBindings(searchElement, defaultBindings)) {
+ searchHandler.addServerBindings(binding);
+ }
+
+ cluster.addComponent(searchHandler);
+ }
+
+ private String[] serverBindings(Element searchElement, String... defaultBindings) {
+ List<Element> bindings = XML.getChildren(searchElement, "binding");
+ if (bindings.isEmpty())
+ return defaultBindings;
+
+ return toBindingList(bindings);
+ }
+
+ private String[] toBindingList(List<Element> bindingElements) {
+ List<String> result = new ArrayList<>();
+
+ for (Element element: bindingElements) {
+ String text = element.getTextContent().trim();
+ if (!text.isEmpty())
+ result.add(text);
+ }
+
+ return result.toArray(new String[result.size()]);
+ }
+
+ private ContainerDocumentApi buildDocumentApi(ContainerCluster cluster, Element spec) {
+ Element documentApiElement = XML.getChild(spec, "document-api");
+ if (documentApiElement == null) {
+ return null;
+ }
+
+ ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement);
+ return new ContainerDocumentApi(cluster, documentApiOptions);
+ }
+
+ private ContainerDocproc buildDocproc(ContainerCluster cluster, Element spec) {
+ Element docprocElement = XML.getChild(spec, "document-processing");
+ if (docprocElement == null)
+ return null;
+
+ addIncludes(docprocElement);
+ DocprocChains chains = new DomDocprocChainsBuilder(null, false).build(cluster, docprocElement);
+
+ ContainerDocproc.Options docprocOptions = DocprocOptionsBuilder.build(docprocElement);
+ return new ContainerDocproc(cluster, chains, docprocOptions, !standaloneBuilder);
+ }
+
+ private void addIncludes(Element parentElement) {
+ List<Element> includes = XML.getChildren(parentElement, IncludeDirs.INCLUDE);
+ if (includes == null || includes.isEmpty()) {
+ return;
+ }
+ if (app == null) {
+ throw new IllegalArgumentException("Element <include> given in XML config, but no application package given.");
+ }
+ for (Element include : includes) {
+ addInclude(parentElement, include);
+ }
+ }
+
+ private void addInclude(Element parentElement, Element include) {
+ String dirName = include.getAttribute(IncludeDirs.DIR);
+ app.validateIncludeDir(dirName);
+
+ List<Element> includedFiles = Xml.allElemsFromPath(app, dirName);
+ for (Element includedFile : includedFiles) {
+ List<Element> includedSubElements = XML.getChildren(includedFile);
+ for (Element includedSubElement : includedSubElements) {
+ Node copiedNode = parentElement.getOwnerDocument().importNode(includedSubElement, true);
+ parentElement.appendChild(copiedNode);
+ }
+ }
+ }
+
+ public static void addConfiguredComponents(ContainerCluster cluster, Element spec, String componentName) {
+ for (Element node : XML.getChildren(spec, componentName)) {
+ cluster.addComponent(new DomComponentBuilder().build(cluster, node));
+ }
+ }
+
+ public static void validateAndAddConfiguredComponents(ContainerCluster cluster, Element spec, String componentName, Consumer<Element> elementValidator) {
+ for (Element node : XML.getChildren(spec, componentName)) {
+ elementValidator.accept(node); // throws exception here if something is wrong
+ cluster.addComponent(new DomComponentBuilder().build(cluster, node));
+ }
+ }
+
+ /**
+ * Disallow renderers named "DefaultRenderer" or "JsonRenderer"
+ */
+ private static void validateRendererElement(Element element) {
+ String idAttr = element.getAttribute("id");
+
+ if (idAttr.equals(xmlRendererId) || idAttr.equals(jsonRendererId)) {
+ throw new IllegalArgumentException(String.format("Renderer id %s is reserved for internal use", idAttr));
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
new file mode 100644
index 00000000000..785ab1f7504
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.ConfigModelUtils;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.Container;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * @author tonytv
+ */
+public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Container> {
+
+ private final String id;
+
+ public ContainerServiceBuilder(String id) {
+ this.id = id;
+ }
+
+ @Override
+ protected Container doBuild(AbstractConfigProducer parent, Element nodeElem) {
+ return new Container(parent, id, readServerPortOverrides(nodeElem));
+ }
+
+ private List<Container.PortOverride> readServerPortOverrides(Element spec) {
+ List<Container.PortOverride> portOverrides = new ArrayList<>();
+
+ for (Element serverPort: XML.getChildren(spec, "server-port")) {
+ ComponentSpecification serverId = XmlHelper.getIdRef(serverPort);
+ int port = Integer.parseInt(serverPort.getAttribute("port"));
+
+ portOverrides.add(new Container.PortOverride(serverId, port));
+ }
+
+ return portOverrides;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java
new file mode 100644
index 00000000000..0b13187bd58
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import org.w3c.dom.Element;
+
+/**
+ * Extracted from DomDocProcClusterBuilder
+ */
+public class DocprocOptionsBuilder {
+ public static ContainerDocproc.Options build(Element spec) {
+ return new ContainerDocproc.Options(
+ getCompression(spec),
+ getMaxMessagesInQueue(spec),
+ getSizeInMegabytes(spec.getAttribute("maxqueuebytesize")),
+ getTime(spec.getAttribute("maxqueuewait")),
+ getFactor(spec.getAttribute("maxconcurrentfactor")),
+ getFactor(spec.getAttribute("documentexpansionfactor")),
+ getInt(spec.getAttribute("containercorememory")));
+ }
+
+ private static Integer getInt(String integer) {
+ return integer == null || integer.trim().isEmpty() ?
+ null:
+ Integer.parseInt(integer);
+ }
+
+ private static boolean getCompression(Element spec) {
+ return (spec.hasAttribute("compressdocuments") && spec.getAttribute("compressdocuments").equals("true"));
+ }
+
+ private static Double getFactor(String factor) {
+ return factor == null || factor.trim().isEmpty() ?
+ null :
+ Double.parseDouble(factor);
+ }
+
+
+ private static Integer getMaxMessagesInQueue(Element spec) {
+ // get max queue size (number of messages), if set
+ Integer maxMessagesInQueue = null;
+ if (spec.hasAttribute("maxmessagesinqueue")) {
+ maxMessagesInQueue = Integer.valueOf(spec.getAttribute("maxmessagesinqueue"));
+ }
+ return maxMessagesInQueue;
+ }
+
+ private static Integer getSizeInMegabytes(String size) {
+ if (size == null) {
+ return null;
+ }
+ size = size.trim();
+ if (size.isEmpty()) {
+ return null;
+ }
+
+ Integer megabyteSize;
+ if (size.endsWith("m")) {
+ size = size.substring(0, size.length() - 1);
+ megabyteSize = Integer.parseInt(size);
+ } else if (size.endsWith("g")) {
+ size = size.substring(0, size.length() - 1);
+ megabyteSize = Integer.parseInt(size) * 1024;
+ } else {
+ throw new IllegalArgumentException("Heap sizes for docproc must be set to Xm or Xg, where X is an integer specifying megabytes or gigabytes, respectively.");
+ }
+ return megabyteSize;
+ }
+
+ private static Integer getTime(String intStr) {
+ if (intStr == null) {
+ return null;
+ }
+ intStr = intStr.trim();
+ if (intStr.isEmpty()) {
+ return null;
+ }
+
+ return 1000 * (int)Double.parseDouble(intStr);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
new file mode 100644
index 00000000000..df2090db166
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.clients.ContainerDocumentApi;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.11
+ */
+public class DocumentApiOptionsBuilder {
+ private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName());
+ private static final String[] DEFAULT_BINDINGS = {"http://*/", "https://*/"};
+
+ public static ContainerDocumentApi.Options build(Element spec) {
+ return new ContainerDocumentApi.Options(
+ getBindings(spec),
+ getAbortOnDocumentError(spec),
+ getRoute(spec),
+ getMaxPendingDocs(spec),
+ getMaxPendingBytes(spec),
+ getRetryEnabled(spec),
+ getRetryDelay(spec),
+ getTimeout(spec),
+ getTracelevel(spec),
+ getMbusPort(spec),
+ getDocprocChain(spec));
+ }
+
+ private static List<String> getBindings(Element spec) {
+ Collection<Element> bindingElems = XML.getChildren(spec, "binding");
+ if (bindingElems.isEmpty())
+ return Arrays.asList(DEFAULT_BINDINGS);
+
+ List<String> bindings = new ArrayList<>();
+ for (Element e :bindingElems) {
+ String binding = getBinding(e);
+ bindings.add(binding);
+ }
+ return bindings;
+ }
+
+ private static String getBinding(Element e) {
+ String binding = XML.getValue(e);
+ if (! binding.endsWith("/")) {
+ log.warning("Adding a trailing '/' to the document-api binding: " + binding + " -> " + binding + "/");
+ binding = binding + "/";
+ }
+ return binding;
+ }
+
+ private static String getCleanValue(Element spec, String name) {
+ Element elem = XML.getChild(spec, name);
+ if (elem == null) {
+ return null;
+ }
+ String value = elem.getFirstChild().getNodeValue();
+ if (value == null) {
+ return null;
+ }
+ value = value.trim();
+ return value.isEmpty() ? null : value;
+ }
+
+ private static String getDocprocChain(Element spec) {
+ return getCleanValue(spec, "docprocchain");
+ }
+
+ private static Integer getMbusPort(Element spec) {
+ String value = getCleanValue(spec, "mbusport");
+ return value == null ? null : Integer.parseInt(value);
+ }
+
+ private static Integer getTracelevel(Element spec) {
+ String value = getCleanValue(spec, "tracelevel");
+ return value == null ? null : Integer.parseInt(value);
+ }
+
+ private static Double getTimeout(Element spec) {
+ String value = getCleanValue(spec, "timeout");
+ return value == null ? null : Double.parseDouble(value);
+ }
+
+ private static Double getRetryDelay(Element spec) {
+ String value = getCleanValue(spec, "retrydelay");
+ return value == null ? null : Double.parseDouble(value);
+ }
+
+ private static Boolean getRetryEnabled(Element spec) {
+ String value = getCleanValue(spec, "retryenabled");
+ return value == null ? null : Boolean.parseBoolean(value);
+ }
+
+ private static Integer getMaxPendingBytes(Element spec) {
+ String value = getCleanValue(spec, "maxpendingbytes");
+ return value == null ? null : Integer.parseInt(value);
+ }
+
+ private static Integer getMaxPendingDocs(Element spec) {
+ String value = getCleanValue(spec, "maxpendingdocs");
+ return value == null ? null : Integer.parseInt(value);
+ }
+
+ private static String getRoute(Element spec) {
+ return getCleanValue(spec, "route");
+ }
+
+ private static Boolean getAbortOnDocumentError(Element spec) {
+ String value = getCleanValue(spec, "abortondocumenterror");
+ return value == null ? null : Boolean.parseBoolean(value);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java
new file mode 100644
index 00000000000..46e7271fe9b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+/**
+ * @author tonytv
+ */
+public class InconsistentSchemaAndCodeError extends Error {}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java
new file mode 100644
index 00000000000..f6ed6c2eb7d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java
@@ -0,0 +1,172 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.container.jdisc.config.MetricDefaultsConfig;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.AccessLogComponent;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
+import com.yahoo.vespa.model.container.http.ConnectorFactory;
+import com.yahoo.vespa.model.container.http.FilterChains;
+import com.yahoo.vespa.model.container.http.Http;
+import com.yahoo.vespa.model.container.http.JettyHttpServer;
+import org.w3c.dom.Element;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.yahoo.collections.CollectionUtil.first;
+import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme;
+import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath;
+
+/**
+ * @author tonytv
+ */
+public final class ManhattanContainerModelBuilder extends ContainerModelBuilder {
+
+ static final String MANHATTAN_FILE_NAME_PATTERN = Defaults.getDefaults().vespaHome() + "logs/jdisc_core/access.%Y-%m-%d-%H";
+ static final String MANHATTAN_ROTATION_INTERVAL = "0 60 ...";
+ static final RotateScheme.Enum MANHATTAN_ROTATION_SCHEME = RotateScheme.DATE;
+ static final String MANHATTAN_SYMLINK_NAME = "access";
+
+ public interface BundleFiles {
+ // TODO: move constants to the DH code base.
+ Set<Path> dhBundles = new HashSet<>(Arrays.asList(
+ Paths.get("apache_avro/avro.jar"),
+ Paths.get("apache_avro/commons-compress.jar"),
+ Paths.get("apache_avro/paranamer.jar"),
+ Paths.get("apache_avro/jackson-core-asl.jar"),
+ Paths.get("apache_avro/jackson-mapper-asl.jar"),
+ Paths.get("dh_rainbow_client_api_java.jar"),
+ Paths.get("dh_rainbow_util_batch_java.jar"),
+ Paths.get("dh_rainbow_util_java.jar")));
+ }
+
+ private final int httpPort;
+ private JettyHttpServer jettyHttpServer;
+
+ public ManhattanContainerModelBuilder(int httpPort) {
+ super(true, Networking.enable);
+ this.httpPort = httpPort;
+ }
+
+ @Override
+ protected void addBundlesForPlatformComponents(ContainerCluster cluster) {
+ super.addBundlesForPlatformComponents(cluster);
+ BundleFiles.dhBundles.forEach(
+ bundleFile -> cluster.addPlatformBundle(absoluteBundlePath(bundleFile)));
+ }
+
+ @Override
+ protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) {
+ cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD);
+ }
+
+ @Override
+ protected void addAccessLogs(ContainerCluster cluster, Element spec) {
+ warnIfAccessLogsDefined(spec);
+
+ checkNotNull(jettyHttpServer, "addHttp must be called first");
+ cluster.addComponent(createManhattanAccessLog());
+ }
+
+ private Component createManhattanAccessLog() {
+ return new AccessLogComponent(AccessLogComponent.AccessLogType.yApacheAccessLog,
+ MANHATTAN_FILE_NAME_PATTERN,
+ MANHATTAN_ROTATION_INTERVAL,
+ MANHATTAN_ROTATION_SCHEME,
+ MANHATTAN_SYMLINK_NAME);
+ }
+
+ private void warnIfAccessLogsDefined(Element spec) {
+ List<Element> accessLogElements = getAccessLogElements(spec);
+ if (!accessLogElements.isEmpty()) {
+ logManhattanInfo("Ignoring " + accessLogElements.size() +
+ " access log elements in services.xml, using default yapache access logging instead.");
+ }
+ }
+
+ @Override
+ protected void addDefaultHandlers(ContainerCluster cluster) {
+ addDefaultHandlersExceptStatus(cluster);
+ }
+
+ @Override
+ protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) {
+ addStatusHandlerForJDiscStatusPackage(cluster, "status.html"); //jdisc_status
+ addStatusHandlerForJDiscStatusPackage(cluster, "akamai"); //jdisc_akamai
+ }
+
+ private static void addStatusHandlerForJDiscStatusPackage(ContainerCluster cluster, String name) {
+ cluster.addComponent(
+ new FileStatusHandlerComponent(name + "-status-handler", Defaults.getDefaults().vespaHome() + "libexec/jdisc/" + name,
+ "http://*/" + name, "https://*/" + name));
+ }
+
+ @Override
+ protected void addHttp(Element spec, ContainerCluster cluster) {
+ super.addHttp(spec, cluster);
+ ensureHasHttp(cluster);
+ ensureOneHttpServer(cluster.getHttp());
+ }
+
+ private void ensureHasHttp(ContainerCluster cluster) {
+ if (cluster.getHttp() == null)
+ cluster.setHttp(createHttp());
+ }
+
+ private Http createHttp() {
+ Http http = new Http(Collections.<Http.Binding>emptyList());
+ http.setFilterChains(new FilterChains(http));
+ return http;
+ }
+
+ private void ensureOneHttpServer(Http http) {
+ if (http.getHttpServer() == null || http.getHttpServer().getConnectorFactories().isEmpty()) {
+ JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("main-http-server"));
+ http.setHttpServer(jettyHttpServer);
+ ConnectorFactory connectorFactory = new ConnectorFactory("main-http-connector",
+ httpPort, null);
+ http.getHttpServer().addConnector(connectorFactory);
+ } else {
+ removeAllButOneConnector(http.getHttpServer());
+ ConnectorFactory connectorFactory = first(http.getHttpServer().getConnectorFactories());
+ connectorFactory.setListenPort(httpPort);
+ }
+ jettyHttpServer = http.getHttpServer();
+ }
+
+ private void removeAllButOneConnector(JettyHttpServer jettyHttpServer) {
+ int removed = 0;
+
+ if (jettyHttpServer.getConnectorFactories().size() > 1) {
+ for (int i = jettyHttpServer.getConnectorFactories().size() - 1; i > 0; i--) {
+ ConnectorFactory c = jettyHttpServer.getConnectorFactories().get(i);
+ jettyHttpServer.removeConnector(c);
+ ++removed;
+ }
+ }
+
+ if (removed > 0) {
+ logManhattanInfo("Using only the first http server " + jettyHttpServer.getConnectorFactories().get(0).getName());
+ }
+ }
+
+ private static <E> List<E> tail(List<E> list) {
+ return list.subList(1, list.size());
+ }
+
+ private void logManhattanInfo(String message) {
+ log.log(Level.INFO, "[Manhattan] " + message);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java
new file mode 100644
index 00000000000..96ab9fd2f52
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.xml.document;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import org.w3c.dom.Element;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Handles the document bindings (concrete document types). Register the concrete document factories as components.
+ *
+ * @author vegardh
+ * @since 5.1.10
+ */
+public class DocumentFactoryBuilder {
+ private static final String CONCRETE_DOC_FACTORY_CLASS = "ConcreteDocumentFactory";
+
+ public static void buildDocumentFactories(ContainerCluster cluster, Element spec) {
+ Map<String, String> types = new LinkedHashMap<>();
+ for (Element e : XML.getChildren(spec, "document")) {
+ String type = e.getAttribute("type");
+ String clazz = e.getAttribute("class");
+ // Empty pkg is forbidden in the documentgen Mojo.
+ if (clazz.indexOf('.')<0) throw new IllegalArgumentException("Malformed class for <document> binding, must be a full class with package: "+clazz);
+ String pkg = clazz.substring(0, clazz.lastIndexOf('.'));
+ String concDocFactory=pkg+"."+CONCRETE_DOC_FACTORY_CLASS;
+ String bundle = e.getAttribute("bundle");
+ Component<AbstractConfigProducer<?>, ComponentModel> component = new Component<>(
+ new ComponentModel(BundleInstantiationSpecification.getFromStrings(concDocFactory, concDocFactory, bundle)));
+ if (!cluster.getComponentsMap().containsKey(component.getComponentId())) cluster.addComponent(component);
+ types.put(type, concDocFactory);
+ }
+ cluster.concreteDocumentTypes().putAll(types);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java
new file mode 100644
index 00000000000..2ae232adcd6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.container.xml;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java
new file mode 100644
index 00000000000..e320e44c237
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+/**
+ * Represents configuration for bucket splitting.
+ */
+public class BucketSplitting implements StorDistributormanagerConfig.Producer {
+ Integer maxDocuments;
+ Integer splitSize;
+ Integer minSplitCount;
+ boolean useInlineBucketSplitting;
+
+ public static class Builder {
+ public BucketSplitting build(ContentCluster cluster, ModelElement clusterElem) {
+ ModelElement tuning = clusterElem.getChild("tuning");
+ if (tuning == null) {
+ return new BucketSplitting(cluster.isMemfilePersistence(), null, null, null);
+ }
+
+ ModelElement bucketSplitting = tuning.getChild("bucket-splitting");
+ if (bucketSplitting != null) {
+ Integer maxDocuments = bucketSplitting.getIntegerAttribute("max-documents");
+ Integer splitSize = bucketSplitting.getIntegerAttribute("max-size");
+ Integer minSplitCount = bucketSplitting.getIntegerAttribute("minimum-bits");
+
+ return new BucketSplitting(cluster.isMemfilePersistence(), maxDocuments, splitSize, minSplitCount);
+ }
+
+ return new BucketSplitting(cluster.isMemfilePersistence(), null, null, null);
+ }
+ }
+
+ public BucketSplitting(boolean useInlineBucketSplitting, Integer maxDocuments, Integer splitSize, Integer minSplitCount) {
+ this.maxDocuments = maxDocuments;
+ this.splitSize = splitSize;
+ this.minSplitCount = minSplitCount;
+ this.useInlineBucketSplitting = useInlineBucketSplitting;
+ }
+
+ @Override
+ public void getConfig(StorDistributormanagerConfig.Builder builder) {
+ if (maxDocuments != null) {
+ builder.splitcount(maxDocuments);
+ builder.joincount(maxDocuments / 2);
+ }
+ if (splitSize != null) {
+ builder.splitsize(splitSize);
+ builder.joinsize(splitSize / 2);
+ }
+ if (minSplitCount != null) {
+ builder.minsplitcount(minSplitCount);
+ }
+
+ builder.inlinebucketsplitting(useInlineBucketSplitting);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
new file mode 100644
index 00000000000..fd1bc8a3362
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.content.FleetcontrollerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.utils.Duration;
+import org.w3c.dom.Element;
+
+/**
+ * Config generation for common parameters for all fleet controllers.
+ *
+ * TODO: Author
+ */
+public class ClusterControllerConfig extends AbstractConfigProducer implements FleetcontrollerConfig.Producer {
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ClusterControllerConfig> {
+ String clusterName;
+ ModelElement clusterElement;
+
+ public Builder(String clusterName, ModelElement clusterElement) {
+ this.clusterName = clusterName;
+ this.clusterElement = clusterElement;
+ }
+
+ @Override
+ protected ClusterControllerConfig doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ ModelElement tuning = null;
+
+ ModelElement clusterTuning = clusterElement.getChild("tuning");
+ if (clusterTuning != null) {
+ tuning = clusterTuning.getChild("cluster-controller");
+ }
+
+ if (tuning != null) {
+ return new ClusterControllerConfig(ancestor, clusterName,
+ tuning.childAsDuration("init-progress-time"),
+ tuning.childAsDuration("transition-time"),
+ tuning.childAsLong("max-premature-crashes"),
+ tuning.childAsDuration("stable-state-period"),
+ tuning.childAsDouble("min-distributor-up-ratio"),
+ tuning.childAsDouble("min-storage-up-ratio"),
+ clusterElement.childAsInteger("tuning.bucket-splitting.minimum-bits"));
+ } else {
+ return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null,
+ clusterElement.childAsInteger("tuning.bucket-splitting.minimum-bits"));
+ }
+ }
+ }
+
+ String clusterName;
+ Duration initProgressTime;
+ Duration transitionTime;
+ Long maxPrematureCrashes;
+ Duration stableStateTimePeriod;
+ Double minDistributorUpRatio;
+ Double minStorageUpRatio;
+ Integer minSplitBits;
+
+ private ClusterControllerConfig(AbstractConfigProducer parent,
+ String clusterName,
+ Duration initProgressTime,
+ Duration transitionTime,
+ Long maxPrematureCrashes,
+ Duration stableStateTimePeriod,
+ Double minDistributorUpRatio,
+ Double minStorageUpRatio,
+ Integer minSplitBits) {
+ super(parent, "fleetcontroller");
+
+ this.clusterName = clusterName;
+ this.initProgressTime = initProgressTime;
+ this.transitionTime = transitionTime;
+ this.maxPrematureCrashes = maxPrematureCrashes;
+ this.stableStateTimePeriod = stableStateTimePeriod;
+ this.minDistributorUpRatio = minDistributorUpRatio;
+ this.minStorageUpRatio = minStorageUpRatio;
+ this.minSplitBits = minSplitBits;
+ }
+
+ @Override
+ public void getConfig(FleetcontrollerConfig.Builder builder) {
+ AbstractConfigProducerRoot root = getRoot();
+ if (root instanceof VespaModel) {
+ String zooKeeperAddress =
+ root.getAdmin().getZooKeepersConfigProvider().getZooKeepersConnectionSpec();
+ builder.zookeeper_server(zooKeeperAddress);
+ } else {
+ builder.zookeeper_server("");
+ }
+
+ builder.index(0);
+ builder.cluster_name(clusterName);
+ builder.fleet_controller_count(getChildren().size());
+
+ if (initProgressTime != null) {
+ builder.init_progress_time((int) initProgressTime.getMilliSeconds());
+ }
+ if (transitionTime != null) {
+ builder.storage_transition_time((int) transitionTime.getMilliSeconds());
+ }
+ if (maxPrematureCrashes != null) {
+ builder.max_premature_crashes(maxPrematureCrashes.intValue());
+ }
+ if (stableStateTimePeriod != null) {
+ builder.stable_state_time_period((int) stableStateTimePeriod.getMilliSeconds());
+ }
+ if (minDistributorUpRatio != null) {
+ builder.min_distributor_up_ratio(minDistributorUpRatio);
+ }
+ if (minStorageUpRatio != null) {
+ builder.min_storage_up_ratio(minStorageUpRatio);
+ }
+ if (minSplitBits != null) {
+ builder.ideal_distribution_bits(minSplitBits);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
new file mode 100644
index 00000000000..b9fe10e4873
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
@@ -0,0 +1,347 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.ConfigModelRepoAdder;
+import com.yahoo.config.model.admin.AdminModel;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.builder.VespaModelBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+import com.yahoo.vespa.model.search.IndexingDocprocChain;
+import com.yahoo.vespa.model.search.SearchNode;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * The config model from a content tag in services.
+ * This consists mostly of a ContentCluster.
+ *
+ * @author balder
+ */
+public class Content extends ConfigModel {
+
+ private static final Logger log = Logger.getLogger(Content.class.getName());
+
+ private ContentCluster cluster;
+ private Optional<ContainerCluster> ownedIndexingCluster = Optional.empty();
+ private final boolean hostedVespa;
+
+ // Dependencies to other models
+ private final AdminModel adminModel;
+ private final Collection<ContainerModel> containers; // to find or add the docproc container
+
+ @SuppressWarnings({ "UnusedDeclaration"}) // Created by reflection in ConfigModelRepo
+ public Content(ConfigModelContext modelContext, AdminModel adminModel, Collection<ContainerModel> containers) {
+ super(modelContext);
+ modelContext.getParentProducer().getRoot();
+ hostedVespa = modelContext.getDeployState().isHostedVespa();
+ this.adminModel = adminModel;
+ this.containers = containers;
+ }
+
+ /** Returns the admin model of this system */
+ public AdminModel adminModel() { return adminModel; }
+
+ /** Called by DomContentBuilder during build */
+ public void setCluster(ContentCluster cluster, ConfigModelContext configModelContext) {
+ this.cluster = cluster;
+ initializeIndexingClusters(containers,
+ configModelContext.getConfigModelRepoAdder(),
+ (ApplicationConfigProducerRoot)configModelContext.getParentProducer());
+ }
+
+ public ContentCluster getCluster() { return cluster; }
+
+ /**
+ * Returns indexing cluster implicitly created by this,
+ * or empty if an explicit cluster is used (or if called before the build phase)
+ */
+ public Optional<ContainerCluster> ownedIndexingCluster() { return ownedIndexingCluster; }
+
+ public void createTlds(ConfigModelRepo modelRepo) {
+ IndexedSearchCluster indexedCluster = cluster.getSearch().getIndexed();
+ if (indexedCluster == null) {
+ return;
+ }
+
+ SimpleConfigProducer tldParent = new SimpleConfigProducer(indexedCluster, "tlds");
+ for (ConfigModel model : modelRepo.asMap().values()) {
+ if (!(model instanceof ContainerModel)) {
+ continue;
+ }
+
+ ContainerCluster containerCluster = ((ContainerModel) model).getCluster();
+ if (containerCluster.getSearch() == null) {
+ continue; // this is not a qrs cluster
+ }
+
+ log.log(LogLevel.DEBUG, "Adding tlds for indexed cluster " + indexedCluster.getClusterName() + ", container cluster " + containerCluster.getName());
+ indexedCluster.addTldsWithSameIdsAsContainers(tldParent, containerCluster);
+ }
+ indexedCluster.setupDispatchGroups();
+ }
+
+ /** Select/creates and initializes the indexing cluster coupled to this */
+ private void initializeIndexingClusters(Collection<ContainerModel> containers,
+ ConfigModelRepoAdder configModelRepoAdder,
+ ApplicationConfigProducerRoot root) {
+ if (getCluster().getSearch().hasIndexedCluster())
+ initializeOrSetExistingIndexingCluster(getCluster().getSearch().getIndexed(), hostedVespa,
+ containers, configModelRepoAdder, root);
+ }
+
+ private void initializeOrSetExistingIndexingCluster(IndexedSearchCluster indexedSearchCluster,
+ boolean isHostedVespa,
+ Collection<ContainerModel> containers,
+ ConfigModelRepoAdder configModelRepoAdder,
+ ApplicationConfigProducerRoot root) {
+ if (indexedSearchCluster.hasExplicitIndexingCluster()) {
+ setExistingIndexingCluster(indexedSearchCluster, containers);
+ } else if (isHostedVespa) {
+ setContainerAsIndexingCluster(indexedSearchCluster, containers, configModelRepoAdder, root);
+ } else {
+ createImplicitIndexingCluster(indexedSearchCluster, configModelRepoAdder, root);
+ }
+ }
+
+ private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster,
+ Collection<ContainerModel> containers,
+ ConfigModelRepoAdder configModelRepoAdder,
+ ApplicationConfigProducerRoot root) {
+ if (containers.isEmpty()) {
+ createImplicitIndexingCluster(indexedSearchCluster, configModelRepoAdder, root);
+ } else {
+ ContainerCluster targetCluster = getContainerWithDocproc(containers);
+ if (targetCluster == null)
+ targetCluster = getContainerWithSearch(containers);
+ if (targetCluster == null)
+ targetCluster = containers.iterator().next().getCluster();
+
+ addDocproc(targetCluster);
+ indexedSearchCluster.setIndexingClusterName(targetCluster.getName());
+ addIndexingChainsTo(targetCluster, indexedSearchCluster);
+ }
+ }
+
+ private void setExistingIndexingCluster(IndexedSearchCluster cluster, Collection<ContainerModel> containers) {
+ String indexingClusterName = cluster.getIndexingClusterName();
+ ContainerModel containerModel = findByName(indexingClusterName, containers);
+ if (containerModel == null)
+ throw new RuntimeException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " +
+ "cluster '" + indexingClusterName + "', but this cluster does not exist.");
+ addIndexingChainsTo(containerModel.getCluster(), cluster);
+ }
+
+ private ContainerModel findByName(String name, Collection<ContainerModel> containers) {
+ for (ContainerModel container : containers)
+ if (container.getId().equals(name))
+ return container;
+ return null;
+ }
+
+ private void addIndexingChainsTo(ContainerCluster indexer, IndexedSearchCluster cluster) {
+ addIndexingChain(indexer);
+ DocprocChain indexingChain;
+ ComponentRegistry<DocprocChain> allChains = indexer.getDocprocChains().allChains();
+ if (cluster.hasExplicitIndexingChain()) {
+ indexingChain = allChains.getComponent(cluster.getIndexingChainName());
+ if (indexingChain == null) {
+ throw new RuntimeException("Indexing cluster " + cluster.getClusterName() + " refers to docproc " +
+ "chain " + cluster.getIndexingChainName() + " for indexing, which does not exist.");
+ } else {
+ checkThatExplicitIndexingChainInheritsCorrectly(allChains, indexingChain.getChainSpecification());
+ }
+ } else {
+ indexingChain = allChains.getComponent(IndexingDocprocChain.NAME);
+ }
+
+ cluster.setIndexingChain(indexingChain);
+ }
+
+ private static boolean checkParentChain(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) {
+ if (IndexingDocprocChain.NAME.equals(chainSpec.componentId.stringValue())) {
+ return true;
+ }
+
+ ChainSpecification.Inheritance inheritance = chainSpec.inheritance;
+ for (ComponentSpecification parentComponentSpec : inheritance.chainSpecifications) {
+ ChainSpecification parentSpec = getChainSpec(allChains, parentComponentSpec);
+ checkParentChain(allChains, parentSpec);
+ }
+
+ return false;
+ }
+
+ private static ChainSpecification getChainSpec(ComponentRegistry<DocprocChain> allChains, ComponentSpecification componentSpec) {
+ DocprocChain docprocChain = allChains.getComponent(componentSpec);
+ if (docprocChain == null) {
+ throw new IllegalArgumentException("Chain '" + componentSpec + "' not found.");
+ }
+ return docprocChain.getChainSpecification();
+ }
+
+ private static void addIndexingChain(ContainerCluster containerCluster) {
+ DocprocChain chainAlreadyPresent = containerCluster.getDocprocChains().allChains().
+ getComponent(new ComponentId(IndexingDocprocChain.NAME));
+ if (chainAlreadyPresent != null) {
+ if (chainAlreadyPresent instanceof IndexingDocprocChain) {
+ return;
+ } else {
+ throw new IllegalArgumentException("A docproc chain may not have the ID '" +
+ IndexingDocprocChain.NAME + ", since this is reserved by Vespa. Please use a different ID.");
+ }
+ }
+
+ containerCluster.getDocprocChains().add(new IndexingDocprocChain());
+ }
+
+ /** Create a new container cluster for indexing and add it to the Vespa model */
+ private void createImplicitIndexingCluster(IndexedSearchCluster cluster,
+ ConfigModelRepoAdder configModelRepoAdder,
+ ApplicationConfigProducerRoot root) {
+ String indexerName = cluster.getIndexingClusterName();
+ AbstractConfigProducer p = root.getChildren().get(ContainerModel.DOCPROC_RESERVED_NAME);
+ if (p == null)
+ p = new SimpleConfigProducer(root, ContainerModel.DOCPROC_RESERVED_NAME);
+ ConfigModelContext context = ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, ContainerModel.DOCPROC_RESERVED_NAME);
+ ContainerCluster indexingCluster = new ContainerCluster(context.getParentProducer(), "cluster." + indexerName, indexerName);
+ ContainerModel indexingClusterModel = new ContainerModel(ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, indexingCluster.getSubId()));
+ indexingClusterModel.setCluster(indexingCluster);
+ configModelRepoAdder.add(indexingClusterModel);
+ ownedIndexingCluster = Optional.of(indexingCluster);
+
+ ContainerModelBuilder.addDefaultHandler_legacyBuilder(indexingCluster);
+
+ addDocproc(indexingCluster);
+
+ List<Container> nodes = new ArrayList<>();
+ int index = 0;
+ Set<HostResource> processedHosts = new LinkedHashSet<>();
+ boolean isElastic = cluster.isElastic();
+ for (SearchNode searchNode : cluster.getSearchNodes()) {
+ HostResource host = searchNode.getHostResource();
+ if (!processedHosts.contains(host)) {
+ String containerName = String.valueOf(isElastic ? searchNode.getDistributionKey() : index++);
+ Container docprocService = new Container(indexingCluster, containerName);
+ docprocService.setBasePort(host.nextAvailableBaseport(docprocService.getPortCount()));
+ docprocService.setHostResource(host);
+ docprocService.initService();
+ nodes.add(docprocService);
+ processedHosts.add(host);
+ }
+ }
+ indexingCluster.addContainers(nodes);
+
+ addIndexingChain(indexingCluster);
+ cluster.setIndexingChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME));
+ }
+
+ private void addDocproc(ContainerCluster cluster) {
+ if (cluster.getDocproc() == null) {
+ DocprocChains chains = new DocprocChains(cluster, "docprocchains");
+ ContainerDocproc containerDocproc = new ContainerDocproc(cluster, chains);
+ cluster.setDocproc(containerDocproc);
+ }
+ }
+
+ private ContainerCluster getContainerWithDocproc(Collection<ContainerModel> containers) {
+ for (ContainerModel container : containers)
+ if (container.getCluster().getDocproc() != null)
+ return container.getCluster();
+ return null;
+ }
+
+ private static ContainerCluster getContainerWithSearch(Collection<ContainerModel> containers) {
+ for (ContainerModel container : containers)
+ if (container.getCluster().getSearch() != null)
+ return container.getCluster();
+ return null;
+ }
+
+ private static void checkThatExplicitIndexingChainInheritsCorrectly(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) {
+ ChainSpecification.Inheritance inheritance = chainSpec.inheritance;
+ boolean found = false;
+ for (ComponentSpecification componentSpec : inheritance.chainSpecifications) {
+ ChainSpecification parentSpec = getChainSpec(allChains, componentSpec);
+ found = checkParentChain(allChains, parentSpec);
+ if (found) {
+ break;
+ }
+ }
+ if (!found) {
+ throw new IllegalArgumentException("Docproc chain '" + chainSpec.componentId + "' does not inherit from 'indexing' chain.");
+ }
+ }
+
+ public static List<Content> getContent(ConfigModelRepo pc) {
+ List<Content> contents = new ArrayList<>();
+
+ for (ConfigModel model : pc.asMap().values()) {
+ if (model instanceof Content) {
+ contents.add((Content)model);
+ }
+ }
+
+ return contents;
+ }
+
+ public static List<AbstractSearchCluster> getSearchClusters(ConfigModelRepo pc) {
+ List<AbstractSearchCluster> clusters = new ArrayList<>();
+
+ for (ContentCluster c : getContentClusters(pc)) {
+ clusters.addAll(c.getSearch().getClusters().values());
+ }
+
+ return clusters;
+ }
+
+ public static List<ContentCluster> getContentClusters(ConfigModelRepo pc) {
+ List<ContentCluster> clusters = new ArrayList<>();
+
+ for (Content c : getContent(pc)) {
+ clusters.add(c.getCluster());
+ }
+
+ return clusters;
+ }
+
+ @Override
+ public void prepare(ConfigModelRepo models) {
+ if (cluster.getRootGroup().useCpuSocketAffinity()) {
+ setCpuSocketAffinity();
+ }
+ if (cluster.getRootGroup().getMmapNoCoreLimit().isPresent()) {
+ for (AbstractService s : cluster.getSearch().getSearchNodes()) {
+ s.setMMapNoCoreLimit(cluster.getRootGroup().getMmapNoCoreLimit().get());
+ }
+ }
+ cluster.prepare();
+ }
+
+ private void setCpuSocketAffinity() {
+ // Currently only distribute affinity for search nodes
+ AbstractService.distributeCpuSocketAffinity(cluster.getSearch().getSearchNodes());
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java
new file mode 100644
index 00000000000..77a90e8a515
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.metrics.MetricsmanagerConfig;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig;
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.config.content.core.StorStatusConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+
+/**
+ * Common class for config producers for storage and distributor nodes.
+ *
+ * TODO: Author
+ */
+@RestartConfigs({StorCommunicationmanagerConfig.class, StorStatusConfig.class,
+ StorServerConfig.class, LoadTypeConfig.class, MetricsmanagerConfig.class})
+public abstract class ContentNode extends AbstractService
+ implements StorCommunicationmanagerConfig.Producer, StorStatusConfig.Producer, StorServerConfig.Producer {
+
+ protected int distributionKey;
+ String rootDirectory;
+
+ public ContentNode(AbstractConfigProducer parent, String clusterName, String rootDirectory, int distributionKey) {
+ super(parent, "" + distributionKey);
+ this.distributionKey = distributionKey;
+ initialize(distributionKey);
+
+ this.rootDirectory = rootDirectory;
+
+ setProp("clustertype", "content");
+ setProp("clustername", clusterName);
+ setProp("index", distributionKey);
+ }
+
+ public int getDistributionKey() {
+ return distributionKey;
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ builder.root_folder(rootDirectory);
+ builder.node_index(distributionKey);
+ }
+
+ public void initialize(int distributionKey) {
+ this.distributionKey = distributionKey;
+ portsMeta.on(0).tag("messaging");
+ portsMeta.on(1).tag("rpc").tag("status");
+ portsMeta.on(2).tag("http").tag("status").tag("state");
+
+ monitorService();
+ }
+
+ @Override
+ public int getPortCount() { return 3; }
+
+
+ @Override
+ public void getConfig(StorCommunicationmanagerConfig.Builder builder) {
+ builder.mbusport(getRelativePort(0));
+ builder.rpcport(getRelativePort(1));
+ }
+
+ @Override
+ public void getConfig(StorStatusConfig.Builder builder) {
+ builder.httpport(getRelativePort(2));
+ }
+
+ @Override
+ public int getHealthPort() {
+ return getRelativePort(2);
+ }
+
+ public String getRootDirectory() {
+ return rootDirectory;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java
new file mode 100644
index 00000000000..fdfb7f75188
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class ContentSearch {
+
+ private final Double queryTimeout;
+ private final Double visibilityDelay;
+
+ private ContentSearch(Builder builder) {
+ queryTimeout = builder.queryTimeout;
+ visibilityDelay = builder.visibilityDelay;
+ }
+
+ public Double getQueryTimeout() {
+ return queryTimeout;
+ }
+
+ public Double getVisibilityDelay() {
+ return visibilityDelay;
+ }
+
+ public static class Builder {
+
+ private Double queryTimeout;
+ private Double visibilityDelay;
+
+ public ContentSearch build() {
+ return new ContentSearch(this);
+ }
+
+ public Builder setQueryTimeout(Double queryTimeout) {
+ this.queryTimeout = queryTimeout;
+ return this;
+ }
+
+ public Builder setVisibilityDelay(Double visibilityDelay) {
+ this.visibilityDelay = visibilityDelay;
+ return this;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
new file mode 100644
index 00000000000..b5cbb23233a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -0,0 +1,299 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.documentmodel.DocumentTypeRepo;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.builder.UserConfigBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder;
+import com.yahoo.vespa.model.search.*;
+import org.w3c.dom.Element;
+
+
+import java.util.*;
+
+/**
+ * Encapsulates the various options for search in a content model.
+ * Wraps a search cluster from com.yahoo.vespa.model.search.
+ */
+public class ContentSearchCluster extends AbstractConfigProducer implements ProtonConfig.Producer, DispatchConfig.Producer {
+
+ private final boolean flushOnShutdown;
+
+ /** If this is set up for streaming search, it is modelled as one search cluster per search definition */
+ private Map<String, AbstractSearchCluster> clusters = new TreeMap<>();
+
+ /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */
+ private IndexedSearchCluster indexedCluster;
+
+ private final String clusterName;
+ Map<String, NewDocumentType> documentDefinitions;
+
+ /** The search nodes of this if it does not have an indexed cluster */
+ List<SearchNode> nonIndexed = new ArrayList<>();
+
+ Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>();
+ private DocumentTypeRepo repo = null;
+ private Optional<ResourceLimits> resourceLimits = Optional.empty();
+
+ public void prepare() {
+ repo = getRoot().getDeployState().getDocumentModel().getDocumentManager();
+ }
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> {
+
+ private final Map<String, NewDocumentType> documentDefinitions;
+
+ public Builder(Map<String, NewDocumentType> documentDefinitions) {
+ this.documentDefinitions = documentDefinitions;
+ }
+
+ @Override
+ protected ContentSearchCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ ModelElement clusterElem = new ModelElement(producerSpec);
+ String clusterName = ContentCluster.getClusterName(clusterElem);
+ Boolean flushOnShutdown = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown");
+
+ ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, flushOnShutdown != null ? flushOnShutdown : false);
+
+ ModelElement tuning = clusterElem.getChildByPath("engine.proton.tuning");
+ if (tuning != null) {
+ search.setTuning(new DomSearchTuningBuilder().build(search, tuning.getXml()));
+ }
+ ModelElement protonElem = clusterElem.getChildByPath("engine.proton");
+ if (protonElem != null) {
+ search.setResourceLimits(DomResourceLimitsBuilder.build(protonElem));
+ }
+
+ buildAllStreamingSearchClusters(clusterElem, clusterName, search);
+ buildIndexedSearchCluster(clusterElem, clusterName, search);
+ return search;
+ }
+
+ private Double getQueryTimeout(ModelElement clusterElem) {
+ return clusterElem.childAsDouble("engine.proton.query-timeout");
+ }
+
+ private void buildAllStreamingSearchClusters(ModelElement clusterElem, String clusterName, ContentSearchCluster search) {
+ ModelElement docElem = clusterElem.getChild("documents");
+
+ if (docElem == null) {
+ return;
+ }
+
+ for (ModelElement docType : docElem.subElements("document")) {
+ String mode = docType.getStringAttribute("mode");
+ if ("streaming".equals(mode)) {
+ buildStreamingSearchCluster(clusterElem, clusterName, search, docType);
+ }
+ }
+ }
+
+ private void buildStreamingSearchCluster(ModelElement clusterElem, String clusterName, ContentSearchCluster search, ModelElement docType) {
+ StreamingSearchCluster cluster = new StreamingSearchCluster(search, clusterName + "." + docType.getStringAttribute("type"), 0, clusterName, clusterName);
+
+ List<ModelElement> def = new ArrayList<>();
+ def.add(docType);
+ search.addSearchCluster(cluster, getQueryTimeout(clusterElem), def);
+ }
+
+ private void buildIndexedSearchCluster(ModelElement clusterElem,
+ String clusterName,
+ ContentSearchCluster search) {
+ List<ModelElement> indexedDefs = getIndexedSearchDefinitions(clusterElem);
+ if (!indexedDefs.isEmpty()) {
+ IndexedSearchCluster isc = new IndexedElasticSearchCluster(search, clusterName, 0);
+ isc.setRoutingSelector(clusterElem.childAsString("documents.selection"));
+
+ Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
+ if (visibilityDelay != null) {
+ isc.setVisibilityDelay(visibilityDelay);
+ }
+
+ search.addSearchCluster(isc, getQueryTimeout(clusterElem), indexedDefs);
+ }
+ }
+
+ private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) {
+ List<ModelElement> indexedDefs = new ArrayList<>();
+ ModelElement docElem = clusterElem.getChild("documents");
+ if (docElem == null) {
+ return indexedDefs;
+ }
+
+ for (ModelElement docType : docElem.subElements("document")) {
+ String mode = docType.getStringAttribute("mode");
+ if ("index".equals(mode)) {
+ indexedDefs.add(docType);
+ }
+ }
+ return indexedDefs;
+ }
+ }
+
+ private ContentSearchCluster(AbstractConfigProducer parent,
+ String clusterName,
+ Map<String, NewDocumentType> documentDefinitions, boolean flushOnShutdown)
+ {
+ super(parent, "search");
+ this.clusterName = clusterName;
+ this.documentDefinitions = documentDefinitions;
+ this.flushOnShutdown = flushOnShutdown;
+ }
+
+ void addSearchCluster(SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) {
+ addSearchDefinitions(documentDefs, cluster);
+
+ if (queryTimeout != null) {
+ cluster.setQueryTimeout(queryTimeout);
+ }
+ cluster.defaultDocumentsConfig();
+ cluster.deriveSearchDefinitions(new ArrayList<>());
+ addCluster(cluster);
+ }
+
+ private void addSearchDefinitions(List<ModelElement> searchDefs, AbstractSearchCluster sc) {
+ for (ModelElement e : searchDefs) {
+ SearchDefinitionXMLHandler searchDefinitionXMLHandler = new SearchDefinitionXMLHandler(e);
+ SearchDefinition searchDefinition =
+ searchDefinitionXMLHandler.getResponsibleSearchDefinition(sc.getRoot().getDeployState().getSearchDefinitions());
+ if (searchDefinition == null)
+ throw new RuntimeException("Search definition parsing error or file does not exist: '" +
+ searchDefinitionXMLHandler.getName() + "'");
+
+ // TODO: remove explicit building of user configs when the complete content model is built using builders.
+ sc.getLocalSDS().add(new AbstractSearchCluster.SearchDefinitionSpec(searchDefinition,
+ UserConfigBuilder.build(e.getXml(), sc.getRoot().getDeployState(), sc.getRoot().deployLogger())));
+ //need to get the document names from this sdfile
+ sc.addDocumentNames(searchDefinition);
+ }
+ }
+
+ public ContentSearchCluster addCluster(AbstractSearchCluster sc) {
+ if (clusters.containsKey(sc.getClusterName())) {
+ throw new IllegalArgumentException("I already have registered cluster '" + sc.getClusterName() + "'");
+ }
+ if (sc instanceof IndexedSearchCluster) {
+ if (indexedCluster != null) {
+ throw new IllegalArgumentException("I already have one indexed cluster named '" + indexedCluster.getClusterName());
+ }
+ indexedCluster = (IndexedSearchCluster)sc;
+ }
+ clusters.put(sc.getClusterName(), sc);
+ return this;
+ }
+
+ public List<SearchNode> getSearchNodes() {
+ return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed;
+ }
+
+ public SearchNode addSearchNode(ContentNode node, StorageGroup parentGroup, ModelElement element) {
+ AbstractConfigProducer parent = hasIndexedCluster() ? getIndexed() : this;
+
+ NodeSpec spec = getNextSearchNodeSpec(parentGroup);
+ SearchNode snode;
+ TransactionLogServer tls;
+ if (element == null) {
+ snode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown);
+ snode.setHostResource(node.getHostResource());
+ snode.initService();
+
+ tls = new TransactionLogServer(snode, clusterName);
+ tls.setHostResource(snode.getHostResource());
+ tls.initService();
+ } else {
+ snode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown).build(parent, element.getXml());
+ tls = new TransactionLogServer.Builder(clusterName).build(snode, element.getXml());
+ }
+ snode.setTls(tls);
+ if (hasIndexedCluster()) {
+ getIndexed().addSearcher(snode);
+ } else {
+ nonIndexed.add(snode);
+ }
+ return snode;
+ }
+
+ private NodeSpec getNextSearchNodeSpec(StorageGroup parentGroup) {
+ NodeSpec spec = groupToSpecMap.get(parentGroup);
+ if (spec == null) {
+ spec = new NodeSpec(groupToSpecMap.size(), 0);
+ } else {
+ spec = new NodeSpec(spec.rowId(), spec.partitionId() + 1);
+ }
+ groupToSpecMap.put(parentGroup, spec);
+ return spec;
+ }
+
+ private Tuning tuning;
+
+ public void setTuning(Tuning t) {
+ tuning = t;
+ }
+
+ public void setResourceLimits(ResourceLimits resourceLimits) {
+ this.resourceLimits = Optional.of(resourceLimits);
+ }
+
+ public boolean usesHierarchicDistribution() {
+ return indexedCluster != null && groupToSpecMap.size() > 1;
+ }
+
+ public void handleRedundancy(Redundancy redundancy) {
+ if (usesHierarchicDistribution()) {
+ indexedCluster.setMaxNodesDownPerFixedRow((redundancy.effectiveFinalRedundancy() / groupToSpecMap.size()) - 1);
+ }
+ }
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0;
+ for (NewDocumentType type : documentDefinitions.values()) {
+ ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
+ ddbB.inputdoctypename(type.getFullName().getName())
+ .configid(getConfigId())
+ .visibilitydelay(visibilityDelay);
+ if (hasIndexedCluster()) {
+ getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB);
+ }
+ builder.documentdb(ddbB);
+ }
+ for (AbstractSearchCluster sc : getClusters().values()) {
+ if (sc instanceof StreamingSearchCluster) {
+ NewDocumentType type = repo.getDocumentType(((StreamingSearchCluster)sc).getSdConfig().getSearch().getName());
+ ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder();
+ ddbB.inputdoctypename(type.getFullName().getName()).configid(getConfigId());
+ builder.documentdb(ddbB);
+ }
+ }
+ int numDocumentDbs = builder.documentdb.size();
+ builder.initialize(new ProtonConfig.Initialize.Builder().threads(numDocumentDbs + 1));
+
+ if (resourceLimits.isPresent()) {
+ resourceLimits.get().getConfig(builder);
+ }
+
+ if (tuning != null) {
+ tuning.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(DispatchConfig.Builder builder) {
+ if (hasIndexedCluster()) {
+ getIndexed().getConfig(builder);
+ }
+ }
+
+ public Map<String, AbstractSearchCluster> getClusters() { return clusters; }
+ public IndexedSearchCluster getIndexed() { return indexedCluster; }
+ public boolean hasIndexedCluster() { return indexedCluster != null; }
+ public String getClusterName() { return clusterName; }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java
new file mode 100644
index 00000000000..ad69cced45b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents the dispatch setup for a content cluster.
+ * This EITHER has a number of dispatch groups OR a an explicit list of groups,
+ * for unknown reasons (talk to Geir).
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DispatchSpec {
+
+ private final Integer numDispatchGroups;
+ private final List<Group> groups;
+
+ private DispatchSpec(Builder builder) {
+ numDispatchGroups = builder.numDispatchGroups;
+ groups = builder.groups;
+ }
+
+ public Integer getNumDispatchGroups() { return numDispatchGroups; }
+
+ public List<Group> getGroups() {
+ return groups;
+ }
+
+ public boolean valid() {
+ return numDispatchGroups != null || groups != null;
+ }
+
+ /**
+ * Reference to a node which is contained in a dispatch group.
+ */
+ public static class Node {
+ private final int distributionKey;
+ public Node(int distributionKey) {
+ this.distributionKey = distributionKey;
+ }
+ public int getDistributionKey() {
+ return distributionKey;
+ }
+ }
+
+ /**
+ * A dispatch group with a list of nodes contained in that group.
+ */
+ public static class Group {
+ private final List<Node> nodes = new ArrayList<>();
+ public Group() {
+
+ }
+ public Group addNode(Node node) {
+ nodes.add(node);
+ return this;
+ }
+ public List<Node> getNodes() {
+ return nodes;
+ }
+ }
+
+ public static class Builder {
+
+ private Integer numDispatchGroups;
+ private List<Group> groups;
+
+ public DispatchSpec build() {
+ return new DispatchSpec(this);
+ }
+
+ public Builder setNumDispatchGroups(Integer numDispatchGroups) {
+ this.numDispatchGroups = numDispatchGroups;
+ return this;
+ }
+
+ public Builder setGroups(List<Group> groups) {
+ this.groups = groups;
+ return this;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java
new file mode 100644
index 00000000000..b6462d5aa9b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+public class DistributionBitCalculator {
+
+ public static int getDistributionBits(int nodes, ContentCluster.DistributionMode mode) {
+ if (mode == ContentCluster.DistributionMode.STRICT) {
+ if (nodes < 5) {
+ return 8; // Require few buckets in small clusters to ease testing
+ } else if (nodes < 15) {
+ return 16;
+ } else if (nodes < 200) {
+ return 21;
+ } else if (nodes < 800) {
+ return 25;
+ } else if (nodes < 1500) {
+ return 28;
+ } else if (nodes < 5000) {
+ return 30;
+ } else {
+ return 32;
+ }
+ } else if (mode == ContentCluster.DistributionMode.LOOSE) {
+ if (nodes < 5) {
+ return 8;
+ } else if (nodes < 200) {
+ return 16;
+ } else {
+ return 24;
+ }
+ } else if (mode == ContentCluster.DistributionMode.LEGACY) {
+ if (nodes == 1) { // min 1 bucket/node
+ return 1;
+ } else if (nodes == 2) { // min 128 buckets/node
+ return 8;
+ } else if (nodes <= 6) { // min 5462 buckets/node
+ return 14;
+ } else if (nodes <= 8) { // min 9362 buckets/node
+ return 16;
+ } else if (nodes <= 10) { // min 14563 buckets/node
+ return 17;
+ } else if (nodes <= 12) { // min 23832 buckets/node
+ return 18;
+ } else if (nodes <= 20) { // min 40329 buckets/node
+ return 19;
+ } else if (nodes <= 32) { // min 49933 buckets/node
+ return 20;
+ } else if (nodes <= 64) { // min 63550 buckets/node
+ return 21;
+ } else if (nodes <= 100) { // min 64528 buckets/node
+ return 22;
+ } else if (nodes <= 256) { // min 83056 buckets/node
+ return 23;
+ } else if (nodes <= 350) { // min 65281 buckets/node
+ return 24;
+ } else if (nodes <= 500) { // min 95597 buckets/node
+ return 25;
+ } else if (nodes <= 1024) { // min 133950 buckets/node
+ return 26;
+ } else if (nodes <= 2048) { // min 130944 buckets/node
+ return 27;
+ } else if (nodes <= 4096) { // min 130008 buckets/node
+ return 28;
+ } else if (nodes <= 8192) { // min 131040 buckets/node
+ return 29;
+ } else if (nodes <= 16384) { // min 131056 buckets/node
+ return 30;
+ } else if (nodes <= 32768) { // min 131064 buckets/node
+ return 31;
+ } else { // min 131068 buckets/node
+ return 32;
+ }
+ } else {
+ throw new IllegalArgumentException("We don't know how to handle mode " + mode);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java
new file mode 100644
index 00000000000..5458bd366d5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.engines.PersistenceEngine;
+import org.w3c.dom.Element;
+
+/**
+ * Represents specific configuration for a given distributor node.
+ */
+public class Distributor extends ContentNode {
+
+ PersistenceEngine provider;
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<Distributor> {
+ ModelElement clusterXml;
+ PersistenceEngine persistenceProvider;
+
+ public Builder(ModelElement clusterXml, PersistenceEngine persistenceProvider) {
+ this.clusterXml = clusterXml;
+ this.persistenceProvider = persistenceProvider;
+ }
+
+ @Override
+ protected Distributor doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ return new Distributor((DistributorCluster)ancestor, new ModelElement(producerSpec).getIntegerAttribute("distribution-key"),
+ clusterXml.getIntegerAttribute("distributor-base-port"), persistenceProvider);
+ }
+ }
+
+ Distributor(DistributorCluster parent, int distributionKey, Integer distributorBasePort, PersistenceEngine provider) {
+ super(parent, parent.getClusterName(), StorageNode.rootFolder + parent.getClusterName() + "/distributor/"
+ + distributionKey, distributionKey);
+
+ this.provider = provider;
+
+ if (distributorBasePort != null) {
+ setBasePort(distributorBasePort);
+ }
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ super.getConfig(builder);
+ provider.getConfig(builder);
+ }
+
+ @Override
+ public String getStartupCommand() {
+ return "exec sbin/distributord -c $VESPA_CONFIG_ID";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
new file mode 100644
index 00000000000..dc27da7461d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.document.select.DocumentSelector;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.metrics.MetricsmanagerConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import org.w3c.dom.Element;
+
+import java.util.logging.Logger;
+
+
+/**
+ * Generates distributor-specific configuration.
+ */
+public class DistributorCluster extends AbstractConfigProducer<Distributor>
+ implements StorDistributormanagerConfig.Producer, StorServerConfig.Producer, MetricsmanagerConfig.Producer {
+
+ public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString());
+
+ private static class GcOptions {
+ public final int interval;
+ public final String selection;
+
+ public GcOptions(int interval, String selection) {
+ this.interval = interval;
+ this.selection = selection;
+ }
+ }
+
+ private final ContentCluster parent;
+ private final BucketSplitting bucketSplitting;
+ private final GcOptions gc;
+ private final boolean hasIndexedDocumentType;
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> {
+ ContentCluster parent;
+
+ public Builder(ContentCluster parent) {
+ this.parent = parent;
+ }
+
+ private String prepareGCSelection(ModelElement documentNode, String selStr) throws ParseException {
+ DocumentSelector s = new DocumentSelector(selStr);
+ boolean enableGC = false;
+ if (documentNode != null) {
+ enableGC = documentNode.getBooleanAttribute("garbage-collection", false);
+ }
+ if (!enableGC) {
+ return null;
+ }
+
+ return s.toString();
+ }
+
+ private int getGCInterval(ModelElement documentNode) throws ParseException {
+ int gcInterval = 3600;
+ if (documentNode != null) {
+ gcInterval = documentNode.getIntegerAttribute("garbage-collection-interval", gcInterval);
+ }
+ return gcInterval;
+ }
+
+ private GcOptions parseGcOptions(ModelElement documentNode) {
+ String gcSelection = parent.getRoutingSelector();
+ int gcInterval;
+ try {
+ if (gcSelection != null) {
+ gcSelection = prepareGCSelection(documentNode, gcSelection);
+ }
+ gcInterval = getGCInterval(documentNode);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Failed to parse garbage collection selection", e);
+ }
+ return new GcOptions(gcInterval, gcSelection);
+ }
+
+ private boolean documentModeImpliesIndexing(String mode) {
+ return "index".equals(mode);
+ }
+
+ private boolean clusterContainsIndexedDocumentType(ModelElement documentsNode) {
+ // The presence of at least one <document> and its mode attribute is schema-enforced.
+ return documentsNode.subElements("document").stream()
+ .anyMatch(node -> documentModeImpliesIndexing(node.getStringAttribute("mode")));
+ }
+
+ @Override
+ protected DistributorCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ final ModelElement clusterElement = new ModelElement(producerSpec);
+ final ModelElement documentsNode = clusterElement.getChild("documents");
+ final GcOptions gc = parseGcOptions(documentsNode);
+ final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode);
+
+ return new DistributorCluster(parent,
+ new BucketSplitting.Builder().build(
+ parent, new ModelElement(producerSpec)), gc, hasIndexedDocumentType);
+ }
+ }
+
+ private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting,
+ GcOptions gc, boolean hasIndexedDocumentType)
+ {
+ super(parent, "distributor");
+ this.parent = parent;
+ this.bucketSplitting = bucketSplitting;
+ this.gc = gc;
+ this.hasIndexedDocumentType = hasIndexedDocumentType;
+ }
+
+ @Override
+ public void getConfig(StorDistributormanagerConfig.Builder builder) {
+ if (gc.selection != null) {
+ builder.garbagecollection(new StorDistributormanagerConfig.Garbagecollection.Builder()
+ .selectiontoremove("not (" + gc.selection + ")")
+ .interval(gc.interval));
+ }
+ builder.enable_revert(parent.getPersistence().supportRevert());
+ builder.disable_bucket_activation(hasIndexedDocumentType == false);
+
+ bucketSplitting.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(MetricsmanagerConfig.Builder builder) {
+ ContentCluster.getMetricBuilder("log", builder).
+ addedmetrics("vds.distributor.docsstored").
+ addedmetrics("vds.distributor.bytesstored").
+ addedmetrics("vds.idealstate.delete_bucket.done_ok").
+ addedmetrics("vds.idealstate.merge_bucket.done_ok").
+ addedmetrics("vds.idealstate.split_bucket.done_ok").
+ addedmetrics("vds.idealstate.join_bucket.done_ok").
+ addedmetrics("vds.idealstate.buckets_rechecking");
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ builder.root_folder("");
+ builder.cluster_name(parent.getName());
+ builder.is_distributor(true);
+ }
+
+ public String getClusterName() {
+ return parent.getName();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java
new file mode 100644
index 00000000000..585ae691773
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.document.select.Visitor;
+import com.yahoo.document.select.rule.*;
+
+public abstract class DocumentTypeVisitor implements Visitor {
+ @Override
+ public void visit(ArithmeticNode arithmeticNode) {
+ for (ArithmeticNode.NodeItem item : arithmeticNode.getItems()) {
+ item.getNode().accept(this);
+ }
+ }
+
+ @Override
+ public void visit(AttributeNode attributeNode) {
+ attributeNode.getValue().accept(this);
+ }
+
+ @Override
+ public void visit(ComparisonNode comparisonNode) {
+ comparisonNode.getLHS().accept(this);
+ comparisonNode.getRHS().accept(this);
+ }
+
+ @Override
+ public void visit(EmbracedNode embracedNode) {
+ embracedNode.getNode().accept(this);
+ }
+
+ @Override
+ public void visit(IdNode idNode) {
+ }
+
+ @Override
+ public void visit(LiteralNode literalNode) {
+ }
+
+ @Override
+ public void visit(LogicNode logicNode) {
+ for (LogicNode.NodeItem item : logicNode.getItems()) {
+ item.getNode().accept(this);
+ }
+ }
+
+ @Override
+ public void visit(NegationNode negationNode) {
+ negationNode.getNode().accept(this);
+ }
+
+ @Override
+ public void visit(NowNode nowNode) {
+ }
+
+ @Override
+ public void visit(SearchColumnNode searchColumnNode) {
+ }
+
+ @Override
+ public void visit(VariableNode variableNode) {
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java
new file mode 100644
index 00000000000..861088a80f5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Class used to validate that hierarchic distribution is correctly setup when having an indexed content cluster.
+ *
+ * Note that this class does not implement the com.yahoo.vespa.model.application.validation.Validator interface,
+ * but is instead used in the context of com.yahoo.vespa.model.ConfigProducer.validate() such that it can be unit tested
+ * without having to build the complete vespa model.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class IndexedHierarchicDistributionValidator {
+
+ private final String clusterName;
+ private final StorageGroup rootGroup;
+ private final Redundancy redundancy;
+ private final TuningDispatch.DispatchPolicy dispatchPolicy;
+
+ public IndexedHierarchicDistributionValidator(String clusterName,
+ StorageGroup rootGroup,
+ Redundancy redundancy,
+ TuningDispatch.DispatchPolicy dispatchPolicy) {
+ this.clusterName = clusterName;
+ this.rootGroup = rootGroup;
+ this.redundancy = redundancy;
+ this.dispatchPolicy = dispatchPolicy;
+ }
+
+ public void validate() throws Exception {
+ validateThatWeHaveOneGroupLevel();
+ validateThatLeafGroupsHasEqualNumberOfNodes();
+ validateThatLeafGroupsCountIsAFactorOfRedundancy();
+ validateThatRedundancyPerGroupIsEqual();
+ validateThatReadyCopiesIsCompatibleWithRedundancy(rootGroup.getSubgroups().size());
+ }
+
+ private void validateThatWeHaveOneGroupLevel() {
+ for (StorageGroup group : rootGroup.getSubgroups()) {
+ if (group.getSubgroups().size() > 0) {
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected all groups under root group '" +
+ rootGroup.getName() + "' to be leaf groups only containing nodes, but sub group '" + group.getName() + "' contains " +
+ group.getSubgroups().size() + " sub groups.");
+ }
+ }
+ }
+
+ private void validateThatLeafGroupsHasEqualNumberOfNodes() {
+ if (dispatchPolicy != TuningDispatch.DispatchPolicy.ROUNDROBIN) return;
+
+ StorageGroup previousGroup = null;
+ for (StorageGroup group : rootGroup.getSubgroups()) {
+ if (previousGroup == null) { // first group
+ previousGroup = group;
+ continue;
+ }
+
+ if (group.getNodes().size() != previousGroup.getNodes().size())
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected leaf groups to contain an equal number of nodes, but leaf group '" +
+ previousGroup.getName() + "' contains " + previousGroup.getNodes().size() + " node(s) while leaf group '" +
+ group.getName() + "' contains " + group.getNodes().size() + " node(s).");
+ previousGroup = group;
+ }
+ }
+
+ private void validateThatLeafGroupsCountIsAFactorOfRedundancy() {
+ if (redundancy.effectiveFinalRedundancy() % rootGroup.getSubgroups().size() != 0) {
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected number of leaf groups (" +
+ rootGroup.getSubgroups().size() + ") to be a factor of redundancy (" +
+ redundancy.effectiveFinalRedundancy() + "), but it is not.");
+ }
+ }
+
+ private void validateThatRedundancyPerGroupIsEqual() {
+ int redundancyPerGroup = redundancy.effectiveFinalRedundancy() / rootGroup.getSubgroups().size();
+ String expPartitions = createDistributionPartitions(redundancyPerGroup, rootGroup.getSubgroups().size());
+ if (!rootGroup.getPartitions().get().equals(expPartitions)) {
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected redundancy per leaf group to be " +
+ redundancyPerGroup + ", but it is not according to distribution partitions '" +
+ rootGroup.getPartitions().get() + "'. Expected distribution partitions should be '" + expPartitions + "'.");
+ }
+ }
+
+ private List<StorageNode> nonRetired(List<StorageNode> nodes) {
+ return nodes.stream().filter((node) -> { return !node.isRetired(); } ).collect(Collectors.toList());
+ }
+
+ private String createDistributionPartitions(int redundancyPerGroup, int numGroups) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < numGroups - 1; ++i) {
+ sb.append(redundancyPerGroup);
+ sb.append("|");
+ }
+ sb.append("*");
+ return sb.toString();
+ }
+
+ private void validateThatReadyCopiesIsCompatibleWithRedundancy(int groupCount) throws Exception {
+ if (redundancy.effectiveFinalRedundancy() % groupCount != 0) {
+ throw new Exception(getErrorMsgPrefix() + "Expected equal redundancy per group.");
+ }
+ if (redundancy.effectiveReadyCopies() % groupCount != 0) {
+ throw new Exception(getErrorMsgPrefix() + "Expected equal amount of ready copies per group, but " +
+ redundancy.effectiveReadyCopies() + " ready copies is specified with " + groupCount + " groups");
+ }
+ if (redundancy.effectiveReadyCopies() == 0) {
+ System.err.println(getErrorMsgPrefix() + "Warning. No ready copies configured. At least one is recommended.");
+ }
+ }
+
+ private String getErrorMsgPrefix() {
+ return "In indexed content cluster '" + clusterName + "' using hierarchic distribution: ";
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java b/config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java
new file mode 100644
index 00000000000..597949ec9f2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: thomasg
+ * Date: 5/7/12
+ * Time: 2:00 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class PriorityMapping {
+ ModelElement clusterXml;
+ Map<DocumentProtocol.Priority, Integer> priorityMappings = new HashMap<>();
+
+ public PriorityMapping(ModelElement clusterXml) {
+ this.clusterXml = clusterXml;
+
+ int val = 50;
+ for (DocumentProtocol.Priority p : DocumentProtocol.Priority.values()) {
+ priorityMappings.put(p, val);
+ val += 10;
+ }
+ priorityMappings.put(DocumentProtocol.Priority.HIGHEST, 0);
+ priorityMappings.put(DocumentProtocol.Priority.LOWEST, 255);
+ }
+
+ public int getPriorityMapping(String priorityName) {
+ return priorityMappings.get(Enum.valueOf(DocumentProtocol.Priority.class, priorityName));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
new file mode 100644
index 00000000000..262c985e733
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.content.StorDistributionConfig;
+
+/**
+ * Configuration of the redundancy of a content cluster.
+ *
+ * @author bratseth
+ */
+public class Redundancy implements StorDistributionConfig.Producer {
+
+ private final int initialRedundancy ;
+ private final int finalRedundancy;
+ private final int readyCopies;
+
+ private int implicitGroups = 1;
+
+ /** The total number of nodes available in this cluster (assigned when this becomes known) */
+ private int totalNodes = 0;
+
+ public Redundancy(int initialRedundancy, int finalRedundancy, int readyCopies) {
+ this.initialRedundancy = initialRedundancy;
+ this.finalRedundancy = finalRedundancy;
+ this.readyCopies = readyCopies;
+ }
+
+ /**
+ * Set the total number of nodes available in this cluster.
+ * This impacts the effective redundancy in the case where there are fewer nodes available than
+ * the requested redundancy.
+ */
+ public void setTotalNodes(int totalNodes) { this.totalNodes = totalNodes; }
+
+ /**
+ * Sets the number of groups resulting from implicit setup (groups attribute)
+ * in this cluster. With implicit groups the redundancy settings are taken to be
+ * <i>per group</i> and are multiplied by this number to get the effective <i>total</i>
+ * values returned in the config.
+ */
+ public void setImplicitGroups(int implicitGroups) { this.implicitGroups = implicitGroups; }
+
+ public int initialRedundancy() { return initialRedundancy; }
+ public int finalRedundancy() { return finalRedundancy; }
+ public int readyCopies() { return readyCopies; }
+
+ public int effectiveInitialRedundancy() { return Math.min(totalNodes, initialRedundancy * implicitGroups); }
+ public int effectiveFinalRedundancy() { return Math.min(totalNodes, finalRedundancy * implicitGroups); }
+ public int effectiveReadyCopies() { return Math.min(totalNodes, readyCopies * implicitGroups); }
+
+ @Override
+ public void getConfig(StorDistributionConfig.Builder builder) {
+ builder.initial_redundancy(effectiveInitialRedundancy());
+ builder.redundancy(effectiveFinalRedundancy());
+ builder.ready_copies(effectiveReadyCopies());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java
new file mode 100644
index 00000000000..bb16d542c20
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.defaults.Defaults;
+
+import java.util.Optional;
+
+/**
+ * Class tracking resource limits for a content cluster with engine proton.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class ResourceLimits implements ProtonConfig.Producer {
+
+ private final Optional<Double> diskLimit;
+ private final Optional<Double> memoryLimit;
+
+ private ResourceLimits(Builder builder) {
+ this.diskLimit = builder.diskLimit;
+ this.memoryLimit = builder.memoryLimit;
+ }
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ ProtonConfig.Writefilter.Builder writeFilterBuilder = new ProtonConfig.Writefilter.Builder();
+ if (diskLimit.isPresent()) {
+ writeFilterBuilder.disklimit(diskLimit.get());
+ }
+ if (memoryLimit.isPresent()) {
+ writeFilterBuilder.memorylimit(memoryLimit.get());
+ }
+ builder.writefilter(writeFilterBuilder);
+ }
+
+ public static class Builder {
+
+ private Optional<Double> diskLimit = Optional.empty();
+ private Optional<Double> memoryLimit = Optional.empty();
+
+ public ResourceLimits build() {
+ return new ResourceLimits(this);
+ }
+
+ public Builder setDiskLimit(double diskLimit) {
+ this.diskLimit = Optional.of(diskLimit);
+ return this;
+ }
+
+ public Builder setMemoryLimit(double memoryLimit) {
+ this.memoryLimit = Optional.of(memoryLimit);
+ return this;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java b/config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java
new file mode 100644
index 00000000000..4c93711ebd3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class SearchCoverage {
+
+ private final Double minimum;
+ private final Double minWaitAfterCoverageFactor;
+ private final Double maxWaitAfterCoverageFactor;
+
+ private SearchCoverage(Builder builder) {
+ minimum = builder.minimum;
+ minWaitAfterCoverageFactor = builder.minWaitAfterCoverageFactor;
+ maxWaitAfterCoverageFactor = builder.maxWaitAfterCoverageFactor;
+ }
+
+ public Double getMinimum() {
+ return minimum;
+ }
+
+ public Double getMinWaitAfterCoverageFactor() {
+ return minWaitAfterCoverageFactor;
+ }
+
+ public Double getMaxWaitAfterCoverageFactor() {
+ return maxWaitAfterCoverageFactor;
+ }
+
+ public static class Builder {
+
+ private Double minimum;
+ private Double minWaitAfterCoverageFactor;
+ private Double maxWaitAfterCoverageFactor;
+
+ public SearchCoverage build() {
+ return new SearchCoverage(this);
+ }
+
+ public Builder setMinimum(Double value) {
+ Preconditions.checkArgument(value == null || (value >= 0 && value <= 1),
+ "Expected value in range [0, 1], got " + value + ".");
+ minimum = value;
+ return this;
+ }
+
+ public Builder setMinWaitAfterCoverageFactor(Double value) {
+ Preconditions.checkArgument(value == null || (value >= 0 && value <= 1),
+ "Expected value in range [0, 1], got " + value + ".");
+ Preconditions.checkArgument(value == null || maxWaitAfterCoverageFactor == null ||
+ value <= maxWaitAfterCoverageFactor,
+ "Minimum wait (got %s) must be no larger than maximum wait (was %s).",
+ value, maxWaitAfterCoverageFactor);
+ minWaitAfterCoverageFactor = value;
+ return this;
+ }
+
+ public Builder setMaxWaitAfterCoverageFactor(Double value) {
+ Preconditions.checkArgument(value == null || (value >= 0 && value <= 1),
+ "Expected value in range [0, 1], got " + value + ".");
+ Preconditions.checkArgument(value == null || minWaitAfterCoverageFactor == null ||
+ value >= minWaitAfterCoverageFactor,
+ "Maximum wait (got %s) must be no smaller than minimum wait (was %s).",
+ value, minWaitAfterCoverageFactor);
+ maxWaitAfterCoverageFactor = value;
+ return this;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
new file mode 100644
index 00000000000..78255ce8f64
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -0,0 +1,420 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.content.engines.PersistenceEngine;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A group of storage nodes/distributors.
+ *
+ * @author unknown, probably thomasg
+ * @author bratseth has done things here recently
+ */
+public class StorageGroup {
+
+ private final boolean useCpuSocketAffinity;
+ private final String index;
+ private Optional<String> partitions;
+ String name;
+ private final ContentCluster owner;
+ private final Optional<Long> mmapNoCoreLimit;
+
+ private final List<StorageGroup> subgroups = new ArrayList<>();
+ private final List<StorageNode> nodes = new ArrayList<>();
+
+ /**
+ * Creates a storage group
+ *
+ * @param owner the cluster this group belongs to
+ * @param name the name of this group
+ * @param index the distribution-key index og this group
+ * @param partitions the distribution strategy to use to distribute content to subgroups or empty
+ * (meaning that the "*" distribution will be used) only if this is a leaf group
+ * (having nodes, not subgroups as children).
+ * @param useCpuSocketAffinity whether processes should be started with socket affinity
+ */
+ private StorageGroup(ContentCluster owner, String name, String index, Optional<String> partitions,
+ boolean useCpuSocketAffinity, Optional<Long> mmapNoCoreLimit)
+ {
+ this.owner = owner;
+ this.index = index;
+ this.name = name;
+ this.partitions = partitions;
+ this.useCpuSocketAffinity = useCpuSocketAffinity;
+ this.mmapNoCoreLimit = mmapNoCoreLimit;
+ }
+
+ /** Returns the name of this group, or null if it is the root group */
+ public String getName() { return name; }
+
+ /** Returns the subgroups of this, or an empty list if it is a leaf group */
+ public List<StorageGroup> getSubgroups() { return subgroups; }
+
+ /** Returns the nodes of this, or an empty list of it is not a leaf group */
+ public List<StorageNode> getNodes() { return nodes; }
+
+ public ContentCluster getOwner() { return owner; }
+
+ /** Returns the index of this group, or null if it is the root group */
+ public String getIndex() { return index; }
+
+ public Optional<String> getPartitions() { return partitions; }
+ public boolean useCpuSocketAffinity() { return useCpuSocketAffinity; }
+ public Optional<Long> getMmapNoCoreLimit() { return mmapNoCoreLimit; }
+
+ /** Returns all the nodes below this group */
+ public List<StorageNode> recursiveGetNodes() {
+ if ( ! nodes.isEmpty()) return nodes;
+ List<StorageNode> nodes = new ArrayList<>();
+ for (StorageGroup subgroup : subgroups)
+ nodes.addAll(subgroup.recursiveGetNodes());
+ return nodes;
+ }
+
+ public Collection<StorDistributionConfig.Group.Builder> getGroupStructureConfig() {
+ List<StorDistributionConfig.Group.Builder> groups = new ArrayList<>();
+
+ StorDistributionConfig.Group.Builder myGroup = new StorDistributionConfig.Group.Builder();
+ getConfig(myGroup);
+ groups.add(myGroup);
+
+ for (StorageGroup g : subgroups) {
+ groups.addAll(g.getGroupStructureConfig());
+ }
+
+ return groups;
+ }
+
+ public void getConfig(StorDistributionConfig.Group.Builder builder) {
+ builder.index(index == null ? "invalid" : index);
+ builder.name(name == null ? "invalid" : name);
+ if (partitions.isPresent())
+ builder.partitions(partitions.get());
+ for (StorageNode node : nodes) {
+ StorDistributionConfig.Group.Nodes.Builder nb = new StorDistributionConfig.Group.Nodes.Builder();
+ nb.index(node.getDistributionKey());
+ nb.retired(node.isRetired());
+ builder.nodes.add(nb);
+ }
+ builder.capacity(getCapacity());
+ }
+
+ public int getNumberOfLeafGroups() {
+ int count = subgroups.isEmpty() ? 1 : 0;
+ for (StorageGroup g : subgroups) {
+ count += g.getNumberOfLeafGroups();
+ }
+ return count;
+ }
+
+ public double getCapacity() {
+ double capacity = 0;
+ for (StorageNode node : nodes) {
+ capacity += node.getCapacity();
+ }
+ for (StorageGroup group : subgroups) {
+ capacity += group.getCapacity();
+ }
+ return capacity;
+ }
+
+ /** Returns the total number of nodes below this group */
+ public int countNodes() {
+ int nodeCount = nodes.size();
+ for (StorageGroup group : subgroups)
+ nodeCount += group.countNodes();
+ return nodeCount;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof StorageGroup) {
+ StorageGroup rhs = (StorageGroup)obj;
+ return this.index.equals(rhs.index) &&
+ this.name.equals(rhs.name) &&
+ this.partitions.equals(rhs.partitions);
+ }
+ return false;
+ }
+
+ public static class Builder {
+
+ private final ModelElement clusterElement;
+ private final ContentCluster owner;
+ private final DeployLogger deployLogger;
+
+ public Builder(ModelElement clusterElement, ContentCluster owner, DeployLogger deployLogger) {
+ this.clusterElement = clusterElement;
+ this.owner = owner;
+ this.deployLogger = deployLogger;
+ }
+
+ public StorageGroup buildRootGroup() {
+ Optional<ModelElement> group = Optional.ofNullable(clusterElement.getChild("group"));
+ Optional<ModelElement> nodes = getNodes(clusterElement);
+
+ if (group.isPresent() && nodes.isPresent())
+ throw new IllegalStateException("Both group and nodes exists, only one of these tags is legal");
+ if (group.isPresent() && (group.get().getStringAttribute("name") != null || group.get().getIntegerAttribute("distribution-key") != null))
+ owner.deployLogger().log(LogLevel.INFO, "'distribution-key' attribute on a content cluster's root group is ignored");
+
+ GroupBuilder groupBuilder = collectGroup(group, nodes, null, null);
+ if (owner.isHostedVespa()) {
+ return groupBuilder.buildHosted(owner, Optional.empty());
+ } else {
+ return groupBuilder.buildNonHosted(owner, Optional.empty());
+ }
+ }
+
+ /**
+ * Represents a storage group and can build storage nodes in both hosted and non-hosted environments.
+ */
+ private static class GroupBuilder {
+
+ private final StorageGroup storageGroup;
+
+ /* The explicitly defined subgroups of this */
+ private final List<GroupBuilder> subGroups;
+ private final List<XmlNodeBuilder> nodeBuilders;
+
+ /** The nodes explicitly specified as a nodes tag in this group, or empty if none */
+ private final Optional<NodesSpecification> nodeRequirement;
+
+ private final DeployLogger deployLogger;
+
+ private GroupBuilder(StorageGroup storageGroup, List<GroupBuilder> subGroups, List<XmlNodeBuilder> nodeBuilders,
+ Optional<NodesSpecification> nodeRequirement, DeployLogger deployLogger) {
+ this.storageGroup = storageGroup;
+ this.subGroups = subGroups;
+ this.nodeBuilders = nodeBuilders;
+ this.nodeRequirement = nodeRequirement;
+ this.deployLogger = deployLogger;
+ }
+
+ /**
+ * Builds a storage group for a nonhosted environment
+ *
+ * @param owner the cluster owning this
+ * @param parent the parent storage group, or empty if this is the root group
+ * @return the storage group build by this
+ */
+ public StorageGroup buildNonHosted(ContentCluster owner, Optional<GroupBuilder> parent) {
+ for (GroupBuilder subGroup : subGroups) {
+ storageGroup.subgroups.add(subGroup.buildNonHosted(owner, Optional.of(this)));
+ }
+ for (XmlNodeBuilder nodeBuilder : nodeBuilders) {
+ storageGroup.nodes.add(nodeBuilder.build(owner, storageGroup));
+ }
+
+ if ( ! parent.isPresent())
+ owner.redundancy().setTotalNodes(storageGroup.countNodes());
+
+ return storageGroup;
+ }
+
+ /**
+ * Builds a storage group for a hosted environment
+ *
+ * @param owner the cluster owning this
+ * @param parent the parent storage group, or empty if this is the root group
+ * @return the storage group build by this
+ */
+ public StorageGroup buildHosted(ContentCluster owner, Optional<GroupBuilder> parent) {
+ Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? allocateHosts(owner) : Collections.emptyMap();
+
+ Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping);
+ if (hostGroups.size() > 1) {
+ if (parent.isPresent())
+ throw new IllegalArgumentException("Cannot specify groups using the groups attribute in nested content groups");
+ owner.redundancy().setTotalNodes(hostMapping.size());
+
+ // Switch redundancy settings to meaning "per group"
+ owner.redundancy().setImplicitGroups(hostGroups.size());
+
+ // Compute partitions expression
+ int redundancyPerGroup = (int)Math.floor(owner.redundancy().effectiveFinalRedundancy() / hostGroups.size());
+ storageGroup.partitions = Optional.of(computePartitions(redundancyPerGroup, hostGroups.size()));
+
+ // create subgroups as returned from allocation
+ for (Map.Entry<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroup : hostGroups.entrySet()) {
+ String groupIndex = hostGroup.getKey().get().value();
+ StorageGroup subgroup = new StorageGroup(owner, groupIndex, groupIndex, Optional.empty(), false, Optional.empty());
+ for (Map.Entry<HostResource, ClusterMembership> host : hostGroup.getValue().entrySet()) {
+ subgroup.nodes.add(createStorageNode(owner, host.getKey(), subgroup, host.getValue()));
+ }
+ storageGroup.subgroups.add(subgroup);
+ }
+ }
+ else { // or otherwise just create the nodes directly on this group, or the explicitly enumerated subgroups
+ for (Map.Entry<HostResource, ClusterMembership> host : hostMapping.entrySet()) {
+ storageGroup.nodes.add(createStorageNode(owner, host.getKey(), storageGroup, host.getValue()));
+ }
+ for (GroupBuilder subGroup : subGroups) {
+ storageGroup.subgroups.add(subGroup.buildHosted(owner, Optional.of(this)));
+ }
+ if ( ! parent.isPresent())
+ owner.redundancy().setTotalNodes(storageGroup.countNodes());
+ }
+ return storageGroup;
+ }
+
+ /** This returns a partition string which specifies equal distribution between all groups */
+ // TODO: Make a partitions object
+ private String computePartitions(int redundancyPerGroup, int numGroups) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < numGroups - 1; ++i) {
+ sb.append(redundancyPerGroup);
+ sb.append("|");
+ }
+ sb.append("*");
+ return sb.toString();
+ }
+
+ private Map<HostResource, ClusterMembership> allocateHosts(ContentCluster parent) {
+ ClusterSpec.Id clusterId = ClusterSpec.Id.from(parent.getStorageNodes().getClusterName());
+ Optional<ClusterSpec.Group> groupId = storageGroup.getIndex() == null ? Optional.empty() : Optional.of(ClusterSpec.Group.from(storageGroup.getIndex()));
+ return nodeRequirement.get().provision(parent.getRoot().getHostSystem(), ClusterSpec.Type.content, clusterId, groupId, deployLogger);
+ }
+
+ /** Collect hosts per group */
+ private Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> collectAllocatedSubgroups(Map<HostResource, ClusterMembership> hostMapping) {
+ Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostsPerGroup = new LinkedHashMap<>();
+ for (Map.Entry<HostResource, ClusterMembership> entry : hostMapping.entrySet()) {
+ Optional<ClusterSpec.Group> group = entry.getValue().cluster().group();
+ Map<HostResource, ClusterMembership> hostsInGroup = hostsPerGroup.get(group);
+ if (hostsInGroup == null) {
+ hostsInGroup = new LinkedHashMap<>();
+ hostsPerGroup.put(group, hostsInGroup);
+ }
+ hostsInGroup.put(entry.getKey(), entry.getValue());
+ }
+ return hostsPerGroup;
+ }
+
+ }
+
+ private static class XmlNodeBuilder {
+ private final ModelElement clusterElement;
+ private final ModelElement element;
+
+ private XmlNodeBuilder(ModelElement clusterElement, ModelElement element) {
+ this.clusterElement = clusterElement;
+ this.element = element;
+ }
+
+ public StorageNode build(ContentCluster parent, StorageGroup storageGroup) {
+ StorageNode sNode = new StorageNode.Builder().build(parent.getStorageNodes(), element.getXml());
+ PersistenceEngine provider = parent.getPersistence().create(sNode, storageGroup, element);
+ new Distributor.Builder(clusterElement, provider).build(parent.getDistributorNodes(), element.getXml());
+ return sNode;
+ }
+ }
+
+ /**
+ * Creates a content group builder from a group and/or nodes element.
+ * These are the possibilities:
+ * <ul>
+ * <li>group and nodes is present: This is a leaf group specifying a set of nodes</li>
+ * <li>only group is present: This is a nonleaf group</li>
+ * <li>only nodes is present: This is the implicitly specified toplevel leaf group, or a set of groups
+ * specified using a group count attrbute.
+ * </ul>
+ */
+ private GroupBuilder collectGroup(Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) {
+ StorageGroup group = new StorageGroup(owner, name, index,
+ childAsString(groupElement, "distribution.partitions"),
+ booleanAttributeOr(groupElement, VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME, false),
+ childAsLong(groupElement, VespaDomBuilder.MMAP_NOCORE_LIMIT));
+
+ List<GroupBuilder> subGroups = groupElement.isPresent() ? collectSubGroups(group, groupElement.get()) : Collections.emptyList();
+
+ List<XmlNodeBuilder> explicitNodes = new ArrayList<>();
+ explicitNodes.addAll(collectExplicitNodes(groupElement));
+ explicitNodes.addAll(collectExplicitNodes(nodesElement));
+
+ if (subGroups.size() > 0 && explicitNodes.size() > 0)
+ throw new IllegalArgumentException("A group can contain either nodes or groups, but not both.");
+
+ Optional<NodesSpecification> nodeRequirement =
+ nodesElement.isPresent() && nodesElement.get().getStringAttribute("count") != null ? Optional.of(NodesSpecification.from(nodesElement.get())) : Optional.empty();
+ if (nodeRequirement.isPresent() && subGroups.size() > 0)
+ throw new IllegalArgumentException("A group can contain either explicit subgroups or a nodes specification, but not both.");
+ return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, deployLogger);
+ }
+
+ private Optional<String> childAsString(Optional<ModelElement> element, String childTagName) {
+ if ( ! element.isPresent()) return Optional.empty();
+ return Optional.ofNullable(element.get().childAsString(childTagName));
+ }
+ private Optional<Long> childAsLong(Optional<ModelElement> element, String childTagName) {
+ if ( ! element.isPresent()) return Optional.empty();
+ return Optional.ofNullable(element.get().childAsLong(childTagName));
+ }
+
+ private boolean booleanAttributeOr(Optional<ModelElement> element, String attributeName, boolean defaultValue) {
+ if ( ! element.isPresent()) return defaultValue;
+ return element.get().getBooleanAttribute(attributeName, defaultValue);
+ }
+
+ private Optional<ModelElement> getNodes(ModelElement groupOrNodesElement) {
+ if (groupOrNodesElement.getXml().getTagName().equals("nodes")) return Optional.of(groupOrNodesElement);
+ return Optional.ofNullable(groupOrNodesElement.getChild("nodes"));
+ }
+
+ private List<XmlNodeBuilder> collectExplicitNodes(Optional<ModelElement> groupOrNodesElement) {
+ if ( ! groupOrNodesElement.isPresent()) return Collections.emptyList();
+ List<XmlNodeBuilder> nodes = new ArrayList<>();
+ for (ModelElement n : groupOrNodesElement.get().subElements("node"))
+ nodes.add(new XmlNodeBuilder(clusterElement, n));
+ return nodes;
+ }
+
+ private List<GroupBuilder> collectSubGroups(StorageGroup parentGroup, ModelElement parentGroupElement) {
+ List<ModelElement> subGroupElements = parentGroupElement.subElements("group");
+ if (subGroupElements.size() > 1 && ! parentGroup.getPartitions().isPresent())
+ throw new IllegalArgumentException("'distribution' attribute is required with multiple subgroups");
+
+ List<GroupBuilder> subGroups = new ArrayList<>();
+ String indexPrefix = "";
+ if (parentGroup.index != null) {
+ indexPrefix = parentGroup.index + ".";
+ }
+ for (ModelElement g : subGroupElements) {
+ subGroups.add(collectGroup(Optional.of(g), Optional.ofNullable(g.getChild("nodes")), g.getStringAttribute("name"),
+ indexPrefix + g.getIntegerAttribute("distribution-key")));
+ }
+ return subGroups;
+ }
+
+ private static StorageNode createStorageNode(ContentCluster parent, HostResource hostResource, StorageGroup parentGroup, ClusterMembership clusterMembership) {
+ StorageNode sNode = new StorageNode(parent.getStorageNodes(), null, clusterMembership.index(), clusterMembership.retired());
+ sNode.setHostResource(hostResource);
+ sNode.initService();
+
+ // TODO: Supplying null as XML is not very nice
+ PersistenceEngine provider = parent.getPersistence().create(sNode, parentGroup, null);
+ Distributor d = new Distributor(parent.getDistributorNodes(), clusterMembership.index(), null, provider);
+ d.setHostResource(sNode.getHostResource());
+ d.initService();
+ return sNode;
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
new file mode 100644
index 00000000000..750a66a0dda
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.vespa.config.content.StorFilestorConfig;
+import com.yahoo.vespa.config.content.core.StorBucketmoverConfig;
+import com.yahoo.vespa.config.storage.StorDevicesConfig;
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.engines.PersistenceEngine;
+import com.yahoo.vespa.model.content.engines.ProtonProvider;
+import com.yahoo.vespa.model.content.storagecluster.StorageCluster;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+
+/**
+ * Class to provide config related to a specific storage node.
+ */
+@RestartConfigs({StorDevicesConfig.class, StorFilestorConfig.class,
+ StorMemfilepersistenceConfig.class, StorBucketmoverConfig.class})
+public class StorageNode extends ContentNode implements StorServerConfig.Producer, StorDevicesConfig.Producer {
+
+ static final String rootFolder = Defaults.getDefaults().vespaHome() + "var/db/vespa/vds/";
+
+ private final Double capacity;
+ private final boolean retired;
+ private final boolean isHostedVespa;
+ private boolean usesVdsEngine = false;
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageNode> {
+ @Override
+ protected StorageNode doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ ModelElement e = new ModelElement(producerSpec);
+ return new StorageNode((StorageCluster)ancestor, e.getDoubleAttribute("capacity"), e.getIntegerAttribute("distribution-key"), false);
+ }
+ }
+
+ StorageNode(StorageCluster cluster, Double capacity, int distributionKey, boolean retired) {
+ super(cluster,
+ cluster.getClusterName(),
+ rootFolder + cluster.getClusterName() + "/storage/" + distributionKey,
+ distributionKey);
+ this.retired = retired;
+ this.capacity = capacity;
+ this.isHostedVespa = cluster.getRoot().getDeployState().getProperties().hostedVespa();
+ }
+
+ @Override
+ public String getStartupCommand() {
+ return isProviderProton()
+ ? null
+ : "exec sbin/storaged -c $VESPA_CONFIG_ID";
+ }
+
+ @Override
+ public void getConfig(StorDevicesConfig.Builder builder) {
+ String root_folder = getRootDirectory();
+ builder.root_folder(root_folder);
+
+ // For VDS in hosted Vespa, we default to using the root_folder as the disk to store the data in.
+ // Setting disk_path will then
+ if (isHostedVespa && usesVdsEngine) {
+ // VDS looks up the first disk at the directory path root_folder/disks/d0.
+ builder.disk_path(Arrays.asList(root_folder + "/disks/d0"));
+ }
+ }
+
+ // 2015-08-11: Needed because of the following circular dependency:
+ // 1. StorageNode is created.
+ // 2. A particular persistence engine is picked depending on things (like the presence of engine/vds element)
+ // that are hidden from the code creating the StorageNode in (1).
+ // 3. The persistence engine depends on the StorageNode, e.g. it's a parent node.
+ //
+ // If the VDSEngine is picked in (2), we would like to know this in StorageNode::getConfig(). Hence this setter.
+ public void useVdsEngine() {
+ usesVdsEngine = true;
+ }
+
+
+ public double getCapacity() {
+ if (capacity != null) {
+ return capacity;
+ } else {
+ return 1.0;
+ }
+ }
+
+ /** Whether this node is configured as retired, which means all content should migrate off the node */
+ public boolean isRetired() { return retired; }
+
+ private boolean isProviderProton() {
+ for (AbstractConfigProducer producer : getChildren().values()) {
+ if (producer instanceof ProtonProvider) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ super.getConfig(builder);
+
+ builder.node_capacity(getCapacity());
+
+ for (AbstractConfigProducer producer : getChildren().values()) {
+ ((PersistenceEngine)producer).getConfig(builder);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java
new file mode 100644
index 00000000000..614be479d38
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class TuningDispatch {
+
+ private final Integer maxHitsPerPartition;
+ public static enum DispatchPolicy { ROUNDROBIN, RANDOM};
+ private final DispatchPolicy dispatchPolicy;
+ private final Boolean useLocalNode;
+ private final Double minGroupCoverage;
+ private final Double minActiveDocsCoverage;
+
+ private TuningDispatch(Builder builder)
+ {
+ maxHitsPerPartition = builder.maxHitsPerPartition;
+ dispatchPolicy = builder.dispatchPolicy;
+ useLocalNode = builder.useLocalNode;
+ minGroupCoverage = builder.minGroupCoverage;
+ minActiveDocsCoverage = builder.minActiveDocsCoverage;
+ }
+
+ public Integer getMaxHitsPerPartition() {
+ return maxHitsPerPartition;
+ }
+ public DispatchPolicy getDispatchPolicy() { return dispatchPolicy; }
+ public Boolean getUseLocalNode() { return useLocalNode; }
+ public Double getMinGroupCoverage() { return minGroupCoverage; }
+ public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; }
+
+ public static class Builder {
+
+ private Integer maxHitsPerPartition;
+ private DispatchPolicy dispatchPolicy = DispatchPolicy.ROUNDROBIN;
+ private Boolean useLocalNode;
+ private Double minGroupCoverage;
+ private Double minActiveDocsCoverage;
+
+ public TuningDispatch build() {
+ return new TuningDispatch(this);
+ }
+
+ public Builder setMaxHitsPerPartition(Integer maxHitsPerPartition) {
+ this.maxHitsPerPartition = maxHitsPerPartition;
+ return this;
+ }
+ public Builder setDispatchPolicy(String policy) {
+ if (policy == null) {
+ } else if ("round-robin".equals(policy.toLowerCase())) {
+ dispatchPolicy = DispatchPolicy.ROUNDROBIN;
+ } else {
+ dispatchPolicy = DispatchPolicy.valueOf(policy.toUpperCase());
+ }
+ return this;
+ }
+
+ public Builder setUseLocalNode(Boolean useLocalNode) {
+ this.useLocalNode = useLocalNode;
+ return this;
+ }
+ public Builder setMinGroupCoverage(Double minGroupCoverage) {
+ this.minGroupCoverage = minGroupCoverage;
+ return this;
+ }
+ public Builder setMinActiveDocsCoverage(Double minCoverage) {
+ this.minActiveDocsCoverage = minCoverage;
+ return this;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
new file mode 100644
index 00000000000..08685659d35
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -0,0 +1,604 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.yahoo.config.model.ConfigModelUtils;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig;
+import com.yahoo.vespa.config.content.FleetcontrollerConfig;
+import com.yahoo.vespa.config.content.StorDistributionConfig;
+import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.metrics.MetricsmanagerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.admin.Metric;
+import com.yahoo.vespa.model.admin.MetricsConsumer;
+import com.yahoo.vespa.model.admin.MonitoringSystem;
+import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster;
+import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent;
+import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerConfigurer;
+import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
+import com.yahoo.vespa.model.content.*;
+import com.yahoo.vespa.model.content.engines.PersistenceEngine;
+import com.yahoo.vespa.model.content.engines.ProtonEngine;
+import com.yahoo.vespa.model.content.engines.VDSEngine;
+import com.yahoo.vespa.model.content.storagecluster.StorageCluster;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+import com.yahoo.vespa.model.search.MultilevelDispatchValidator;
+import com.yahoo.vespa.model.search.Tuning;
+import org.w3c.dom.Element;
+
+import java.util.*;
+import java.util.logging.Level;
+import java.util.stream.Collectors;
+
+/**
+ * A content cluster.
+ *
+ * @author mostly somebody unknown
+ * @author bratseth
+ */
+public class ContentCluster extends AbstractConfigProducer implements StorDistributionConfig.Producer,
+ StorDistributormanagerConfig.Producer,
+ FleetcontrollerConfig.Producer,
+ MetricsmanagerConfig.Producer,
+ MessagetyperouteselectorpolicyConfig.Producer {
+
+ // TODO: Make private
+ private String documentSelection;
+ ContentSearchCluster search;
+ final Map<String, NewDocumentType> documentDefinitions;
+ com.yahoo.vespa.model.content.StorageGroup rootGroup;
+ StorageCluster storageNodes;
+ DistributorCluster distributorNodes;
+ private Redundancy redundancy;
+ ClusterControllerConfig clusterControllerConfig;
+ PersistenceEngine.PersistenceFactory persistenceFactory;
+ String clusterName;
+ Integer maxNodesPerMerge;
+
+ /**
+ * If multitenant or a cluster controller was explicitly configured in this cluster:
+ * The cluster controller cluster of this particular content cluster.
+ *
+ * Otherwise: null - the cluster controller is shared by all content clusters and part of Admin.
+ */
+ private ContainerCluster clusterControllers;
+
+ public enum DistributionMode { LEGACY, STRICT, LOOSE }
+ DistributionMode distributionMode;
+
+ public static class Builder {
+
+ /** The admin model of this system or null if none (which only happens in tests) */
+ private final Admin admin;
+ private final DeployLogger deployLogger;
+
+ public Builder(Admin admin, DeployLogger deployLogger) {
+ this.admin = admin;
+ this.deployLogger = deployLogger;
+ }
+
+ public ContentCluster build(AbstractConfigProducer ancestor, Element w3cContentElement) {
+ ModelElement contentElement = new ModelElement(w3cContentElement);
+
+ ModelElement documentsElement = contentElement.getChild("documents");
+ Map<String, NewDocumentType> documentDefinitions =
+ new SearchDefinitionBuilder().build(ancestor.getRoot().getDeployState().getDocumentModel().getDocumentManager(), documentsElement);
+
+ String routingSelection = new DocumentSelectionBuilder().build(contentElement.getChild("documents"));
+ Redundancy redundancy = new RedundancyBuilder().build(contentElement);
+
+ ContentCluster c = new ContentCluster(ancestor, getClusterName(contentElement), documentDefinitions,
+ routingSelection, redundancy);
+ c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(c, contentElement.getXml());
+ c.search = new ContentSearchCluster.Builder(documentDefinitions).build(c, contentElement.getXml());
+ c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c);
+ c.storageNodes = new StorageCluster.Builder().build(c, w3cContentElement);
+ c.distributorNodes = new DistributorCluster.Builder(c).build(c, w3cContentElement);
+ c.rootGroup = new StorageGroup.Builder(contentElement, c, deployLogger).buildRootGroup();
+ validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup);
+ c.search.handleRedundancy(redundancy);
+
+ IndexedSearchCluster index = c.search.getIndexed();
+ if (index != null) {
+ setupIndexedCluster(index, contentElement);
+ }
+
+ if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) {
+ throw new RuntimeException("If you have indexed search you need to have proton as engine");
+ }
+
+ if (documentsElement != null) {
+ ModelElement e = documentsElement.getChild("document-processing");
+ if (e != null) {
+ setupDocumentProcessing(c, e);
+ }
+ } else if (c.persistenceFactory != null) {
+ throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified.");
+ }
+
+ ModelElement tuning = contentElement.getChild("tuning");
+ if (tuning != null) {
+ setupTuning(c, tuning);
+ }
+
+ AbstractConfigProducerRoot root = ancestor.getRoot();
+ if (root == null) return c;
+
+ addClusterControllers(root, c.rootGroup, contentElement, c.clusterName, c);
+ return c;
+ }
+
+ private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element) {
+ ContentSearch search = DomContentSearchBuilder.build(element);
+ Double queryTimeout = search.getQueryTimeout();
+ if (queryTimeout != null) {
+ Preconditions.checkState(index.getQueryTimeout() == null,
+ "You may not specify query-timeout in both proton and content.");
+ index.setQueryTimeout(queryTimeout);
+ }
+ Double visibilityDelay = search.getVisibilityDelay();
+ if (visibilityDelay != null) {
+ index.setVisibilityDelay(visibilityDelay);
+ }
+ index.setSearchCoverage(DomSearchCoverageBuilder.build(element));
+ index.setDispatchSpec(DomDispatchBuilder.build(element));
+ if (index.useMultilevelDispatchSetup()) {
+ // We must validate this before we add tlds and setup the dispatch groups.
+ // This must therefore happen before the regular validate() step.
+ new MultilevelDispatchValidator(index.getClusterName(), index.getDispatchSpec(), index.getSearchNodes()).validate();
+ }
+
+ // TODO: This should be cleaned up to avoid having to change code in 100 places
+ // every time we add a dispatch option.
+ TuningDispatch tuningDispatch = DomTuningDispatchBuilder.build(element);
+ Integer maxHitsPerPartition = tuningDispatch.getMaxHitsPerPartition();
+ Boolean useLocalNode = tuningDispatch.getUseLocalNode();
+
+ if (index.getTuning() == null) {
+ index.setTuning(new Tuning(index));
+ }
+ if (index.getTuning().dispatch == null) {
+ index.getTuning().dispatch = new Tuning.Dispatch();
+ }
+ if (maxHitsPerPartition != null) {
+ index.getTuning().dispatch.maxHitsPerPartition = maxHitsPerPartition;
+ }
+ if (useLocalNode != null) {
+ index.getTuning().dispatch.useLocalNode = useLocalNode;
+ }
+ index.getTuning().dispatch.minGroupCoverage = tuningDispatch.getMinGroupCoverage();
+ index.getTuning().dispatch.minActiveDocsCoverage = tuningDispatch.getMinActiveDocsCoverage();
+ index.getTuning().dispatch.policy = tuningDispatch.getDispatchPolicy();
+ }
+
+ private void setupDocumentProcessing(ContentCluster c, ModelElement e) {
+ String docprocCluster = e.getStringAttribute("cluster");
+ if (docprocCluster != null) {
+ docprocCluster = docprocCluster.trim();
+ }
+ if (c.getSearch().hasIndexedCluster()) {
+ if (docprocCluster != null && !docprocCluster.isEmpty()) {
+ c.getSearch().getIndexed().setIndexingClusterName(docprocCluster);
+ }
+ }
+
+ String docprocChain = e.getStringAttribute("chain");
+ if (docprocChain != null) {
+ docprocChain = docprocChain.trim();
+ }
+ if (c.getSearch().hasIndexedCluster()) {
+ if (docprocChain != null && !docprocChain.isEmpty()) {
+ c.getSearch().getIndexed().setIndexingChainName(docprocChain);
+ }
+ }
+ }
+
+ private void setupTuning(ContentCluster c, ModelElement tuning) {
+ ModelElement distribution = tuning.getChild("distribution");
+ if (distribution != null) {
+ String attr = distribution.getStringAttribute("type");
+ if (attr != null) {
+ if (attr.toLowerCase().equals("strict")) {
+ c.distributionMode = DistributionMode.STRICT;
+ } else if (attr.toLowerCase().equals("loose")) {
+ c.distributionMode = DistributionMode.LOOSE;
+ } else if (attr.toLowerCase().equals("legacy")) {
+ c.distributionMode = DistributionMode.LEGACY;
+ } else {
+ throw new IllegalStateException("Distribution type " + attr + " not supported.");
+ }
+ }
+ }
+ ModelElement merges = tuning.getChild("merges");
+ if (merges != null) {
+ Integer attr = merges.getIntegerAttribute("max-nodes-per-merge");
+ if (attr != null) {
+ c.maxNodesPerMerge = attr;
+ }
+ }
+ }
+
+ private void validateGroupSiblings(String cluster, StorageGroup group) {
+ HashSet<String> siblings = new HashSet<>();
+ for (StorageGroup g : group.getSubgroups()) {
+ String name = g.getName();
+ if (siblings.contains(name)) {
+ throw new IllegalArgumentException("Cluster '" + cluster + "' has multiple groups " +
+ "with name '" + name + "' in the same subgroup. Group sibling names must be unique.");
+ }
+ siblings.add(name);
+ }
+ }
+
+ private void validateThatGroupSiblingsAreUnique(String cluster, StorageGroup group) {
+ if (group == null) {
+ return; // Unit testing case
+ }
+ validateGroupSiblings(cluster, group);
+ for (StorageGroup g : group.getSubgroups()) {
+ validateThatGroupSiblingsAreUnique(cluster, g);
+ }
+ }
+
+ private void addClusterControllers(AbstractConfigProducerRoot root, StorageGroup rootGroup, ModelElement contentElement, String contentClusterName, ContentCluster contentCluster) {
+ if (admin == null) return; // only in tests
+ if (contentCluster.getPersistence() == null) return;
+
+ ContainerCluster clusterControllers;
+
+ ContentCluster overlappingCluster = findOverlappingCluster(root, contentCluster);
+ if (overlappingCluster != null && overlappingCluster.getClusterControllers() != null) {
+ // Borrow the cluster controllers of the other cluster in this case.
+ // This condition only obtains on non-hosted systems with a shared config server,
+ // a combination which only exists in system tests
+ clusterControllers = overlappingCluster.getClusterControllers();
+ }
+ else if (admin.multitenant()) {
+ String clusterName = contentClusterName + "-controllers";
+ NodesSpecification nodesSpecification =
+ NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers")).orElse(NodesSpecification.nonDedicated(3));
+ Collection<HostResource> hosts = nodesSpecification.isDedicated() ?
+ getControllerHosts(nodesSpecification, admin, clusterName) :
+ drawContentHosts(nodesSpecification.count(), rootGroup);
+
+ clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, true);
+ contentCluster.clusterControllers = clusterControllers;
+ }
+ else {
+ clusterControllers = admin.getClusterControllers();
+ if (clusterControllers == null) {
+ List<HostResource> hosts = admin.getClusterControllerHosts();
+ if (hosts.size() > 1) {
+ admin.deployLogger().log(Level.INFO, "When having content cluster(s) and more than 1 config server it is recommended to configure cluster controllers explicitly." +
+ " See " + ConfigModelUtils.createDocLink("reference/services-admin.html#cluster-controller"));
+ }
+ clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", false);
+ admin.setClusterControllers(clusterControllers);
+ }
+ }
+
+ addClusterControllerComponentsForThisCluster(clusterControllers, contentCluster);
+ }
+
+ /** Returns any other content cluster which shares nodes with this, or null if none are built */
+ private ContentCluster findOverlappingCluster(AbstractConfigProducerRoot root, ContentCluster contentCluster) {
+ for (ContentCluster otherContentCluster : root.getChildrenByTypeRecursive(ContentCluster.class)) {
+ if (otherContentCluster != contentCluster && overlaps(contentCluster, otherContentCluster))
+ return otherContentCluster;
+ }
+ return null;
+ }
+
+ private boolean overlaps(ContentCluster c1, ContentCluster c2) {
+ Set<HostResource> c1Hosts = c1.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
+ Set<HostResource> c2Hosts = c2.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet());
+ return ! Sets.intersection(c1Hosts, c2Hosts).isEmpty();
+ }
+
+ private Collection<HostResource> getControllerHosts(NodesSpecification nodesSpecification, Admin admin, String clusterName) {
+ return nodesSpecification.provision(admin.getHostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), Optional.empty(), deployLogger).keySet();
+ }
+
+ private List<HostResource> drawContentHosts(int count, StorageGroup rootGroup) {
+ List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup);
+ if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case)
+ hosts = hosts.subList(0, hosts.size()-1);
+ return hosts;
+ }
+
+ /**
+ * Draw <code>count</code> nodes from as many different content groups below this as possible.
+ * This will only achieve maximum spread in the case where the groups are balanced and never on the same
+ * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy.
+ */
+ // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes
+ // which overlaps with previously drawn nodes as this will prevent rolling upgrade
+ private List<HostResource> drawContentHostsRecursively(int count, StorageGroup group) {
+ Set<HostResource> hosts = new HashSet<>();
+ if (group.getNodes().isEmpty()) {
+ int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size());
+ for (StorageGroup subgroup : group.getSubgroups())
+ hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup));
+ }
+ else {
+ hosts.addAll(group.getNodes().stream()
+ .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry
+ .map(StorageNode::getHostResource).collect(Collectors.toList()));
+ }
+ List<HostResource> sortedHosts = new ArrayList<>(hosts);
+ Collections.sort(sortedHosts);
+ sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size()));
+ return sortedHosts;
+ }
+
+ private ContainerCluster createClusterControllers(AbstractConfigProducer parent, Collection<HostResource> hosts, String name, boolean multitenant) {
+ ContainerCluster clusterControllers = new ContainerCluster(parent, name, name);
+ List<Container> containers = new ArrayList<>();
+ // Add a cluster controller on each config server (there is always at least one).
+ if (clusterControllers.getContainers().isEmpty()) {
+ int index = 0;
+ for (HostResource host : hosts) {
+ ClusterControllerContainer clusterControllerContainer = new ClusterControllerContainer(clusterControllers, index, multitenant);
+ clusterControllerContainer.setHostResource(host);
+ clusterControllerContainer.initService();
+ clusterControllerContainer.setProp("clustertype", "admin")
+ .setProp("clustername", clusterControllers.getName())
+ .setProp("index", String.valueOf(index));
+ containers.add(clusterControllerContainer);
+ ++index;
+ }
+ }
+ clusterControllers.addContainers(containers);
+ ContainerModelBuilder.addDefaultHandler_legacyBuilder(clusterControllers);
+ return clusterControllers;
+ }
+
+ private void addClusterControllerComponentsForThisCluster(ContainerCluster clusterControllers, ContentCluster contentCluster) {
+ int index = 0;
+ for (Container container : clusterControllers.getContainers()) {
+ if ( ! hasClusterControllerComponent(container))
+ container.addComponent(new ClusterControllerComponent());
+ container.addComponent(new ClusterControllerConfigurer(contentCluster, index++, clusterControllers.getContainers().size()));
+ }
+
+ }
+
+ private boolean hasClusterControllerComponent(Container container) {
+ for (Object o : container.getComponents().getComponents())
+ if (o instanceof ClusterControllerComponent) return true;
+ return false;
+ }
+
+ }
+
+ private ContentCluster(AbstractConfigProducer parent,
+ String clusterName,
+ Map<String, NewDocumentType> documentDefinitions,
+ String routingSelection,
+ Redundancy redundancy) {
+ super(parent, clusterName);
+ this.clusterName = clusterName;
+ this.documentDefinitions = documentDefinitions;
+ this.documentSelection = routingSelection;
+ this.redundancy = redundancy;
+ }
+
+ public void prepare() {
+ search.prepare();
+
+ if (clusterControllers != null) {
+ clusterControllers.prepare();
+ }
+ }
+
+ /** Returns cluster controllers if this is multitenant, null otherwise */
+ public ContainerCluster getClusterControllers() { return clusterControllers; }
+
+ public DistributionMode getDistributionMode() {
+ if (distributionMode != null) return distributionMode;
+ return getPersistence().getDefaultDistributionMode();
+ }
+
+ public boolean isMemfilePersistence() {
+ return persistenceFactory instanceof VDSEngine.Factory;
+ }
+
+ public static String getClusterName(ModelElement clusterElem) {
+ String clusterName = clusterElem.getStringAttribute("id");
+ if (clusterName == null) {
+ clusterName = "content";
+ }
+
+ return clusterName;
+ }
+
+ public String getName() { return clusterName; }
+
+ public String getRoutingSelector() { return documentSelection; }
+
+ public DistributorCluster getDistributorNodes() { return distributorNodes; }
+
+ public StorageCluster getStorageNodes() { return storageNodes; }
+
+ public ClusterControllerConfig getClusterControllerConfig() { return clusterControllerConfig; }
+
+ public PersistenceEngine.PersistenceFactory getPersistence() { return persistenceFactory; }
+
+ /**
+ * The list of documentdefinitions declared at the cluster level.
+ * @return the set of documenttype names
+ */
+ public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; }
+
+ public final ContentSearchCluster getSearch() { return search; }
+
+ public Redundancy redundancy() { return redundancy; }
+
+ @Override
+ public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) {
+ if (!getSearch().hasIndexedCluster()) return;
+ builder.
+ defaultroute(com.yahoo.vespa.model.routing.DocumentProtocol.getDirectRouteName(getConfigId())).
+ route(new MessagetyperouteselectorpolicyConfig.Route.Builder().
+ messagetype(DocumentProtocol.MESSAGE_PUTDOCUMENT).
+ name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))).
+ route(new MessagetyperouteselectorpolicyConfig.Route.Builder().
+ messagetype(DocumentProtocol.MESSAGE_REMOVEDOCUMENT).
+ name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))).
+ route(new MessagetyperouteselectorpolicyConfig.Route.Builder().
+ messagetype(DocumentProtocol.MESSAGE_UPDATEDOCUMENT).
+ name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId())));
+ }
+
+ public com.yahoo.vespa.model.content.StorageGroup getRootGroup() {
+ return rootGroup;
+ }
+
+ @Override
+ public void getConfig(StorDistributionConfig.Builder builder) {
+ if (rootGroup != null) {
+ builder.group.addAll(rootGroup.getGroupStructureConfig());
+ }
+
+ if (redundancy != null) {
+ redundancy.getConfig(builder);
+ }
+
+ if (search.usesHierarchicDistribution()) {
+ builder.active_per_leaf_group(true);
+ }
+ }
+
+ int getNodeCount() {
+ return storageNodes.getChildren().size();
+ }
+
+ int getNodeCountPerGroup() {
+ return rootGroup != null ? getNodeCount() / rootGroup.getNumberOfLeafGroups() : getNodeCount();
+ }
+
+ @Override
+ public void getConfig(FleetcontrollerConfig.Builder builder) {
+ builder.ideal_distribution_bits(distributionBits());
+ if (getNodeCount() < 5) {
+ builder.min_storage_up_count(1);
+ builder.min_distributor_up_ratio(0);
+ builder.min_storage_up_ratio(0);
+ }
+ }
+
+ @Override
+ public void getConfig(StorDistributormanagerConfig.Builder builder) {
+ builder.minsplitcount(distributionBits());
+ if (maxNodesPerMerge != null) {
+ builder.maximum_nodes_per_merge(maxNodesPerMerge);
+ }
+ }
+
+ /**
+ * Returns the distribution bits this cluster should use.
+ * OnHosted Vespa this is hardcoded not computed from the nodes because reducing the number of nodes is a common
+ * operation while reducing the number of distribution bits can lead to consistency problems.
+ * This hardcoded value should work fine from 1-200 nodes. Those who have more will need to set this value
+ * in config and not remove it again if they reduce the node count.
+ */
+ public int distributionBits() {
+ // if (hostedVespa) return 16; TODO: Re-enable this later (Nov 2015, ref VESPA-1702)
+ return DistributionBitCalculator.getDistributionBits(getNodeCountPerGroup(), getDistributionMode());
+ }
+
+ @Override
+ public void validate() throws Exception {
+ super.validate();
+ if (search.usesHierarchicDistribution() && ! isHostedVespa()) {
+ // validate manually configured groups
+ new IndexedHierarchicDistributionValidator(search.getClusterName(), rootGroup, redundancy, search.getIndexed().getTuning().dispatch.policy).validate();
+ if (search.getIndexed().useMultilevelDispatchSetup()) {
+ throw new IllegalArgumentException("In indexed content cluster '" + search.getClusterName() + "': Using multi-level dispatch setup is not supported when using hierarchical distribution.");
+ }
+ }
+ }
+
+ public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>();
+ static {
+ METRIC_INDEX_MAP.put("status", 0);
+ METRIC_INDEX_MAP.put("log", 1);
+ METRIC_INDEX_MAP.put("yamas", 2);
+ METRIC_INDEX_MAP.put("health", 3);
+ METRIC_INDEX_MAP.put("fleetcontroller", 4);
+ METRIC_INDEX_MAP.put("statereporter", 5);
+ }
+
+ public static MetricsmanagerConfig.Consumer.Builder getMetricBuilder(String name, MetricsmanagerConfig.Builder builder) {
+ Integer index = METRIC_INDEX_MAP.get(name);
+ if (index != null) {
+ return builder.consumer.get(index);
+ }
+
+ MetricsmanagerConfig.Consumer.Builder retVal = new MetricsmanagerConfig.Consumer.Builder();
+ retVal.name(name);
+ builder.consumer(retVal);
+ return retVal;
+ }
+
+ @Override
+ public void getConfig(MetricsmanagerConfig.Builder builder) {
+ MonitoringSystem monitoringSystem = getMonitoringService();
+ if (monitoringSystem != null) {
+ builder.snapshot(new MetricsmanagerConfig.Snapshot.Builder().
+ periods(monitoringSystem.getIntervalSeconds()).periods(300));
+ }
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("status").
+ addedmetrics("*").
+ removedtags("partofsum"));
+
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("log").
+ tags("logdefault").
+ removedtags("loadtype"));
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("yamas").
+ tags("yamasdefault").
+ removedtags("loadtype"));
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("health"));
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("fleetcontroller"));
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("statereporter").
+ addedmetrics("*").
+ removedtags("thread").
+ tags("disk"));
+
+ Map<String, MetricsConsumer> consumers = getRoot().getAdmin().getUserMetricsConsumers();
+ if (consumers != null) {
+ for (Map.Entry<String, MetricsConsumer> e : consumers.entrySet()) {
+ MetricsmanagerConfig.Consumer.Builder b = getMetricBuilder(e.getKey(), builder);
+ for (Metric m : e.getValue().getMetrics().values()) {
+ b.addedmetrics(m.getName());
+ }
+ }
+ }
+
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java
new file mode 100644
index 00000000000..b145637b6c5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.document.select.DocumentSelector;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.select.rule.DocumentNode;
+import com.yahoo.vespa.model.content.DocumentTypeVisitor;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * @author thomasg
+ */
+public class DocumentSelectionBuilder {
+
+ private static class AllowedDocumentTypesChecker extends DocumentTypeVisitor {
+ String allowedType;
+
+ private AllowedDocumentTypesChecker(String allowedType) {
+ this.allowedType = allowedType;
+ }
+
+ @Override
+ public void visit(DocumentNode documentNode) {
+ if (!documentNode.getType().equals(this.allowedType)) {
+ if (this.allowedType == null) {
+ throw new IllegalArgumentException("Document type references are not allowed " +
+ "in global <documents> tag selection attribute (found reference to type '" +
+ documentNode.getType() + "')");
+ } else {
+ throw new IllegalArgumentException("Selection for document type '" +
+ this.allowedType + "' can not contain references to other " +
+ "document types (found reference to type '" + documentNode.getType() + "')");
+ }
+ }
+ }
+ }
+
+ private void validateSelectionExpression(String sel, String allowedType) {
+ DocumentSelector s;
+ try {
+ s = new DocumentSelector(sel);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse document routing selection: " + sel, e);
+ }
+ AllowedDocumentTypesChecker checker = new AllowedDocumentTypesChecker(allowedType);
+ s.visit(checker);
+ }
+
+ public String build(ModelElement elem) {
+ StringBuilder sb = new StringBuilder();
+ if (elem != null) {
+ for (ModelElement e : elem.subElements("document")) {
+ if (sb.length() > 0) {
+ sb.append(" OR ");
+ }
+ sb.append('(');
+ String type = e.getStringAttribute("type");
+ sb.append(type);
+ String selection = e.getStringAttribute("selection");
+ if (selection != null) {
+ validateSelectionExpression(selection, type);
+ sb.append(" AND (");
+ sb.append(selection);
+ sb.append(')');
+ }
+ sb.append(')');
+ }
+
+ String globalSelection = elem.getStringAttribute("selection");
+ if (globalSelection != null) {
+ validateSelectionExpression(globalSelection, null);
+ StringBuilder global = new StringBuilder();
+ global.append('(').append(globalSelection).append(") AND (")
+ .append(sb.toString()).append(')');
+ return global.toString();
+ }
+ }
+ return sb.toString();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java
new file mode 100644
index 00000000000..b43b856b38a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.vespa.model.content.ContentSearch;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class DomContentSearchBuilder {
+
+ public static ContentSearch build(ModelElement contentXml) {
+ ContentSearch.Builder builder = new ContentSearch.Builder();
+ ModelElement searchElement = contentXml.getChild("search");
+ if (searchElement == null) {
+ return builder.build();
+ }
+ builder.setQueryTimeout(searchElement.childAsDouble("query-timeout"));
+ builder.setVisibilityDelay(searchElement.childAsDouble("visibility-delay"));
+ return builder.build();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java
new file mode 100644
index 00000000000..4796c8b1382
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+
+import com.yahoo.vespa.model.content.DispatchSpec;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Builder for the dispatch setup for a content cluster.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DomDispatchBuilder {
+
+ public static DispatchSpec build(ModelElement contentXml) {
+ DispatchSpec.Builder builder = new DispatchSpec.Builder();
+ ModelElement dispatchElement = contentXml.getChild("dispatch");
+ if (dispatchElement == null) {
+ return builder.build();
+ }
+ builder.setNumDispatchGroups(dispatchElement.childAsInteger("num-dispatch-groups"));
+
+ List<ModelElement> groupsElement = dispatchElement.subElements("group");
+ if (groupsElement != null) {
+ builder.setGroups(buildGroups(groupsElement));
+ }
+ return builder.build();
+ }
+
+ private static List<DispatchSpec.Group> buildGroups(List<ModelElement> groupsElement) {
+ List<DispatchSpec.Group> groups = new ArrayList<>();
+ for (ModelElement groupElement : groupsElement) {
+ groups.add(buildGroup(groupElement));
+ }
+ return groups;
+ }
+
+ private static DispatchSpec.Group buildGroup(ModelElement groupElement) {
+ List<ModelElement> nodes = groupElement.subElements("node");
+ DispatchSpec.Group group = new DispatchSpec.Group();
+ for (ModelElement nodeElement : nodes) {
+ group.addNode(buildNode(nodeElement));
+ }
+ return group;
+ }
+
+ private static DispatchSpec.Node buildNode(ModelElement nodeElement) {
+ return new DispatchSpec.Node(nodeElement.getIntegerAttribute("distribution-key"));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java
new file mode 100644
index 00000000000..8c83957eb26
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.ResourceLimits;
+
+/**
+ * Builder for resource limits for a content cluster with engine proton.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DomResourceLimitsBuilder {
+
+ public static ResourceLimits build(ModelElement contentXml) {
+ ResourceLimits.Builder builder = new ResourceLimits.Builder();
+ ModelElement resourceLimits = contentXml.getChild("resource-limits");
+ if (resourceLimits == null) {
+ return builder.build();
+ }
+ builder.setDiskLimit(resourceLimits.childAsDouble("disk"));
+ builder.setMemoryLimit(resourceLimits.childAsDouble("memory"));
+ return builder.build();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java
new file mode 100644
index 00000000000..83602c3003c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.vespa.model.content.SearchCoverage;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class DomSearchCoverageBuilder {
+
+ public static SearchCoverage build(ModelElement contentXml) {
+ SearchCoverage.Builder builder = new SearchCoverage.Builder();
+ ModelElement searchElement = contentXml.getChild("search");
+ if (searchElement == null) {
+ return builder.build();
+ }
+ ModelElement coverageElement = searchElement.getChild("coverage");
+ if (coverageElement == null) {
+ return builder.build();
+ }
+ builder.setMinimum(coverageElement.childAsDouble("minimum"));
+ builder.setMinWaitAfterCoverageFactor(coverageElement.childAsDouble("min-wait-after-coverage-factor"));
+ builder.setMaxWaitAfterCoverageFactor(coverageElement.childAsDouble("max-wait-after-coverage-factor"));
+ return builder.build();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
new file mode 100644
index 00000000000..1d2bf938758
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.vespa.model.content.TuningDispatch;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class DomTuningDispatchBuilder {
+
+ public static TuningDispatch build(ModelElement contentXml) {
+ TuningDispatch.Builder builder = new TuningDispatch.Builder();
+ ModelElement tuningElement = contentXml.getChild("tuning");
+ if (tuningElement == null) {
+ return builder.build();
+ }
+ ModelElement dispatchElement = tuningElement.getChild("dispatch");
+ if (dispatchElement == null) {
+ return builder.build();
+ }
+ builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition"));
+ builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy"));
+ builder.setUseLocalNode(dispatchElement.childAsBoolean("use-local-node"));
+ builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage"));
+ builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage"));
+
+ return builder.build();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java
new file mode 100644
index 00000000000..374f970739d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.engines.*;
+
+/**
+ * Creates the correct engine factory from XML.
+ */
+public class EngineFactoryBuilder {
+ public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) {
+ ModelElement persistence = clusterElem.getChild("engine");
+ if (persistence != null) {
+ if (c.getSearch().hasIndexedCluster() && persistence.getChild("proton") == null) {
+ throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use <proton> as your engine.");
+ }
+
+ ModelElement e;
+ if ((e = persistence.getChild("vds")) != null) {
+ return new VDSEngine.Factory(e);
+ } else if (persistence.getChild("proton") != null) {
+ return new ProtonEngine.Factory(c.getSearch());
+ } else if (persistence.getChild("dummy") != null) {
+ return new com.yahoo.vespa.model.content.engines.DummyPersistence.Factory();
+ } else if (persistence.getChild("rpc") != null) {
+ return new RPCEngine.Factory();
+ }
+ }
+
+ return new ProtonEngine.Factory(c.getSearch());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
new file mode 100644
index 00000000000..d80f7029320
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.Redundancy;
+
+/**
+ * Builds redundancy config for a content cluster.
+ */
+public class RedundancyBuilder {
+ Redundancy build(ModelElement clusterXml) {
+ Integer initialRedundancy = 2;
+ Integer finalRedundancy = 3;
+ Integer readyCopies = 2;
+
+ ModelElement redundancyElement = clusterXml.getChild("redundancy");
+ if (redundancyElement != null) {
+ initialRedundancy = redundancyElement.getIntegerAttribute("reply-after");
+ finalRedundancy = (int)redundancyElement.asLong();
+
+ if (initialRedundancy == null) {
+ initialRedundancy = finalRedundancy;
+ } else {
+ if (finalRedundancy < initialRedundancy) {
+ throw new IllegalArgumentException("Final redundancy must be higher than or equal to initial redundancy");
+ }
+ }
+
+ readyCopies = clusterXml.childAsInteger("engine.proton.searchable-copies");
+ if (readyCopies == null) {
+ readyCopies = Math.min(finalRedundancy, 2);
+ }
+ if (readyCopies > finalRedundancy) {
+ throw new IllegalArgumentException("Number of searchable copies can not be higher than final redundancy");
+ }
+ }
+
+ return new Redundancy(initialRedundancy, finalRedundancy, readyCopies);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java
new file mode 100644
index 00000000000..3f43dbe4491
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.documentmodel.DocumentTypeRepo;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+* Created with IntelliJ IDEA.
+* User: thomasg
+* Date: 9/28/12
+* Time: 1:20 PM
+* To change this template use File | Settings | File Templates.
+*/
+public class SearchDefinitionBuilder {
+ public Map<String, NewDocumentType> build(DocumentTypeRepo repo, ModelElement elem) {
+ Map<String, NewDocumentType> docTypes = new TreeMap<>();
+
+ if (elem != null) {
+ for (ModelElement e : elem.subElements("document")) {
+ String name = e.getStringAttribute("type"); // Schema-guaranteed presence
+ NewDocumentType documentType = repo.getDocumentType(name);
+ if (documentType != null) {
+ docTypes.put(documentType.getName(), documentType);
+ } else {
+ throw new RuntimeException("Document type '" + name + "' not found in application package");
+ }
+ }
+ }
+
+ return docTypes;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java
new file mode 100644
index 00000000000..47504d5d0cd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.content.cluster;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java
new file mode 100644
index 00000000000..b52b3e0d248
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.StorageGroup;
+import com.yahoo.vespa.model.content.StorageNode;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+public class DummyPersistence extends PersistenceEngine {
+ public DummyPersistence(StorageNode parent) {
+ super(parent, "provider");
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ builder.persistence_provider(new StorServerConfig.Persistence_provider.Builder().type(StorServerConfig.Persistence_provider.Type.Enum.DUMMY));
+ }
+
+ public static class Factory implements PersistenceFactory {
+
+ @Override
+ public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) {
+ return new DummyPersistence(storageNode);
+ }
+
+ @Override
+ public boolean supportRevert() {
+ return true;
+ }
+
+ @Override
+ public boolean enableMultiLevelSplitting() {
+ return true;
+ }
+
+ @Override
+ public ContentCluster.DistributionMode getDefaultDistributionMode() {
+ return ContentCluster.DistributionMode.LOOSE;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java
new file mode 100644
index 00000000000..29ed02ab2b6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.StorageGroup;
+import com.yahoo.vespa.model.content.StorageNode;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+public abstract class PersistenceEngine extends AbstractConfigProducer implements StorServerConfig.Producer {
+
+ public PersistenceEngine(AbstractConfigProducer parent, String name) {
+ super(parent, name);
+ }
+
+ /**
+ * Creates a config producer for the engines provider at a given node.
+ */
+ public static interface PersistenceFactory {
+ public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement);
+
+ /**
+ * If a write request succeeds on some nodes and fails on others, causing request to
+ * fail to client, the content layer will revert the operation where it succeeded if
+ * reverts are supported. (Typically require backend to keep multiple entries of the
+ * same document identifier persisted at the same time)
+ */
+ public boolean supportRevert();
+
+ /**
+ * Multi level splitting can increase split performance a lot where documents have been
+ * co-localized, for backends where retrieving document identifiers contained in bucket
+ * is cheap. Backends where split is cheaper than fetching document identifiers will
+ * not want to enable multi level splitting.
+ */
+ public boolean enableMultiLevelSplitting();
+
+ public ContentCluster.DistributionMode getDefaultDistributionMode();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java
new file mode 100644
index 00000000000..595da4da2a0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.ContentSearchCluster;
+import com.yahoo.vespa.model.content.StorageGroup;
+import com.yahoo.vespa.model.content.StorageNode;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.search.SearchNode;
+
+/**
+ * Initializes the engines engine on each storage node. May include creating other
+ * nodes.
+ */
+public class ProtonEngine {
+ public static class Factory implements PersistenceEngine.PersistenceFactory {
+ ContentSearchCluster search;
+
+ public Factory(ContentSearchCluster search) {
+ this.search = search;
+ }
+
+ @Override
+ public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) {
+ SearchNode searchNode = search.addSearchNode(storageNode, parentGroup, storageNodeElement);
+ return new ProtonProvider(storageNode, searchNode);
+ }
+
+ @Override
+ public boolean supportRevert() {
+ return false;
+ }
+
+ @Override
+ public boolean enableMultiLevelSplitting() {
+ return false;
+ }
+
+ @Override
+ public ContentCluster.DistributionMode getDefaultDistributionMode() {
+ return ContentCluster.DistributionMode.LOOSE;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java
new file mode 100644
index 00000000000..d665df3d53c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.vespa.model.content.StorageNode;
+import com.yahoo.vespa.model.search.SearchNode;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: balder
+ * Date: 20.11.12
+ * Time: 20:04
+ * To change this template use File | Settings | File Templates.
+ */
+public class ProtonProvider extends RPCEngine {
+ public ProtonProvider(StorageNode parent, SearchNode searchNode) {
+ super(parent, searchNode);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java
new file mode 100644
index 00000000000..ec0d7dc35f4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.StorageGroup;
+import com.yahoo.vespa.model.content.StorageNode;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.search.SearchNode;
+
+public class RPCEngine extends PersistenceEngine {
+
+ private SearchNode searchNode;
+ public RPCEngine(StorageNode parent) {
+ super(parent, "provider");
+ }
+
+ public RPCEngine(StorageNode parent, SearchNode searchNode) {
+ super(parent, "provider");
+ this.searchNode = searchNode;
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ StorServerConfig.Persistence_provider.Builder provider =
+ new StorServerConfig.Persistence_provider.Builder();
+ provider.type(StorServerConfig.Persistence_provider.Type.Enum.RPC);
+
+ if (searchNode != null) {
+ provider.rpc(new StorServerConfig.Persistence_provider.Rpc.Builder().connectspec("tcp/localhost:" + searchNode.getPersistenceProviderRpcPort()));
+ }
+
+ builder.persistence_provider(provider);
+ }
+
+ public static class Factory implements PersistenceFactory {
+ @Override
+ public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) {
+ return new RPCEngine(storageNode);
+ }
+
+ @Override
+ public boolean supportRevert() {
+ return false;
+ }
+
+ @Override
+ public boolean enableMultiLevelSplitting() {
+ return false;
+ }
+
+ @Override
+ public ContentCluster.DistributionMode getDefaultDistributionMode() {
+ return ContentCluster.DistributionMode.LOOSE;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java
new file mode 100644
index 00000000000..5e407938159
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig;
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.StorageGroup;
+import com.yahoo.vespa.model.content.StorageNode;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+/**
+ * Configuration class to generate config for the memfile engines provider.
+ */
+public class VDSEngine extends PersistenceEngine
+ implements StorMemfilepersistenceConfig.Producer
+{
+ ModelElement tuning;
+
+ public VDSEngine(StorageNode parent, ModelElement vdsConfig) {
+ super(parent, "provider");
+
+ if (vdsConfig != null) {
+ this.tuning = vdsConfig.getChild("tuning");
+ }
+
+ if (parent != null) {
+ parent.useVdsEngine();
+ }
+ }
+
+ @Override
+ public void getConfig(StorMemfilepersistenceConfig.Builder builder) {
+ if (tuning == null) {
+ return;
+ }
+
+ ModelElement diskFullRatio = tuning.getChild("disk-full-ratio");
+ if (diskFullRatio != null) {
+ builder.disk_full_factor(diskFullRatio.asDouble());
+ }
+
+ ModelElement cacheSize = tuning.getChild("cache-size");
+ if (cacheSize != null) {
+ builder.cache_size(cacheSize.asLong());
+ }
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ builder.persistence_provider(
+ new StorServerConfig.Persistence_provider.Builder().type(
+ StorServerConfig.Persistence_provider.Type.Enum.STORAGE)
+ );
+ }
+
+ public static class Factory implements PersistenceFactory {
+ ModelElement vdsConfig;
+
+ public Factory(ModelElement vdsConfig) {
+ this.vdsConfig = vdsConfig;
+ }
+
+ @Override
+ public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) {
+ return new VDSEngine(storageNode, vdsConfig);
+ }
+
+ @Override
+ public boolean supportRevert() {
+ return true;
+ }
+
+ @Override
+ public boolean enableMultiLevelSplitting() {
+ return true;
+ }
+
+ @Override
+ public ContentCluster.DistributionMode getDefaultDistributionMode() {
+ return ContentCluster.DistributionMode.STRICT;
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java
new file mode 100644
index 00000000000..28652c82996
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.content.engines;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java
new file mode 100644
index 00000000000..a0af4d2a5fd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.content;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
new file mode 100644
index 00000000000..6a5f14899ee
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.vespa.config.content.StorFilestorConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.PriorityMapping;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Serves stor-filestor for storage clusters.
+ */
+public class FileStorProducer implements StorFilestorConfig.Producer {
+ public static class Builder {
+ protected FileStorProducer build(ContentCluster parent, ModelElement clusterElem) {
+ return new FileStorProducer(parent, getThreads(clusterElem));
+ }
+
+ private List<StorFilestorConfig.Threads.Builder> getThreads(ModelElement clusterElem) {
+ ModelElement tuning = clusterElem.getChild("tuning");
+ if (tuning == null) {
+ return null;
+ }
+ ModelElement threads = tuning.getChild("persistence-threads");
+ if (threads == null) {
+ return null;
+ }
+
+ List<StorFilestorConfig.Threads.Builder> retVal = new ArrayList<>();
+
+ PriorityMapping mapping = new PriorityMapping(clusterElem);
+
+ for (ModelElement thread : threads.subElements("thread")) {
+ String priorityName = thread.getStringAttribute("lowest-priority");
+ if (priorityName == null) {
+ priorityName = "LOWEST";
+ }
+
+ Integer count = thread.getIntegerAttribute("count");
+ if (count == null) {
+ count = 1;
+ }
+
+ for (int i = 0; i < count; ++i) {
+ retVal.add(new StorFilestorConfig.Threads.Builder().lowestpri(mapping.getPriorityMapping(priorityName)));
+ }
+ }
+
+ return retVal;
+ }
+ }
+
+ private List<StorFilestorConfig.Threads.Builder> threads;
+ private ContentCluster cluster;
+
+ public FileStorProducer(ContentCluster parent, List<StorFilestorConfig.Threads.Builder> threads) {
+ this.threads = threads;
+ this.cluster = parent;
+ }
+
+ @Override
+ public void getConfig(StorFilestorConfig.Builder builder) {
+ if (threads != null) {
+ for (StorFilestorConfig.Threads.Builder t : threads) {
+ builder.threads.add(t);
+ }
+ }
+ builder.enable_multibit_split_optimalization(cluster.getPersistence().enableMultiLevelSplitting());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java
new file mode 100644
index 00000000000..69bf29ca307
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig;
+import com.yahoo.config.model.ConfigModelUtils;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * Serves stor-integritychecker config for storage clusters.
+ */
+public class IntegrityCheckerProducer implements StorIntegritycheckerConfig.Producer {
+
+ public static class Builder {
+ protected IntegrityCheckerProducer build(ModelElement clusterElem) {
+ ModelElement tuning = clusterElem.getChild("tuning");
+
+ if (tuning == null) {
+ return new IntegrityCheckerProducer();
+ }
+
+ ModelElement maintenance = tuning.getChild("maintenance");
+ if (maintenance == null) {
+ return new IntegrityCheckerProducer();
+ }
+
+ Integer startTime = null;
+ Integer stopTime = null;
+ String weeklyCycle = null;
+
+ String start = maintenance.getStringAttribute("start");
+ if (start != null) {
+ startTime = ConfigModelUtils.getTimeOfDay(start);
+ }
+
+ String stop = maintenance.getStringAttribute("stop");
+ if (stop != null) {
+ stopTime = ConfigModelUtils.getTimeOfDay(stop);
+ }
+
+ String high = maintenance.getStringAttribute("high");
+
+ if (high != null) {
+ int weekday = ConfigModelUtils.getDayOfWeek(high);
+ char[] weeklycycle = "rrrrrrr".toCharArray();
+ weeklycycle[weekday] = 'R';
+ weeklyCycle = String.valueOf(weeklycycle);
+ }
+
+ return new IntegrityCheckerProducer(startTime, stopTime, weeklyCycle);
+ }
+ }
+
+ private Integer startTime;
+ private Integer stopTime;
+ private String weeklyCycle;
+
+ IntegrityCheckerProducer() {
+ }
+
+ IntegrityCheckerProducer(Integer startTime, Integer stopTime, String weeklyCycle) {
+ this.startTime = startTime;
+ this.stopTime = stopTime;
+ this.weeklyCycle = weeklyCycle;
+ }
+
+ @Override
+ public void getConfig(StorIntegritycheckerConfig.Builder builder) {
+ if (startTime != null) {
+ builder.dailycyclestart(startTime);
+ }
+
+ if (stopTime != null) {
+ builder.dailycyclestop(stopTime);
+ }
+
+ if (weeklyCycle != null) {
+ builder.weeklycycle(weeklyCycle);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java
new file mode 100644
index 00000000000..d632329192b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.vespa.config.content.PersistenceConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.utils.Duration;
+
+/**
+ * Serves engines config for storage clusters.
+ */
+public class PersistenceProducer implements PersistenceConfig.Producer {
+
+ public static class Builder {
+ public PersistenceProducer build(ModelElement element) {
+ ModelElement persistence = element.getChild("engine");
+ if (persistence == null) {
+ return new PersistenceProducer();
+ }
+
+ return new PersistenceProducer(
+ persistence.childAsBoolean("fail-partition-on-error"),
+ persistence.childAsDuration("recovery-time"),
+ persistence.childAsDuration("revert-time"));
+ }
+ }
+
+ Boolean failOnError;
+ Duration recoveryPeriod;
+ Duration revertTimePeriod;
+
+ public PersistenceProducer() {}
+
+ public PersistenceProducer(Boolean failOnError, Duration recoveryPeriod, Duration revertTimePeriod) {
+ this.failOnError = failOnError;
+ this.recoveryPeriod = recoveryPeriod;
+ this.revertTimePeriod = revertTimePeriod;
+ }
+
+ @Override
+ public void getConfig(PersistenceConfig.Builder builder) {
+ if (failOnError != null) {
+ builder.fail_partition_on_error(failOnError);
+ }
+ if (recoveryPeriod != null) {
+ builder.keep_remove_time_period((int)recoveryPeriod.getSeconds());
+ }
+ if (revertTimePeriod != null) {
+ builder.revert_time_period((int)revertTimePeriod.getSeconds());
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
new file mode 100644
index 00000000000..e6f767e881c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * Serves config for stor-server for storage clusters (clusters of storage nodes).
+ */
+public class StorServerProducer implements StorServerConfig.Producer {
+ public static class Builder {
+ StorServerProducer build(ModelElement element) {
+ ModelElement tuning = element.getChild("tuning");
+
+ if (tuning == null) {
+ return new StorServerProducer(ContentCluster.getClusterName(element), null, null);
+ }
+
+ ModelElement merges = tuning.getChild("merges");
+ if (merges == null) {
+ return new StorServerProducer(ContentCluster.getClusterName(element), null, null);
+ }
+
+ return new StorServerProducer(ContentCluster.getClusterName(element),
+ merges.getIntegerAttribute("max-per-node"),
+ merges.getIntegerAttribute("max-queue-size"));
+ }
+ }
+
+ private String clusterName;
+ private Integer maxMergesPerNode;
+ private Integer queueSize;
+
+ public StorServerProducer(String clusterName, Integer maxMergesPerNode, Integer queueSize) {
+ this.clusterName = clusterName;
+ this.maxMergesPerNode = maxMergesPerNode;
+ this.queueSize = queueSize;
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ builder.root_folder("");
+ builder.is_distributor(false);
+
+ if (clusterName != null) {
+ builder.cluster_name(clusterName);
+ }
+ if (maxMergesPerNode != null) {
+ builder.max_merges_per_node(maxMergesPerNode);
+ }
+ if (queueSize != null) {
+ builder.max_merge_queue_size(queueSize);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java
new file mode 100644
index 00000000000..b04ef0d15a4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.vespa.config.content.core.StorVisitorConfig;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+
+/**
+ * Serves stor-visitor config for storage clusters.
+ */
+public class StorVisitorProducer implements StorVisitorConfig.Producer {
+ public static class Builder {
+ public StorVisitorProducer build(ModelElement element) {
+ ModelElement tuning = element.getChild("tuning");
+ if (tuning == null) {
+ return new StorVisitorProducer();
+ }
+
+ ModelElement visitors = tuning.getChild("visitors");
+ if (visitors == null) {
+ return new StorVisitorProducer();
+ }
+
+ return new StorVisitorProducer(visitors.getIntegerAttribute("thread-count"),
+ visitors.getIntegerAttribute("max-queue-size"),
+ visitors.childAsInteger("max-concurrent.fixed"),
+ visitors.childAsInteger("max-concurrent.variable"));
+ }
+ }
+
+ Integer threadCount;
+ Integer maxQueueSize;
+ Integer maxConcurrentFixed;
+ Integer maxConcurrentVariable;
+
+ public StorVisitorProducer() {}
+
+ StorVisitorProducer(Integer threadCount, Integer maxQueueSize, Integer maxConcurrentFixed, Integer maxConcurrentVariable) {
+ this.threadCount = threadCount;
+ this.maxQueueSize = maxQueueSize;
+ this.maxConcurrentFixed = maxConcurrentFixed;
+ this.maxConcurrentVariable = maxConcurrentVariable;
+ }
+
+ @Override
+ public void getConfig(StorVisitorConfig.Builder builder) {
+ if (threadCount != null) {
+ builder.visitorthreads(threadCount);
+ }
+ if (maxQueueSize != null) {
+ builder.maxvisitorqueuesize(maxQueueSize);
+ }
+ if (maxConcurrentFixed != null) {
+ builder.maxconcurrentvisitors_fixed(maxConcurrentFixed);
+ }
+ if (maxConcurrentVariable != null) {
+ builder.maxconcurrentvisitors_variable(maxConcurrentVariable);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java
new file mode 100644
index 00000000000..6c9cbadd21a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig;
+import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig;
+import com.yahoo.vespa.config.content.core.StorBucketmoverConfig;
+import com.yahoo.vespa.config.content.core.StorVisitorConfig;
+import com.yahoo.vespa.config.content.StorFilestorConfig;
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.config.content.PersistenceConfig;
+import com.yahoo.metrics.MetricsmanagerConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import com.yahoo.vespa.model.content.StorageNode;
+import org.w3c.dom.Element;
+
+/**
+ * Represents configuration that is common to all storage nodes.
+ */
+public class StorageCluster extends AbstractConfigProducer<StorageNode>
+ implements StorServerConfig.Producer,
+ StorBucketmoverConfig.Producer,
+ StorMemfilepersistenceConfig.Producer,
+ StorIntegritycheckerConfig.Producer,
+ StorFilestorConfig.Producer,
+ StorVisitorConfig.Producer,
+ PersistenceConfig.Producer,
+ MetricsmanagerConfig.Producer
+{
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageCluster> {
+ @Override
+ protected StorageCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ ModelElement clusterElem = new ModelElement(producerSpec);
+
+ return new StorageCluster(ancestor,
+ ContentCluster.getClusterName(clusterElem),
+ new FileStorProducer.Builder().build(((ContentCluster)ancestor), clusterElem),
+ new IntegrityCheckerProducer.Builder().build(clusterElem),
+ new StorServerProducer.Builder().build(clusterElem),
+ new StorVisitorProducer.Builder().build(clusterElem),
+ new PersistenceProducer.Builder().build(clusterElem));
+ }
+ }
+
+ private Integer bucketMoverMaxFillAboveAverage = null;
+ private Long cacheSize = null;
+ private Double diskFullPercentage = null;
+
+ private String clusterName;
+ private FileStorProducer fileStorProducer;
+ private IntegrityCheckerProducer integrityCheckerProducer;
+ private StorServerProducer storServerProducer;
+ private StorVisitorProducer storVisitorProducer;
+ private PersistenceProducer persistenceProducer;
+
+ StorageCluster(AbstractConfigProducer parent,
+ String clusterName,
+ FileStorProducer fileStorProducer,
+ IntegrityCheckerProducer integrityCheckerProducer,
+ StorServerProducer storServerProducer,
+ StorVisitorProducer storVisitorProducer,
+ PersistenceProducer persistenceProducer) {
+ super(parent, "storage");
+ this.clusterName = clusterName;
+ this.fileStorProducer = fileStorProducer;
+ this.integrityCheckerProducer = integrityCheckerProducer;
+ this.storServerProducer = storServerProducer;
+ this.storVisitorProducer = storVisitorProducer;
+ this.persistenceProducer = persistenceProducer;
+ }
+
+ @Override
+ public void getConfig(StorBucketmoverConfig.Builder builder) {
+ if (bucketMoverMaxFillAboveAverage != null) {
+ builder.max_target_fill_rate_above_average(bucketMoverMaxFillAboveAverage);
+ }
+ }
+
+ @Override
+ public void getConfig(StorMemfilepersistenceConfig.Builder builder) {
+ if (cacheSize != null) {
+ builder.cache_size(cacheSize);
+ }
+
+ if (diskFullPercentage != null) {
+ builder.disk_full_factor(diskFullPercentage / 100.0);
+ builder.disk_full_factor_move(diskFullPercentage / 100.0 * 0.9);
+ }
+ }
+
+ @Override
+ public void getConfig(MetricsmanagerConfig.Builder builder) {
+ ContentCluster.getMetricBuilder("fleetcontroller", builder).
+ addedmetrics("vds.filestor.*.allthreads.put.sum").
+ addedmetrics("vds.filestor.*.allthreads.get.sum").
+ addedmetrics("vds.filestor.*.allthreads.multi.sum").
+ addedmetrics("vds.filestor.*.allthreads.update.sum").
+ addedmetrics("vds.filestor.*.allthreads.remove.sum").
+ addedmetrics("vds.filestor.*.allthreads.operations").
+ addedmetrics("vds.datastored.alldisks.docs").
+ addedmetrics("vds.datastored.alldisks.bytes").
+ addedmetrics("vds.datastored.alldisks.buckets");
+
+ ContentCluster.getMetricBuilder("log", builder).
+ addedmetrics("vds.filestor.alldisks.allthreads.put.sum").
+ addedmetrics("vds.filestor.alldisks.allthreads.get.sum").
+ addedmetrics("vds.filestor.alldisks.allthreads.remove.sum").
+ addedmetrics("vds.filestor.alldisks.allthreads.update.sum").
+ addedmetrics("vds.datastored.alldisks.docs").
+ addedmetrics("vds.datastored.alldisks.bytes").
+ addedmetrics("vds.filestor.alldisks.queuesize").
+ addedmetrics("vds.filestor.alldisks.averagequeuewait.sum").
+ addedmetrics("vds.visitor.cv_queuewaittime").
+ addedmetrics("vds.visitor.allthreads.averagequeuewait").
+ addedmetrics("vds.visitor.allthreads.averagevisitorlifetime").
+ addedmetrics("vds.visitor.allthreads.created.sum");
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ @Override
+ public void getConfig(StorIntegritycheckerConfig.Builder builder) {
+ integrityCheckerProducer.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(StorServerConfig.Builder builder) {
+ storServerProducer.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(StorVisitorConfig.Builder builder) {
+ storVisitorProducer.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(PersistenceConfig.Builder builder) {
+ persistenceProducer.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(StorFilestorConfig.Builder builder) {
+ fileStorProducer.getConfig(builder);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java
new file mode 100644
index 00000000000..6b3ca6a06b7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.content.storagecluster;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
new file mode 100644
index 00000000000..c3fcdddb273
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.filedistribution;
+
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.Host;
+import com.yahoo.vespa.model.admin.FileDistributionOptions;
+
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class FileDistributionConfigProducer extends AbstractConfigProducer {
+ private final Map<Host, FileDistributorService> fileDistributorServices = new IdentityHashMap<>();
+ private final FileDistributor fileDistributor;
+ private final FileDistributionOptions options;
+
+ private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor, FileDistributionOptions options) {
+ super(parent, "filedistribution");
+ this.fileDistributor = fileDistributor;
+ this.options = options;
+ }
+
+ public String getFileDistributionServiceConfigId(Host host) {
+ FileDistributorService service = fileDistributorServices.get(host);
+ if (service == null) {
+ throw new IllegalStateException("No file distribution service for host " + host);
+ }
+ return service.getConfigId();
+ }
+
+ public FileDistributor getFileDistributor() {
+ return fileDistributor;
+ }
+
+ public FileDistributionOptions getOptions() {
+ return options;
+ }
+
+ public void addFileDistributionService(Host host, FileDistributorService fds) {
+ fileDistributorServices.put(host, fds);
+ }
+
+ public static class Builder {
+
+ private final FileDistributionOptions options;
+
+ public Builder(FileDistributionOptions fileDistributionOptions) {
+ this.options = fileDistributionOptions;
+ }
+
+ public FileDistributionConfigProducer build(AbstractConfigProducer ancestor, FileRegistry fileRegistry) {
+ FileDistributor fileDistributor = new FileDistributor(fileRegistry);
+ return new FileDistributionConfigProducer(ancestor, fileDistributor, options);
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
new file mode 100644
index 00000000000..4dc24618a61
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.filedistribution;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.vespa.model.Host;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
+
+
+/**
+ * Responsible for directing distribution of files to hosts.
+ *
+ * @author tonytv
+ */
+public class FileDistributor {
+
+ private final FileRegistry fileRegistry;
+
+ /** A map from files to the hosts to which that file should be distributed */
+ private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>();
+
+ /**
+ * Adds the given file to the associated application packages' registry of file and marks the file
+ * for distribution to the given hosts.
+ * <b>Note: This class receives ownership of the given collection.</b>
+ *
+ * @return the reference to the file, created by the application package
+ */
+ public FileReference sendFileToHosts(String relativePath, Collection<Host> hosts) {
+ FileReference reference = fileRegistry.addFile(relativePath);
+ addToFilesToDistribute(reference, hosts);
+
+ return reference;
+ }
+
+ /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */
+ public FileReference sendFileToHost(String relativePath, Host host) {
+ return sendFileToHosts(relativePath, Arrays.asList(host));
+ }
+
+ private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) {
+ Set<Host> oldHosts = getHosts(reference);
+ oldHosts.addAll(hosts);
+ }
+
+ private Set<Host> getHosts(FileReference reference) {
+ Set<Host> hosts = filesToHosts.get(reference);
+ if (hosts == null) {
+ hosts = new HashSet<>();
+ filesToHosts.put(reference, hosts);
+ }
+ return hosts;
+ }
+
+ public FileDistributor(FileRegistry fileRegistry) {
+ this.fileRegistry = fileRegistry;
+ }
+
+ /** Returns the files which has been marked for distribution to the given host */
+ public Set<FileReference> filesToSendToHost(Host host) {
+ Set<FileReference> files = new HashSet<>();
+
+ for (Map.Entry<FileReference,Set<Host>> e : filesToHosts.entrySet()) {
+ if (e.getValue().contains(host)) {
+ files.add(e.getKey());
+ }
+ }
+ return files;
+ }
+
+ public Set<Host> getTargetHosts() {
+ Set<Host> hosts = new HashSet<>();
+ for (Set<Host> hostSubset: filesToHosts.values())
+ hosts.addAll(hostSubset);
+ return hosts;
+ }
+
+ public Set<String> getTargetHostnames() {
+ return getTargetHosts().stream().map(Host::getHostName).collect(Collectors.toSet());
+ }
+
+ /** Returns the host which is the source of the files */
+ public String fileSourceHost() {
+ return fileRegistry.fileSourceHost();
+ }
+
+ public Set<FileReference> allFilesToSend() {
+ return filesToHosts.keySet();
+ }
+
+ // should only be called during deploy
+ public void sendDeployedFiles(FileDistribution dbHandler) {
+ String fileSourceHost = fileSourceHost();
+ for (Host host : getTargetHosts()) {
+ if ( ! host.getHostName().equals(fileSourceHost)) {
+ dbHandler.sendDeployedFiles(host.getHostName(), filesToSendToHost(host));
+ }
+ }
+ dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend());
+ dbHandler.limitSendingOfDeployedFilesTo(union(getTargetHostnames(), fileSourceHost));
+ dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames());
+ }
+
+ // should only be called during deploy, and only once, since it leads to file distributor
+ // rescanning all files, which is very expensive ATM (April 2016)
+ public void reloadDeployFileDistributor(FileDistribution dbHandler) {
+ dbHandler.reloadDeployFileDistributor();
+ }
+
+ private Set<String> union(Set<String> hosts, String... additionalHosts) {
+ Set<String> result = new HashSet<>(hosts);
+ result.addAll(asList(additionalHosts));
+ return result;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java
new file mode 100644
index 00000000000..bfa32c003e0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.filedistribution;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.cloud.config.filedistribution.*;
+import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.admin.FileDistributionOptions;
+
+import java.io.File;
+import java.util.Collection;
+
+/**
+ * @author tonytv
+ */
+public class FileDistributorService extends AbstractService implements
+ FiledistributorConfig.Producer,
+ FiledistributorrpcConfig.Producer,
+ FilereferencesConfig.Producer {
+ private final static int BASEPORT = 19092;
+
+ private final FileDistributor fileDistributor;
+ private final FileDistributionOptions fileDistributionOptions;
+ private final boolean sendAllFiles;
+
+ private Collection<FileReference> getFileReferences() {
+ if (sendAllFiles) {
+ return fileDistributor.allFilesToSend();
+ } else {
+ return fileDistributor.filesToSendToHost(getHost());
+ }
+ }
+
+ public FileDistributorService(AbstractConfigProducer parent,
+ String name,
+ FileDistributor fileDistributor,
+ FileDistributionOptions fileDistributionOptions,
+ boolean sendAllFiles) {
+ super(parent, name);
+ portsMeta.on(0).tag("rpc");
+ portsMeta.on(1).tag("torrent");
+ portsMeta.on(2).tag("http").tag("state");
+ setProp("clustertype", "filedistribution");
+ setProp("clustername", "admin");
+
+ this.fileDistributor = fileDistributor;
+ this.fileDistributionOptions = fileDistributionOptions;
+ this.sendAllFiles = sendAllFiles;
+ monitorService();
+ }
+
+ @Override
+ public String getStartupCommand() {
+ return "exec $ROOT/sbin/filedistributor"
+ + " --configid " + getConfigId();
+ }
+
+ @Override
+ public boolean getAutostartFlag() {
+ return true;
+ }
+
+ @Override
+ public boolean getAutorestartFlag() {
+ return true;
+ }
+
+ public int getPortCount() {
+ return 3;
+ }
+
+ @Override
+ public int getWantedPort() {
+ return BASEPORT;
+ }
+
+ @Override
+ public void getConfig(FiledistributorConfig.Builder builder) {
+ fileDistributionOptions.getConfig(builder);
+ builder.torrentport(getRelativePort(1));
+ builder.stateport(getRelativePort(2));
+ builder.hostname(getHostName());
+ builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString());
+ }
+
+ @Override
+ public void getConfig(FiledistributorrpcConfig.Builder builder) {
+ builder.connectionspec("tcp/" + getHostName() + ":" + getRelativePort(0));
+ }
+
+ @Override
+ public void getConfig(FilereferencesConfig.Builder builder) {
+ for (FileReference reference : getFileReferences()) {
+ builder.filereferences(reference.value());
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java
new file mode 100644
index 00000000000..bc53f9222de
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.filedistribution;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java
new file mode 100644
index 00000000000..7bbe9d14bc5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic;
+
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class GenericServicesBuilder extends ConfigModelBuilder<GenericServicesModel> {
+
+ public GenericServicesBuilder() {
+ super(GenericServicesModel.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() {
+ return Arrays.asList(ConfigModelId.fromName("service"));
+ }
+
+ @Override
+ public void doBuild(GenericServicesModel model, Element spec, ConfigModelContext modelContext) {
+ String name = spec.getAttribute("name");
+ model.addCluster(new DomServiceClusterBuilder(name).build(modelContext.getParentProducer(), spec));
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java
new file mode 100644
index 00000000000..c6b2e305658
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic;
+
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.vespa.model.generic.service.ServiceCluster;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class GenericServicesModel extends ConfigModel {
+ private final List<ServiceCluster> clusters = new ArrayList<>();
+ public GenericServicesModel(ConfigModelContext modelContext) {
+ super(modelContext);
+ }
+
+ public void addCluster(ServiceCluster cluster) {
+ clusters.add(cluster);
+ }
+
+ public List<ServiceCluster> serviceClusters() {
+ return clusters;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java
new file mode 100644
index 00000000000..f4470f0dc9b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic.builder;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.generic.service.Module;
+import org.w3c.dom.Element;
+
+/**
+ * Produces sub services for generic services.
+ */
+public class DomModuleBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Module> {
+ private final String name;
+
+ public DomModuleBuilder(String name) {
+ this.name = name;
+ }
+
+ private void addChildren(Module s, Element subServiceSpec) {
+ for (Element nodeSpec : XML.getChildren(subServiceSpec, "module")) {
+ new DomModuleBuilder(nodeSpec.getAttribute("name")).build(s, nodeSpec);
+ }
+ }
+
+ @Override
+ protected Module doBuild(AbstractConfigProducer ancestor, Element subServiceSpec) {
+ Module s = new Module(ancestor, name);
+ addChildren(s, subServiceSpec);
+ return s;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java
new file mode 100644
index 00000000000..3cfd85c956b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic.builder;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.generic.service.Service;
+import com.yahoo.vespa.model.generic.service.ServiceCluster;
+import org.w3c.dom.Element;
+
+/**
+* @author lulf
+* @since 5.1
+*/
+public class DomServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Service> {
+ private final int i;
+
+ public DomServiceBuilder(int i) {
+ this.i = i;
+ }
+
+ @Override
+ protected com.yahoo.vespa.model.generic.service.Service doBuild(AbstractConfigProducer parent,
+ Element serviceSpec) {
+ ServiceCluster sc = (ServiceCluster) parent;
+ com.yahoo.vespa.model.generic.service.Service service = new com.yahoo.vespa.model.generic.service.Service(sc, i + "");
+ for (Element subServiceSpec : XML.getChildren(serviceSpec, "module")) {
+ new DomModuleBuilder(subServiceSpec.getAttribute("name")).build(service, subServiceSpec);
+ }
+ return service;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java
new file mode 100644
index 00000000000..d537afdd30e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic.builder;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.generic.service.ServiceCluster;
+import org.w3c.dom.Element;
+import java.util.Map;
+
+/**
+* @author lulf
+* @since 5.1
+*/
+public class DomServiceClusterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ServiceCluster> {
+
+ private String name;
+
+ public DomServiceClusterBuilder(String name) {
+ this.name = name;
+ }
+
+ @Override
+ protected ServiceCluster doBuild(AbstractConfigProducer ancestor, Element spec) {
+ ServiceCluster cluster = new ServiceCluster(ancestor, name, spec.getAttribute("command"));
+ int nodeIndex = 0;
+ for (Element nodeSpec : XML.getChildren(spec, "node")) {
+ com.yahoo.vespa.model.generic.service.Service service = new DomServiceBuilder(nodeIndex).build(cluster, nodeSpec);
+
+ // TODO: Currently creates the config for each service. Should instead build module tree first
+ // and store them in ServiceCluster. Then have some way of referencing them from each service.
+ for (Element subServiceSpec : XML.getChildren(spec, "module")) {
+ String subServiceName = subServiceSpec.getAttribute("name");
+ Map<String, AbstractConfigProducer<?>> map = service.getChildren();
+ // Add only non-conflicting modules. Does not merge unspecified configs that are specified in root though.
+ if (!map.containsKey(subServiceName))
+ new DomModuleBuilder(subServiceName).build(service, subServiceSpec);
+ }
+ nodeIndex++;
+ }
+ return cluster;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java
new file mode 100644
index 00000000000..77679e00d08
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.generic.builder;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java
new file mode 100644
index 00000000000..6922e6138f5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.generic;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java
new file mode 100644
index 00000000000..82190e3344e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic.service;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * A simple sub service that is essentially just to have a node with a nice name
+ * in the tree. Could might as well have used an AbstractConfigProducer as well,
+ * but that makes the code very confusing to read.
+ *
+ * @author lulf
+ */
+public class Module extends AbstractConfigProducer {
+
+ public Module(AbstractConfigProducer parent, String subId) {
+ super(parent, subId);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java
new file mode 100644
index 00000000000..77ebde12634
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic.service;
+
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.HostResource;
+
+/**
+ * An application specific generic service
+ * @author vegardh
+ *
+ */
+public class Service extends AbstractService {
+ private static final long serialVersionUID = 1L;
+
+ public Service(ServiceCluster parent, String id) {
+ super(parent, id);
+ setProp("clustertype", parent.getName());
+ setProp("clustername", parent.getName());
+ }
+
+ @Override
+ public int getPortCount() {
+ return 0;
+ }
+
+ @Override
+ public String getStartupCommand() {
+ return ((ServiceCluster) getParent()).getCommand();
+ }
+
+ private String getClusterName() {
+ return ((ServiceCluster) getParent()).getName();
+ }
+
+ /**
+ * Different services are represented using same class here, so we must take service name into account too
+ *
+ * @param host a host
+ * @return the index of the host
+ */
+ protected int getIndex(HostResource host) {
+ int i = 0;
+ for (com.yahoo.vespa.model.Service s : host.getServices()) {
+ if (!s.getClass().equals(getClass())) continue;
+ Service other = (Service)s;
+ if (s!=this && other.getClusterName().equals(getClusterName())) {
+ i++;
+ }
+ }
+ return i + 1;
+ }
+
+ @Override
+ public String getServiceType() {
+ return getClusterName();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java
new file mode 100644
index 00000000000..b81841442de
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.generic.service;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.HostSystem;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * A cluster of nodes running one application specific generic service. These are defined on the top level in the Vespa config
+ * @author vegardh
+ *
+ */
+public class ServiceCluster extends AbstractConfigProducer {
+
+ private static final long serialVersionUID = 1L;
+ private String command;
+ private String name;
+ private HostSystem hostSystem; // A generic cluster can resolve hosts for its nodes
+
+ public ServiceCluster(AbstractConfigProducer parent, String name, String command) {
+ super(parent, name);
+ this.command=command;
+ this.name=name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ String getCommand() {
+ return command;
+ }
+
+ public Collection<Service> services() {
+ Collection<Service> ret = new ArrayList<>();
+ for (Object child : getChildren().values()) {
+ if (child instanceof Service) ret.add((Service) child);
+ }
+ return ret;
+ }
+
+ @Override
+ public HostSystem getHostSystem() {
+ if (hostSystem!=null) return hostSystem;
+ return super.getHostSystem();
+ }
+
+ /**
+ * Sets the host system for this.
+ * @param hostSystem a {@link com.yahoo.vespa.model.HostSystem}
+ */
+ public void setHostSystem(HostSystem hostSystem) {
+ this.hostSystem = hostSystem;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java
new file mode 100644
index 00000000000..49683ab3d97
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.generic.service;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java
new file mode 100644
index 00000000000..4d185d0bb19
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java
@@ -0,0 +1,188 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ Provides the classes for the Vespa config model framework.
+
+ <p>The {@link com.yahoo.vespa.model.VespaModel VespaModel} class
+ is the natural starting point. It reads the user-defined
+ application specification, handles <a
+ href="#plugin_loading">plugin loading</a> and currently
+ instantiates one {@link com.yahoo.config.model.ApplicationConfigProducerRoot Vespa}
+ object. VespaModel is the root node in a tree of {@link
+com.yahoo.config.model.producer.AbstractConfigProducer
+ AbstractConfigProducers} that is built from the structure of the
+ user's specification. In a future version, the VespaModel can
+ contain multiple Vespa instances, each built from a separate user
+ specification (currently called 'services.xml').
+ </p>
+
+ <p>Each AbstractConfigProducer in the tree represents an actual
+ service or another logical unit in the Vespa system. An example of
+ a logical unit is a cluster that holds a set of services. Each
+ child class of {@link com.yahoo.config.model.producer.AbstractConfigProducer
+ AbstractConfigProducer} can contain hard-wired config that should
+ be delivered to the Vespa unit it represents, and its children. It
+ can also keep track of the status of the unit.
+ </p>
+
+ <p>A service that runs on a hardware host is always represented by
+ an {@link com.yahoo.vespa.model.AbstractService AbstractService}
+ object, containing the command that will be used to start the
+ service, which host it is running on, and the ports that it
+ uses. Each hardware host in the Vespa system is represented by a
+ {@link com.yahoo.vespa.model.Host Host} object, and the set of
+ hosts is handled by the {@link com.yahoo.vespa.model.HostSystem
+ HostSystem}. Each Host is responsible for avoiding port collisions
+ between services, see <a href="#port_allocation">port
+ allocation</a>.
+ </p>
+
+ <h3>Config Generation</h3>
+
+ <p>The method {@link
+ com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder, String)
+ VespaModel.getConfig} looks up the ConfigProducer with the config
+ ID that config is requested for. The composition of the actual
+ config starts from the root node of the ConfigProducer tree, which
+ is always an instance of the {@link com.yahoo.config.model.ApplicationConfigProducerRoot
+ Vespa} class, and traverses each level of the tree back down to
+ the ConfigProducer that got the first call from the root node.
+ This is handled in such a way
+ that config from the root node gets the lowest priority, and the
+ ConfigProducer itself has the highest priority when the same
+ parameter is given different values in the path down the tree.</p>
+
+ <p>User defined configuration can be embedded in the service setup
+ file in the application specification. Currently this is done by adding
+ &lt;config&gt; tags at the desired position in the file named
+ 'services.xml', where each position corresponds to a
+ ConfigProducer. These config values have a higher priority than
+ the default config returned from the same method. However, it can be overridden by the config
+ from a ConfigProducer at a lower level, both by its getConfig
+ method and by user defined config.
+ </p>
+
+ <h4>Example:</h4>
+ <p>
+ Say we have a config named 'sample' with an integer parameter
+ named 'v'. If the VespaModel root node's {@link
+ com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder,String)
+ getConfig(builder, configid)} method returns a hardcoded value of
+ 'v=2' for that parameter, this becomes the default value for all
+ ConfigProducers when asking for the 'sample' config. Now, let's
+ assume that we need the 'sample' config for a ConfigProducer of
+ class 'Grandchild', which has a configId
+ 'grandchild_0'. grandchild_0's parent in the ConfigProducer tree
+ is a ConfigProducer of class 'Child' and configId 'child_0' which
+ is a direct child of the Vespa root node:
+ </p>
+
+ <p>The initial step when retrieving a config is always a call to
+ {@link
+ com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder,String)
+ VespaModel.getConfig(builder, configId}. Here, the call
+ could look like this:
+ VespaModel.getConfig(builder, &quot;grandchild_0&quot;).
+ This triggers a call to the {@link
+ com.yahoo.config.model.producer.AbstractConfigProducer#cascadeConfig(com.yahoo.config.ConfigInstance.Builder)}) AbstractConfigProducer.cascadeConfig} method for
+ grandchild_0 which calls the same method in child_0, and finally
+ in the VespaModel root node, where the {@link
+ com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder,String)
+ getConfig (name, namespace)} method returns the value 'v=2' as
+ previously mentioned. This value might be overridden on the
+ traversal back down in the tree, first in child_0, which could
+ return the value 'v=1'. Now, if the user specification for child_0
+ contains the value 'v=0', this overrides the previous values. The
+ same happens for grandchild_0: if there is a value returned from
+ the getConfig() method, this overrides the value from child_0, and
+ if there is a value from the user specification for grandchild_0,
+ that will always become the final result.
+ </p>
+
+
+ <h3><a name="plugin_loading">Plugin Loading</a></h3>
+
+ <p>Each highest-level node in the setup file from the user's
+ application specification corresponds to a {@link
+com.yahoo.config.model.builder.xml.ConfigModelBuilder ConfigModelBuilder}. The
+ builders are loaded when the system is started. Each builder produce
+ a {@link com.yahoo.config.model.ConfigModel ConfigModel}. The model can depend
+ on other models by having them injected in its constructor. This ensures
+ that the builders are invoked in the correct order as well.
+ In its build method, the builder is responsible for building all its
+ ConfigProducers, and linking them to the parent ConfigProducer
+ given as input argument.
+ </p>
+
+ <p>The built models are given to other models that depends on it.
+ </p>
+
+ <h4>Important notes for plugin developers:</h4>
+ <ul>
+
+ <li>The constructors of all child classes of {@link
+com.yahoo.config.model.producer.AbstractConfigProducer
+ AbstractConfigProducer} should throw a new 'RuntimeException' upon
+ errors in xml or other initialization problems. This allows the
+ exception to be nested upwards, adding valuable information from
+ each level in the ConfigProducer tree to the error message output
+ to the user. The exception should contain detailed information
+ about the error that occurred.
+ </li>
+
+ <li>The plugins are not allowed to put any constraints on the
+ contents of the hosts specification file (currently named
+ 'hosts.xml'), such as demanding special hostnames for
+ different service types. This file belongs solely to the
+ vespamodel framework.
+ </li>
+ </ul>
+
+
+ <h3><a name="port_allocation">Port Allocation</a></h3>
+
+ <p>Each {@link com.yahoo.vespa.model.Host Host} has an available
+ dynamic port range running from {@link
+ com.yahoo.vespa.model.HostResource#BASE_PORT BASE_PORT} (currently 19100)
+ with {@link com.yahoo.vespa.model.HostResource#MAX_PORTS MAX_PORTS}
+ (currently 799) ports upwards. When an instance of a subclass of
+ {@link com.yahoo.vespa.model.AbstractService AbstractService} is
+ assigned to a host, it is given the lowest available base port in
+ this range. The service owns a continuous port range of {@link
+ com.yahoo.vespa.model.Service#getPortCount Service.getPortCount}
+ ports upwards from the base port.
+ </p>
+
+ <p>The base port for a specific service instance on a host is
+ decided by {@link
+ com.yahoo.vespa.model.AbstractService #getInstanceWantedPort
+ AbstractService.getInstanceWantedPort}. The most important aspects
+ are described below:
+ </p>
+
+ <p>It is not possible to reserve a certain port inside the dynamic
+ range, but a service can specify that it wants a base port outside
+ the range by overriding the {@link
+ com.yahoo.vespa.model.Service #getWantedPort Service.getWantedPort}
+ method. If the service type is required to run with the specified
+ base port, it must also override the {@link
+ com.yahoo.vespa.model.Service #requiresWantedPort
+ Service.requiresWantedPort}. The user specified port number
+ returned from {@link com.yahoo.vespa.model.Service #getWantedPort
+ getWantedPort} applies to the first instance of that specific
+ subclass on each host, and the next instance on the same host
+ <em>must</em> have its baseport specified by the 'baseport' attribute
+ in 'services.xml'
+ </p>
+
+ <p>The user-defined application specification can also give a
+ required base port for each individual service. Currently this is
+ done by adding a 'baseport' attribute to the service's tag in the
+ file named 'hosts.xml'. If the port is not available, an
+ exception will be thrown.
+ </p>
+
+*/
+@ExportPackage
+package com.yahoo.vespa.model;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
new file mode 100644
index 00000000000..344c5e16d29
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
@@ -0,0 +1,385 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.routing;
+
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig;
+import com.yahoo.document.select.DocumentSelector;
+import com.yahoo.messagebus.routing.ApplicationSpec;
+import com.yahoo.messagebus.routing.HopSpec;
+import com.yahoo.messagebus.routing.RouteSpec;
+import com.yahoo.messagebus.routing.RoutingTableSpec;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * This class is responsible for generating all hops and routes for the Document protocol running on message bus. All
+ * the code within could really be part of {@link Routing}, but it has been partitioned out to allow better readability
+ * and also more easily maintainable as the number of protocols increase.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public final class DocumentProtocol implements Protocol, DocumentrouteselectorpolicyConfig.Producer {
+
+ private static final String NAME = "document";
+ private ApplicationSpec application;
+ private RoutingTableSpec routingTable;
+ ConfigModelRepo repo;
+
+ public static String getIndexedRouteName(String configId) {
+ return configId + "-index";
+ }
+
+ public static String getDirectRouteName(String configId) {
+ return configId + "-direct";
+ }
+
+ /**
+ * Constructs a new document protocol based on the content of the given plugins.
+ *
+ * @param plugins The plugins to reflect on.
+ */
+ public DocumentProtocol(ConfigModelRepo plugins) {
+ application = createApplicationSpec(plugins);
+ routingTable = createRoutingTable(plugins);
+ this.repo = plugins;
+ }
+
+ /**
+ * Creates a service index based on the plugins loaded. This means to fill the index with all services known by this
+ * protocol by traversing the plugins.
+ *
+ * @param plugins All initialized plugins of the Vespa model.
+ * @return The index of all known services.
+ */
+ private static ApplicationSpec createApplicationSpec(ConfigModelRepo plugins) {
+ ApplicationSpec ret = new ApplicationSpec();
+
+ for (ContentCluster cluster : Content.getContentClusters(plugins)) {
+ for (com.yahoo.vespa.model.content.Distributor node : cluster.getDistributorNodes().getChildren().values()) {
+ ret.addService(NAME, node.getConfigId() + "/default");
+ }
+ }
+
+ for (ContainerCluster containerCluster: ContainerModel.containerClusters(plugins)) {
+ ContainerDocproc containerDocproc = containerCluster.getDocproc();
+ if (containerDocproc != null) {
+ createDocprocChainSpec(ret,
+ containerDocproc.getChains().allChains().allComponents(),
+ containerCluster.getContainers());
+ }
+ }
+
+ return ret;
+ }
+
+ private static void createDocprocChainSpec(ApplicationSpec spec,
+ List<DocprocChain> docprocChains,
+ List<Container> containerNodes) {
+ for (DocprocChain chain: docprocChains) {
+ for (Container node: containerNodes)
+ spec.addService(NAME, node.getConfigId() + "/chain." + chain.getComponentId().stringValue());
+ }
+ }
+
+ @Override
+ public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) {
+ for (ContentCluster cluster : Content.getContentClusters(repo)) {
+ addRoute(cluster.getConfigId(), cluster.getRoutingSelector(), builder);
+ }
+ }
+
+ private static void addRoute(String clusterConfigId, String selector, DocumentrouteselectorpolicyConfig.Builder builder) {
+ try {
+ new DocumentSelector(selector);
+ } catch (com.yahoo.document.select.parser.ParseException e) {
+ throw new IllegalArgumentException("Failed to parse selector '" + selector + "' for route '" + clusterConfigId +
+ "' in policy 'DocumentRouteSelector'.");
+ }
+ DocumentrouteselectorpolicyConfig.Route.Builder routeBuilder = new DocumentrouteselectorpolicyConfig.Route.Builder();
+ routeBuilder.name(clusterConfigId);
+ routeBuilder.selector(selector);
+ builder.route(routeBuilder);
+ }
+
+ /**
+ * This function extrapolates any routes for the document protocol that it can from the vespa model.
+ *
+ * @param plugins All initialized plugins of the vespa model.
+ * @return Routing table for the document protocol.
+ */
+ private static RoutingTableSpec createRoutingTable(ConfigModelRepo plugins) {
+ // Build simple hops and routes.
+ List<ContentCluster> content = Content.getContentClusters(plugins);
+ Collection<ContainerCluster> containerClusters = ContainerModel.containerClusters(plugins);
+
+ RoutingTableSpec table = new RoutingTableSpec(NAME);
+ addContainerClusterDocprocHops(containerClusters, table);
+ addContentRouting(content, table);
+
+ // Build the indexing hop if it is possible to derive.
+ addIndexingHop(content, table);
+
+ // Build the default route if is is possible to derive.
+ addDefaultRoute(content, containerClusters, table);
+
+ // Return the complete routing table.
+ simplifyRouteNames(table);
+ return table;
+ }
+
+ private static void addContainerClusterDocprocHops(Collection<ContainerCluster> containerClusters,
+ RoutingTableSpec table) {
+
+ for (ContainerCluster cluster: containerClusters) {
+ ContainerDocproc docproc = cluster.getDocproc();
+
+ if (docproc != null) {
+ String policy = policy(docproc);
+
+ for (DocprocChain chain : docproc.getChains().allChains().allComponents()) {
+ addChainHop(table, cluster.getConfigId(), policy, chain);
+ }
+ }
+ }
+ }
+
+ private static void addChainHop(RoutingTableSpec table, String configId, String policy, DocprocChain chain) {
+ final String selector;
+ if (policy != null) {
+ selector = configId + "/" + policy + "/" + chain.getSessionName();
+ } else {
+ selector = "[LoadBalancer:cluster=" + configId +
+ ";session=" + chain.getSessionName() + "]";
+ }
+ table.addHop(new HopSpec(chain.getServiceName(), selector));
+ }
+
+ private static String policy(ContainerDocproc docproc) {
+ if (docproc.getNumNodesPerClient() > 0) {
+ return "[SubsetService:" + docproc.getNumNodesPerClient() + "]";
+ } else if (docproc.isPreferLocalNode()) {
+ return "[LocalService]";
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Create hops to all configured storage nodes for the Document protocol. The "Distributor" policy resolves its
+ * recipients using slobrok lookups, so it requires no configured recipients.
+ *
+ * @param content The storage model from {@link com.yahoo.vespa.model.VespaModel}.
+ * @param table The routing table to add to.
+ */
+ private static void addContentRouting(List<ContentCluster> content, RoutingTableSpec table) {
+
+ for (ContentCluster cluster : content) {
+ RouteSpec spec = new RouteSpec(cluster.getConfigId());
+
+ if (cluster.getSearch().hasIndexedCluster()) {
+ table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]"));
+ table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId()))
+ .addHop(cluster.getSearch().getIndexed().getIndexingServiceName())
+ .addHop("[Content:cluster=" + cluster.getName() + "]"));
+ table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId()))
+ .addHop("[Content:cluster=" + cluster.getName() + "]"));
+ } else {
+ table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]"));
+ }
+ table.addRoute(new RouteSpec("storage/cluster." + cluster.getName())
+ .addHop("route:" + cluster.getConfigId()));
+ }
+ }
+
+ /**
+ * Create the "indexing" hop. This hop contains all non-streaming search clusters as recipients, and the routing
+ * policy "SearchCluster" will decide which cluster(s) are to receive every document passed through it based on a
+ * document select string derived from services.xml.
+ *
+ * @param table The routing table to add to.
+ */
+ private static void addIndexingHop(List<ContentCluster> content, RoutingTableSpec table) {
+ if (content.isEmpty()) {
+ return;
+ }
+ HopSpec hop = new HopSpec("indexing", "[DocumentRouteSelector]");
+ for (ContentCluster cluster : content) {
+ hop.addRecipient(cluster.getConfigId());
+ }
+ if (hop.hasRecipients()) {
+ table.addHop(hop);
+ }
+ }
+
+ /**
+ * Create the "default" route for the Document protocol. This route will be either a route to storage or a route to
+ * search. Since we will be supporting recovery from storage, storage takes precedence over search when deciding on
+ * the final target of the default route. If there is an unambigous docproc cluster in the application, the default
+ * route will pass through this.
+ *
+ * @param content The content model from {@link com.yahoo.vespa.model.VespaModel}.
+ * @param containerClusters a collection of {@link com.yahoo.vespa.model.container.ContainerCluster}s
+ * @param table The routing table to add to.
+ */
+ private static void addDefaultRoute(List<ContentCluster> content,
+ Collection<ContainerCluster> containerClusters,
+ RoutingTableSpec table) {
+ List<String> hops = new ArrayList<>();
+ if (!content.isEmpty()) {
+ boolean found = false;
+ for (int i = 0, len = table.getNumHops(); i < len; ++i) {
+ if (table.getHop(i).getName().equals("indexing")) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ hops.add("indexing");
+ }
+ }
+ if (!hops.isEmpty()) {
+ RouteSpec route = new RouteSpec("default");
+ String hop = getContainerClustersDocprocHop(containerClusters);
+ if (hop != null) {
+ route.addHop(hop);
+ }
+ int numHops = hops.size();
+ if (numHops == 1) {
+ route.addHop(hops.get(0));
+ } else {
+ StringBuilder str = new StringBuilder();
+ for (int i = 0; i < numHops; ++i) {
+ str.append(hops.get(i)).append(i < numHops - 1 ? " " : "");
+ }
+ route.addHop("[AND:" + str.toString() + "]");
+ }
+ table.addRoute(route);
+ }
+ }
+
+ private static String getContainerClustersDocprocHop(Collection<ContainerCluster> containerClusters) {
+ DocprocChain result = null;
+
+ for (ContainerCluster containerCluster: containerClusters) {
+ DocprocChain defaultChain = getDefaultChain(containerCluster.getDocproc());
+ if (defaultChain != null) {
+ if (result != null)
+ throw new RuntimeException("Only a single default docproc chain is allowed across all container clusters");
+
+ result = defaultChain;
+ }
+ }
+
+ return result == null ?
+ null:
+ result.getServiceName();
+ }
+
+ private static DocprocChain getDefaultChain(ContainerDocproc docproc) {
+ return docproc == null ?
+ null:
+ docproc.getChains().allChains().getComponent("default");
+ }
+
+ /**
+ * Attempts to simplify all route names by removing prefixing plugin name and whatever comes before the dot (.) in
+ * the second naming element. This can only be done to those routes that do not share primary name elements with
+ * other routes (e.g. a search clusters with the same name as a storage cluster).
+ *
+ * @param table The routing table whose route names are to be simplified.
+ */
+ private static void simplifyRouteNames(RoutingTableSpec table) {
+ if (table == null || !table.hasRoutes()) {
+ return;
+ }
+
+ // Pass 1: Determine which simplifications are in conflict.
+ Map<String, Set<String>> simple = new TreeMap<>();
+ List<String> broken = new ArrayList<>();
+ for (int i = 0, len = table.getNumRoutes(); i < len; ++i) {
+ String before = table.getRoute(i).getName();
+ String after = simplifyRouteName(before);
+ if (simple.containsKey(after)) {
+ Set<String> l = simple.get(after);
+ l.add(before);
+ if (!(l.contains("content/" + after) && l.contains("storage/cluster." + after) && (l.size() == 2))) {
+ broken.add(after);
+ }
+ } else {
+ Set<String> l = new HashSet<>();
+ l.add(before);
+ simple.put(after, l);
+ }
+ }
+
+ // Pass 2: Simplify all non-conflicting route names by alias.
+ Set<RouteSpec> alias = new HashSet<>();
+ Set<String> unique = new HashSet<>();
+ for (int i = 0; i < table.getNumRoutes(); ) {
+ RouteSpec route = table.getRoute(i);
+ String before = route.getName();
+ String after = simplifyRouteName(before);
+ if (!before.equals(after)) {
+ if (!broken.contains(after)) {
+ if (route.getNumHops() == 1 && route.getHop(0).equals(route.getName())) {
+ alias.add(new RouteSpec(after).addHop(route.getHop(0))); // full route name is redundant
+ unique.add(after);
+ table.removeRoute(i);
+ continue; // do not increment i
+ } else {
+ if (!unique.contains(after)) {
+ alias.add(new RouteSpec(after).addHop("route:" + before));
+ unique.add(after);
+ }
+ }
+ }
+ }
+ ++i;
+ }
+ for (RouteSpec rs : alias) {
+ table.addRoute(rs);
+ }
+ }
+
+ /**
+ * Returns a simplified version of the given route name. This method will remove the first component of the name as
+ * separated by a forward slash, and then remove the first component of the remaining name as separated by a dot.
+ *
+ * @param name The route name to simplify.
+ * @return The simplified route name.
+ */
+ private static String simplifyRouteName(String name) {
+ String[] foo = name.split("/", 2);
+ if (foo.length < 2) {
+ return name;
+ }
+ String[] bar = foo[1].split("\\.", 2);
+ if (bar.length < 2) {
+ return foo[1];
+ }
+ return bar[1];
+ }
+
+ @Override
+ public ApplicationSpec getApplicationSpec() {
+ return application;
+ }
+
+ @Override
+ public RoutingTableSpec getRoutingTableSpec() {
+ return routingTable;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java
new file mode 100644
index 00000000000..20b503aac96
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.routing;
+
+import com.yahoo.messagebus.routing.ApplicationSpec;
+import com.yahoo.messagebus.routing.RoutingTableSpec;
+
+/**
+ * This interface defines the necessary api for {@link Routing} to prepare and combine routing tables for all available
+ * protocols.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface Protocol {
+
+ /**
+ * Returns the specification for the routing table of this protocol.
+ *
+ * @return The routing table spec.
+ */
+ public RoutingTableSpec getRoutingTableSpec();
+
+ /**
+ * Returns the specification of the application as seen by this protocol.
+ *
+ * @return The application spec.
+ */
+ public ApplicationSpec getApplicationSpec();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java
new file mode 100644
index 00000000000..d5a987869e9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java
@@ -0,0 +1,201 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.routing;
+
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.messagebus.routing.*;
+import com.yahoo.messagebus.MessagebusConfig;
+import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig;
+import java.util.*;
+
+/**
+ * This is the routing plugin of the Vespa model. This class is responsible for parsing all routing information given
+ * explicitly by the user in the optional &lt;routing&gt; element. If there is no such element, only default routes and
+ * hops will be available.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Routing extends ConfigModel {
+
+ private final List<String> errors = new ArrayList<>();
+ private ApplicationSpec explicitApplication = null;
+ private RoutingSpec explicitRouting = null;
+ private List<Protocol> protocols = new ArrayList<>();
+ private RoutingSpec derivedRouting;
+
+ public Routing(ConfigModelContext modelContext) {
+ super(modelContext);
+ }
+
+ /**
+ * Sets the application specification to include when verifying the complete routing config. This needs to be
+ * invoked before {@link #deriveCommonSettings(com.yahoo.config.model.ConfigModelRepo)} to be included.
+ *
+ * @param app The application specification to include.
+ */
+ public void setExplicitApplicationSpec(ApplicationSpec app) {
+ explicitApplication = app;
+ }
+
+ /**
+ * Sets the routing specification to include in the derived routing config. This needs to be invoked before
+ * {@link #deriveCommonSettings(com.yahoo.config.model.ConfigModelRepo)} to be included.
+ *
+ * @param routing The routing specification to include.
+ */
+ public void setExplicitRoutingSpec(RoutingSpec routing) {
+ explicitRouting = routing;
+ }
+
+ public final List<Protocol> getProtocols() { return protocols; }
+
+ /**
+ * Derives all routing settings that can be found by inspecting the given plugin container.
+ *
+ * @param plugins All initialized plugins of the vespa model.
+ */
+ public void deriveCommonSettings(ConfigModelRepo plugins) {
+ // Combine explicit routing with protocol derived routing.
+ ApplicationSpec app = explicitApplication != null ? new ApplicationSpec(explicitApplication) : new ApplicationSpec();
+ RoutingSpec routing = explicitRouting != null ? new RoutingSpec(explicitRouting) : new RoutingSpec();
+ protocols.clear();
+ protocols.add(new DocumentProtocol(plugins));
+ for (Protocol protocol : protocols) {
+ app.add(protocol.getApplicationSpec());
+ addRoutingTable(routing, protocol.getRoutingTableSpec());
+ }
+
+ // Add default routes where appropriate, and sort content.
+ for (int i = 0, len = routing.getNumTables(); i < len; ++i) {
+ RoutingTableSpec table = routing.getTable(i);
+ if ( ! table.hasRoute("default") && table.getNumRoutes() == 1) {
+ table.addRoute(new RouteSpec("default").addHop("route:" + table.getRoute(0).getName()));
+ }
+ table.sort();
+ }
+
+ // Verify and export all produced configs.
+ errors.clear();
+ if (routing.verify(app, errors)) {
+ this.derivedRouting=routing;
+ }
+ }
+
+ public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) {
+ for (Protocol protocol : protocols) {
+ if (protocol instanceof DocumentProtocol) {
+ ((DocumentProtocol)protocol).getConfig(builder);
+ }
+ }
+ }
+
+ public void getConfig(MessagebusConfig.Builder builder) {
+ if (derivedRouting==null) {
+ // The error list should be populated then
+ return;
+ }
+ if (derivedRouting.hasTables()) {
+ for (int tableIdx = 0, numTables = derivedRouting.getNumTables(); tableIdx < numTables; ++tableIdx) {
+ RoutingTableSpec table = derivedRouting.getTable(tableIdx);
+ MessagebusConfig.Routingtable.Builder tableBuilder = new MessagebusConfig.Routingtable.Builder();
+ tableBuilder.protocol(table.getProtocol());
+ if (table.hasHops()) {
+ for (int hopIdx = 0, numHops = table.getNumHops(); hopIdx < numHops; ++hopIdx) {
+ MessagebusConfig.Routingtable.Hop.Builder hopBuilder = new MessagebusConfig.Routingtable.Hop.Builder();
+ HopSpec hop = table.getHop(hopIdx);
+ hopBuilder.name(hop.getName());
+ hopBuilder.selector(hop.getSelector());
+ if (hop.getIgnoreResult()) {
+ hopBuilder.ignoreresult(true);
+ }
+ if (hop.hasRecipients()) {
+ for (int recipientIdx = 0, numRecipients = hop.getNumRecipients();
+ recipientIdx < numRecipients; ++recipientIdx)
+ {
+ hopBuilder.recipient(hop.getRecipient(recipientIdx));
+ }
+ }
+ tableBuilder.hop(hopBuilder);
+ }
+ }
+ if (table.hasRoutes()) {
+ for (int routeIdx = 0, numRoutes = table.getNumRoutes(); routeIdx < numRoutes; ++routeIdx) {
+ MessagebusConfig.Routingtable.Route.Builder routeBuilder = new MessagebusConfig.Routingtable.Route.Builder();
+ RouteSpec route = table.getRoute(routeIdx);
+ routeBuilder.name(route.getName());
+ if (route.hasHops()) {
+ for (int hopIdx = 0, numHops = route.getNumHops(); hopIdx < numHops; ++hopIdx) {
+ routeBuilder.hop(route.getHop(hopIdx));
+ }
+ }
+ tableBuilder.route(routeBuilder);
+ }
+ }
+ builder.routingtable(tableBuilder);
+ }
+ }
+ }
+
+ /**
+ * Adds the given routing table to the given routing spec. This method will not copy hops or routes that are already
+ * defined in the target table.
+ *
+ * @param routing The routing spec to add to.
+ * @param from The table to copy content from.
+ */
+ private static void addRoutingTable(RoutingSpec routing, RoutingTableSpec from) {
+ RoutingTableSpec to = getRoutingTable(routing, from.getProtocol());
+ if (to != null) {
+ Set<String> names = new HashSet<>();
+ for (int i = 0, len = to.getNumHops(); i < len; ++i) {
+ names.add(to.getHop(i).getName());
+ }
+ for (int i = 0, len = from.getNumHops(); i < len; ++i) {
+ HopSpec hop = from.getHop(i);
+ if (!names.contains(hop.getName())) {
+ to.addHop(hop);
+ }
+ }
+
+ names.clear();
+ for (int i = 0, len = to.getNumRoutes(); i < len; ++i) {
+ names.add(to.getRoute(i).getName());
+ }
+ for (int i = 0, len = from.getNumRoutes(); i < len; ++i) {
+ RouteSpec route = from.getRoute(i);
+ if (!names.contains(route.getName())) {
+ to.addRoute(route);
+ }
+ }
+ } else {
+ routing.addTable(from);
+ }
+ }
+
+ /**
+ * Returns the routing table from the given routing spec that belongs to the named protocol.
+ *
+ * @param routing The routing whose tables to search through.
+ * @param protocol The name of the protocol whose table to return.
+ * @return The routing table found, or null.
+ */
+ private static RoutingTableSpec getRoutingTable(RoutingSpec routing, String protocol) {
+ for (int i = 0, len = routing.getNumTables(); i < len; ++i) {
+ RoutingTableSpec table = routing.getTable(i);
+ if (protocol.equals(table.getProtocol())) {
+ return table;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns a list of errors found when preparing the routing configuration.
+ *
+ * @return The error list.
+ */
+ public List<String> getErrors() {
+ return Collections.unmodifiableList(errors);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
new file mode 100644
index 00000000000..a9dd764458c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.UserConfigRepo;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import java.util.*;
+
+/**
+ * Superclass for search clusters.
+ *
+ * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a>
+ */
+public abstract class AbstractSearchCluster extends AbstractConfigProducer
+ implements
+ DocumentdbInfoConfig.Producer,
+ IndexInfoConfig.Producer,
+ IlscriptsConfig.Producer
+{
+ private Double queryTimeout;
+ protected String clusterName;
+ protected int index;
+ private Double visibilityDelay = 0.0;
+ private List<String> documentNames = new ArrayList<>();
+
+ protected List<SearchDefinitionSpec> localSDS = new LinkedList<>();
+
+ public static final class IndexingMode {
+
+ public static final IndexingMode REALTIME = new IndexingMode("REALTIME");
+ public static final IndexingMode STREAMING = new IndexingMode("STREAMING");
+
+ public static IndexingMode createIndexingMode(String ixm) {
+ if ("REALTIME".equalsIgnoreCase(ixm)) {
+ return REALTIME;
+ } else if ("STREAMING".equalsIgnoreCase(ixm)) {
+ return STREAMING;
+ }
+ return null;
+ }
+
+ private String name;
+
+ private IndexingMode(String name) {
+ this.name = name;
+ }
+
+ public String getName() { return name; }
+
+ public String toString() {
+ return "indexingmode: " + name;
+ }
+ }
+
+ public static final class SearchDefinitionSpec {
+ private final SearchDefinition searchDefinition;
+ private final UserConfigRepo userConfigRepo;
+
+ public SearchDefinitionSpec(SearchDefinition searchDefinition,
+ UserConfigRepo userConfigRepo) {
+ this.searchDefinition = searchDefinition;
+ this.userConfigRepo = userConfigRepo;
+ }
+
+ public SearchDefinition getSearchDefinition() {
+ return searchDefinition;
+ }
+
+ public UserConfigRepo getUserConfigs() {
+ return userConfigRepo;
+ }
+ }
+
+ public AbstractSearchCluster(AbstractConfigProducer parent,
+ String clusterName, int index) {
+ super(parent, "cluster." + clusterName);
+ this.clusterName = clusterName;
+ this.index = index;
+ }
+ public void addDocumentNames(SearchDefinition searchDefinition) {
+ String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName();
+ documentNames.add(dName);
+ }
+
+ /**
+ * Returns a List with document names used in this search cluster
+ * @return contained document names
+ */
+ public List<String> getDocumentNames() { return documentNames; }
+
+ public List<SearchDefinitionSpec> getLocalSDS() {
+ return localSDS;
+ }
+
+ public String getClusterName() { return clusterName; }
+ public final String getIndexingModeName() { return getIndexingMode().getName(); }
+ public final boolean isRealtime() { return getIndexingMode() == IndexingMode.REALTIME; }
+ public final boolean isStreaming() { return getIndexingMode() == IndexingMode.STREAMING; }
+ public final AbstractSearchCluster setQueryTimeout(Double to) {
+ this.queryTimeout=to;
+ return this;
+ }
+ public final AbstractSearchCluster setVisibilityDelay(double delay) {
+ this.visibilityDelay=delay;
+ return this;
+ }
+ protected abstract IndexingMode getIndexingMode();
+ public final Double getVisibilityDelay() { return visibilityDelay; }
+ public final Double getQueryTimeout() { return queryTimeout; }
+ public abstract int getRowBits();
+ public final void setClusterIndex(int index) { this.index = index; }
+ public final int getClusterIndex() { return index; }
+ protected abstract void assureSdConsistent();
+
+ @Override
+ public abstract void getConfig(DocumentdbInfoConfig.Builder builder);
+ @Override
+ public abstract void getConfig(IndexInfoConfig.Builder builder);
+ @Override
+ public abstract void getConfig(IlscriptsConfig.Builder builder);
+ public abstract void getConfig(RankProfilesConfig.Builder builder);
+ public abstract void getConfig(AttributesConfig.Builder builder);
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java
new file mode 100644
index 00000000000..ebe43bc07db
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java
@@ -0,0 +1,226 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.vespa.config.search.core.FdispatchrcConfig;
+import com.yahoo.vespa.config.search.core.PartitionsConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+import com.yahoo.vespa.model.content.SearchCoverage;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a dispatch (top-level (tld) or mid-level).
+ * There must be one or more tld instances in a search cluster.
+ *
+ * @author arnej27959
+ */
+@RestartConfigs({FdispatchrcConfig.class, PartitionsConfig.class})
+public class Dispatch extends AbstractService implements SearchInterface,
+ FdispatchrcConfig.Producer,
+ PartitionsConfig.Producer {
+
+ private static final String TLD_NAME = "tld";
+ private static final String DISPATCH_NAME = "dispatch";
+
+ private static final long serialVersionUID = 1L;
+ private final DispatchGroup dispatchGroup;
+ private final NodeSpec nodeSpec;
+ private final int dispatchLevel;
+ private final boolean preferLocalRow;
+ private final boolean isTopLevel;
+
+ private Dispatch(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String subConfigId,
+ NodeSpec nodeSpec, int dispatchLevel, boolean preferLocalRow, boolean isTopLevel) {
+ super(parent, subConfigId);
+ this.dispatchGroup = dispatchGroup;
+ this.nodeSpec = nodeSpec;
+ this.dispatchLevel = dispatchLevel;
+ this.preferLocalRow = preferLocalRow;
+ this.isTopLevel = isTopLevel;
+ portsMeta.on(0).tag("rpc").tag("admin");
+ portsMeta.on(1).tag("fs4");
+ portsMeta.on(2).tag("http").tag("json").tag("health").tag("state");
+ setProp("clustertype", "search")
+ .setProp("clustername", dispatchGroup.getClusterName())
+ .setProp("index", nodeSpec.rowId());
+ monitorService();
+ }
+
+ public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId) {
+ return createTld(dispatchGroup, parent, rowId, false);
+ }
+
+ public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId, boolean preferLocalRow) {
+ String subConfigId = TLD_NAME + "." + rowId;
+ return new Dispatch(dispatchGroup, parent, subConfigId, new NodeSpec(rowId, 0), 0, preferLocalRow, true);
+ }
+
+ public static Dispatch createTldWithContainerIdInName(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String containerName, int containerIndex) {
+ String subConfigId = containerName + "." + containerIndex + "." + TLD_NAME + "." + containerIndex;
+ return new Dispatch(dispatchGroup, parent, subConfigId, new NodeSpec(containerIndex, 0), 0, false, true);
+ }
+
+ public static Dispatch createDispatchWithStableConfigId(DispatchGroup dispatchGroup, AbstractConfigProducer parent, NodeSpec nodeSpec, int distributionKey, int dispatchLevel) {
+ String subConfigId = DISPATCH_NAME + "." + distributionKey;
+ return new Dispatch(dispatchGroup, parent, subConfigId, nodeSpec, dispatchLevel, false, false);
+ }
+
+ /**
+ * Override the default service-type
+ * @return String "topleveldispatch"
+ */
+ public String getServiceType() {
+ return "topleveldispatch";
+ }
+
+ /**
+ * @return the startup command
+ */
+ public String getStartupCommand() {
+ return "exec sbin/fdispatch -c $VESPA_CONFIG_ID";
+ }
+
+ public int getFrtPort() { return getRelativePort(0); }
+ public int getDispatchPort() { return getRelativePort(1); }
+ public @Override int getHealthPort() { return getRelativePort(2); }
+
+ /**
+ * Twice the default of the number of threads in the container.
+ * Could have been unbounded if it was not roundrobin, but stack based usage in dispatch.
+ * We are not putting to much magic into this one as this will disappear as soon as
+ * dispatch is implemented in Java.
+ */
+ public int getMaxThreads() { return 500*2; }
+
+ public String getHostname() {
+ return getHost().getHostName();
+ }
+
+ @Override
+ public NodeSpec getNodeSpec() {
+ return nodeSpec;
+ }
+
+ public String getDispatcherConnectSpec() {
+ return "tcp/" + getHost().getHostName() + ":" + getDispatchPort();
+ }
+
+ public DispatchGroup getDispatchGroup() {
+ return dispatchGroup;
+ }
+
+ @Override
+ public void getConfig(FdispatchrcConfig.Builder builder) {
+ builder.ptport(getDispatchPort()).
+ frtport(getFrtPort()).
+ healthport(getHealthPort()).
+ maxthreads(getMaxThreads());
+ if (!isTopLevel) {
+ builder.partition(getNodeSpec().partitionId());
+ builder.dispatchlevel(dispatchLevel);
+ }
+ }
+
+ @Override
+ public void getConfig(PartitionsConfig.Builder builder) {
+ int rowbits = dispatchGroup.getRowBits();
+ final PartitionsConfig.Dataset.Builder datasetBuilder = new PartitionsConfig.Dataset.Builder().
+ id(0).
+ refcost(1).
+ rowbits(rowbits).
+ numparts(dispatchGroup.getNumPartitions()).
+ mpp(dispatchGroup.getMinNodesPerColumn());
+ if (dispatchGroup.useFixedRowInDispatch()) {
+ datasetBuilder.querydistribution(PartitionsConfig.Dataset.Querydistribution.Enum.FIXEDROW);
+ datasetBuilder.maxnodesdownperfixedrow(dispatchGroup.getMaxNodesDownPerFixedRow());
+ }
+ SearchCoverage coverage = dispatchGroup.getSearchCoverage();
+ if (coverage != null) {
+ if (coverage.getMinimum() != null) {
+ datasetBuilder.minimal_searchcoverage(coverage.getMinimum() * 100); // as percentage
+ }
+ if (coverage.getMinWaitAfterCoverageFactor() != null) {
+ datasetBuilder.higher_coverage_minsearchwait(coverage.getMinWaitAfterCoverageFactor());
+ }
+ if (coverage.getMaxWaitAfterCoverageFactor() != null) {
+ datasetBuilder.higher_coverage_maxsearchwait(coverage.getMaxWaitAfterCoverageFactor());
+ }
+ }
+
+ Tuning tuning = dispatchGroup.getTuning();
+ boolean useLocalNode = false;
+ if (tuning != null && tuning.dispatch != null) {
+ useLocalNode = tuning.dispatch.useLocalNode;
+ }
+ final List<PartitionsConfig.Dataset.Engine.Builder> allEngines = new ArrayList<>();
+ for (SearchInterface searchNode : dispatchGroup.getSearchersIterable()) {
+ final PartitionsConfig.Dataset.Engine.Builder engineBuilder = new PartitionsConfig.Dataset.Engine.Builder().
+ name_and_port(searchNode.getDispatcherConnectSpec()).
+ rowid(searchNode.getNodeSpec().rowId()).
+ partid(searchNode.getNodeSpec().partitionId());
+ allEngines.add(engineBuilder);
+ if (preferLocalRow) {
+ if (getHostname().equals(searchNode.getHostName())) {
+ engineBuilder.refcost(1);
+ } else {
+ engineBuilder.refcost(Integer.MAX_VALUE);
+ }
+ }
+
+ if (!useLocalNode || getHostname().equals(searchNode.getHostName())) {
+ if (useLocalNode) {
+ engineBuilder.rowid(0);
+ }
+ datasetBuilder.engine.add(engineBuilder);
+
+ }
+ }
+ //Do not create empty engine list for a dataset if no local search nodes found
+ if(datasetBuilder.engine.isEmpty() && useLocalNode) {
+ for(PartitionsConfig.Dataset.Engine.Builder engineBuilder: allEngines) {
+ datasetBuilder.engine.add(engineBuilder);
+ }
+ }
+
+ builder.dataset.add(datasetBuilder);
+
+ if (tuning != null) {
+ tuning.getConfig(builder);
+ scaleMaxHitsPerPartitions(builder, tuning);
+ }
+ }
+
+ private int getNumLeafNodesInGroup() {
+ int numSearchers = 0;
+ for (SearchInterface search : dispatchGroup.getSearchersIterable()) {
+ if (search instanceof Dispatch) {
+ numSearchers += ((Dispatch) search).getNumLeafNodesInGroup();
+ } else {
+ numSearchers++;
+ }
+ }
+ if (numSearchers > 0) {
+ // Divide by number of partitions, otherwise we would count the same leaf node partition number of times.
+ return numSearchers / dispatchGroup.getNumPartitions();
+ }
+ return 0;
+ }
+
+ private void scaleMaxHitsPerPartitions(PartitionsConfig.Builder builder, Tuning tuning) {
+ if (tuning == null || tuning.dispatch == null || tuning.dispatch.maxHitsPerPartition == null) {
+ return;
+ }
+ int numLeafNodes = getNumLeafNodesInGroup();
+ for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) {
+ dataset.maxhitspernode(tuning.dispatch.maxHitsPerPartition * numLeafNodes);
+ }
+ }
+
+ /**
+ * @return the number of ports needed
+ */
+ public int getPortCount() { return 3; }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java
new file mode 100644
index 00000000000..e9d01f50fb6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.vespa.model.content.SearchCoverage;
+
+import java.util.*;
+
+/**
+ * Class representing a group of @link{SearchInterface} nodes and a set of @link{Dispatch} nodes.
+ *
+ * Each @link{Dispatch} has a reference to an instance of this class and use it when producing config.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class DispatchGroup {
+
+ private final List<Dispatch> dispatchers = new ArrayList<>();
+ private final Map<Integer, Map<Integer, SearchInterface> > searchers = new TreeMap<>();
+
+ final private IndexedSearchCluster sc;
+
+ public DispatchGroup(IndexedSearchCluster sc) {
+ this.sc = sc;
+ }
+
+ DispatchGroup addDispatcher(Dispatch dispatch) {
+ dispatchers.add(dispatch);
+ return this;
+ }
+
+ DispatchGroup addSearcher(SearchInterface search) {
+ Map<Integer, SearchInterface> rows = searchers.get(search.getNodeSpec().partitionId());
+ if (rows == null) {
+ rows = new TreeMap<>();
+ rows.put(search.getNodeSpec().rowId(), search);
+ searchers.put(search.getNodeSpec().partitionId(), rows);
+ } else {
+ if (rows.containsKey(search.getNodeSpec().rowId())) {
+ throw new IllegalArgumentException("Already contains a search node with row id '" + search.getNodeSpec().rowId() + "'");
+ }
+ rows.put(search.getNodeSpec().rowId(), search);
+ }
+ return this;
+ }
+
+ DispatchGroup clearSearchers() {
+ searchers.clear();
+ return this;
+ }
+
+ List<Dispatch> getDispatchers() {
+ return Collections.unmodifiableList(dispatchers);
+ }
+
+ public Iterable getSearchersIterable() {
+ return new Iterable(searchers);
+ }
+
+ public int getRowBits() {
+ return sc.getRowBits();
+ }
+
+ public int getNumPartitions() {
+ return searchers.size();
+ }
+
+ public boolean useFixedRowInDispatch() {
+ return sc.useFixedRowInDispatch();
+ }
+
+ public int getMinNodesPerColumn() {
+ return sc.getMinNodesPerColumn();
+ }
+
+ public int getMaxNodesDownPerFixedRow() {
+ return sc.getMaxNodesDownPerFixedRow();
+ }
+
+ SearchCoverage getSearchCoverage() {
+ return sc.getSearchCoverage();
+ }
+
+ Tuning getTuning() {
+ return sc.getTuning();
+ }
+
+ String getClusterName() {
+ return sc.getClusterName();
+ }
+
+ static class Iterator implements java.util.Iterator<SearchInterface> {
+ private java.util.Iterator<Map<Integer, SearchInterface>> it1;
+ private java.util.Iterator<SearchInterface> it2;
+ Iterator(Map<Integer, Map<Integer, SearchInterface> > s) {
+ it1 = s.values().iterator();
+ if (it1.hasNext()) {
+ it2 = it1.next().values().iterator();
+ }
+ }
+ @Override
+ public boolean hasNext() {
+ if (it2 == null) {
+ return false;
+ }
+ while (!it2.hasNext() && it1.hasNext()) {
+ it2 = it1.next().values().iterator();
+ }
+ return it2.hasNext();
+ }
+
+ @Override
+ public SearchInterface next() {
+ return it2.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new IllegalStateException("'remove' not implemented");
+ }
+ }
+
+ public static class Iterable implements java.lang.Iterable<SearchInterface> {
+ final Map<Integer, Map<Integer, SearchInterface> > searchers;
+ Iterable(Map<Integer, Map<Integer, SearchInterface> > searchers) { this.searchers = searchers; }
+ @Override
+ public java.util.Iterator<SearchInterface> iterator() {
+ return new Iterator(searchers);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java
new file mode 100644
index 00000000000..9b1f63e813e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.SimpleConfigProducer;
+import com.yahoo.vespa.model.content.DispatchSpec;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class used to build the mid-level dispatch groups in an indexed content cluster.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DispatchGroupBuilder {
+
+ private final SimpleConfigProducer dispatchParent;
+ private final DispatchGroup rootDispatch;
+ private final IndexedSearchCluster searchCluster;
+
+ public DispatchGroupBuilder(SimpleConfigProducer dispatchParent,
+ DispatchGroup rootDispatch,
+ IndexedSearchCluster searchCluster) {
+ this.dispatchParent = dispatchParent;
+ this.rootDispatch = rootDispatch;
+ this.searchCluster = searchCluster;
+ }
+
+ public void build(List<DispatchSpec.Group> groupsSpec,
+ List<SearchNode> searchNodes) {
+ Map<Integer, SearchNode> searchNodeMap = buildSearchNodeMap(searchNodes);
+ for (int partId = 0; partId < groupsSpec.size(); ++partId) {
+ DispatchSpec.Group groupSpec = groupsSpec.get(partId);
+ DispatchGroup group = new DispatchGroup(searchCluster);
+ populateDispatchGroup(group, groupSpec.getNodes(), searchNodeMap, partId);
+ }
+ }
+
+ private void populateDispatchGroup(DispatchGroup group,
+ List<DispatchSpec.Node> nodeList,
+ Map<Integer, SearchNode> searchNodesMap,
+ int partId) {
+ for (int rowId = 0; rowId < nodeList.size(); ++rowId) {
+ int distributionKey = nodeList.get(rowId).getDistributionKey();
+ SearchNode searchNode = searchNodesMap.get(distributionKey);
+ Dispatch dispatch = buildDispatch(group, new NodeSpec(rowId, partId), distributionKey, searchNode.getHostResource());
+ group.addDispatcher(dispatch);
+ rootDispatch.addSearcher(dispatch);
+
+ // Note: the rowId in this context will be the partId for the underlying search node.
+ group.addSearcher(buildSearchInterface(searchNode, rowId));
+ }
+ }
+
+ /**
+ * Builds a mid-level dispatcher with a configId containing the same stable distribution-key as the search node it
+ * is located on.
+ *
+ * If this.dispatchParent has subConfigId 'dispatchers', the config ids of the mid-level
+ * dispatchers are '../dispatchers/dispatch.X' where X is the distribution-key of the search node.
+ *
+ * The dispatch group that will contain this mid-level dispatcher is no longer part of the config producer tree,
+ * but only contains information about the dispatchers and searchers in this group.
+ */
+ private Dispatch buildDispatch(DispatchGroup group, NodeSpec nodeSpec, int distributionKey, HostResource hostResource) {
+ Dispatch dispatch = Dispatch.createDispatchWithStableConfigId(group, dispatchParent, nodeSpec, distributionKey, 1);
+ dispatch.setHostResource(hostResource);
+ dispatch.initService();
+ return dispatch;
+ }
+
+ private static SearchInterface buildSearchInterface(SearchNode searchNode, int partId) {
+ searchNode.updatePartition(partId); // ensure that search node uses the same partId as dispatch sees
+ return new SearchNodeWrapper(new NodeSpec(0, partId), searchNode);
+ }
+
+ private static Map<Integer, SearchNode> buildSearchNodeMap(List<SearchNode> searchNodes) {
+ Map<Integer, SearchNode> retval = new LinkedHashMap<>();
+ for (SearchNode node : searchNodes) {
+ retval.put(node.getDistributionKey(), node);
+ }
+ return retval;
+ }
+
+ public static List<DispatchSpec.Group> createDispatchGroups(List<SearchNode> searchNodes,
+ int numDispatchGroups) {
+ if (numDispatchGroups > searchNodes.size())
+ numDispatchGroups = searchNodes.size();
+
+ List<DispatchSpec.Group> groupsSpec = new ArrayList<>();
+ int numNodesPerGroup = searchNodes.size() / numDispatchGroups;
+ if (searchNodes.size() % numDispatchGroups != 0) {
+ numNodesPerGroup += 1;
+ }
+ int searchNodeIdx = 0;
+ for (int i = 0; i < numDispatchGroups; ++i) {
+ DispatchSpec.Group groupSpec = new DispatchSpec.Group();
+ for (int j = 0; j < numNodesPerGroup && searchNodeIdx < searchNodes.size(); ++j) {
+ groupSpec.addNode(new DispatchSpec.Node(searchNodes.get(searchNodeIdx++).getDistributionKey()));
+ }
+ groupsSpec.add(groupSpec);
+ }
+ assert(searchNodeIdx == searchNodes.size());
+ return groupsSpec;
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java
new file mode 100644
index 00000000000..d44f81d571b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.config.search.IndexschemaConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.SummarymapConfig;
+import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+
+/**
+ * Represents a document database and the backend configuration needed for this database.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class DocumentDatabase extends AbstractConfigProducer implements
+ IndexInfoConfig.Producer,
+ IlscriptsConfig.Producer,
+ AttributesConfig.Producer,
+ RankProfilesConfig.Producer,
+ IndexschemaConfig.Producer,
+ JuniperrcConfig.Producer,
+ SummarymapConfig.Producer,
+ SummaryConfig.Producer {
+
+ private final String inputDocType;
+ private final DerivedConfiguration derivedCfg;
+
+ public DocumentDatabase(AbstractConfigProducer parent, String inputDocType, DerivedConfiguration derivedCfg) {
+ super(parent, inputDocType);
+ this.inputDocType = inputDocType;
+ this.derivedCfg = derivedCfg;
+ }
+
+ public String getName() {
+ return inputDocType;
+ }
+
+ public String getInputDocType() {
+ return inputDocType;
+ }
+
+ public DerivedConfiguration getDerivedConfiguration() {
+ return derivedCfg;
+ }
+
+ @Override
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ derivedCfg.getIndexInfo().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ derivedCfg.getIndexingScript().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ derivedCfg.getAttributeFields().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ derivedCfg.getRankProfileList().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(IndexschemaConfig.Builder builder) {
+ derivedCfg.getIndexSchema().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(JuniperrcConfig.Builder builder) {
+ derivedCfg.getJuniperrc().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(SummarymapConfig.Builder builder) {
+ derivedCfg.getSummaryMap().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(SummaryConfig.Builder builder) {
+ derivedCfg.getSummaries().getConfig(builder);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java
new file mode 100644
index 00000000000..942b389fce9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.document.select.*;
+import com.yahoo.document.select.convert.SelectionExpressionConverter;
+import com.yahoo.document.select.parser.ParseException;
+import java.util.Map;
+
+
+/**
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class DocumentSelectionConverter {
+
+ private final DocumentSelector selector;
+ private final Map<String, String> queryExpressionMap;
+
+ public DocumentSelectionConverter(String selection) throws ParseException, UnsupportedOperationException, IllegalArgumentException {
+ this.selector = new DocumentSelector(selection);
+ NowCheckVisitor nowChecker = new NowCheckVisitor();
+ selector.visit(nowChecker);
+ if (nowChecker.requiresConversion()) {
+ SelectionExpressionConverter converter = new SelectionExpressionConverter();
+ selector.visit(converter);
+ this.queryExpressionMap = converter.getQueryMap();
+ } else {
+ this.queryExpressionMap = null;
+ }
+ }
+
+ /**
+ * Transforms the selection into a search query.
+ * @return A search query representing the selection.
+ */
+ public String getQuery(String documentType) {
+ if (queryExpressionMap == null)
+ return null;
+ if (!queryExpressionMap.containsKey(documentType))
+ return null;
+ return queryExpressionMap.get(documentType);
+ }
+
+ /**
+ * Transforms the selection into an inverted search query.
+ * @return A search query representing the selection.
+ */
+ public String getInvertedQuery(String documentType) {
+ String query = getQuery(documentType);
+ if (query == null)
+ return null;
+ return query.replaceAll(">", "<");
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java
new file mode 100644
index 00000000000..88ebef9da86
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+
+/**
+ * @author balder
+ */
+public class IndexedElasticSearchCluster extends IndexedSearchCluster {
+
+ public IndexedElasticSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
+ super(parent, clusterName, index);
+ }
+
+ @Override
+ public boolean getAllowFeedingWhenNodesDown() {
+ return true;
+ }
+
+ @Override
+ public int getMinNodesPerColumn() { return 0; }
+
+ @Override
+ protected void assureSdConsistent() { }
+
+ @Override
+ public int getRowBits() { return 8; }
+
+ @Override
+ boolean useFixedRowInDispatch() {
+ for (SearchNode node : getSearchNodes()) {
+ if (node.getNodeSpec().rowId() > 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isElastic() { return true; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
new file mode 100644
index 00000000000..aa664264ba6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -0,0 +1,390 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.searchdefinition.UnproperSearch;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.SimpleConfigProducer;
+import com.yahoo.vespa.model.container.Container;
+import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+import com.yahoo.vespa.model.content.DispatchSpec;
+import com.yahoo.vespa.model.content.SearchCoverage;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public abstract class IndexedSearchCluster extends SearchCluster
+ implements
+ DocumentdbInfoConfig.Producer,
+ // TODO consider removing, these only produced by UnionConfiguration and DocumentDatabase?
+ IndexInfoConfig.Producer,
+ IlscriptsConfig.Producer,
+ DispatchConfig.Producer
+{
+
+ /**
+ * Class used to retrieve combined configuration from multiple document databases.
+ * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs,
+ * that is handled (by delegating to this) by the {@link IndexedSearchCluster}
+ * which is the parent to this. This avoids building the config multiple times.
+ */
+ public static class UnionConfiguration
+ extends AbstractConfigProducer
+ implements AttributesConfig.Producer {
+ private final List<DocumentDatabase> docDbs;
+
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ for (DocumentDatabase docDb : docDbs) {
+ docDb.getConfig(builder);
+ }
+ }
+
+ public UnionConfiguration(AbstractConfigProducer parent, List<DocumentDatabase> docDbs) {
+ super(parent, "union");
+ this.docDbs = docDbs;
+ }
+ }
+
+ private static final Logger log = Logger.getLogger(IndexedSearchCluster.class.getName());
+
+ private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config.
+ private String indexingChainName = null;
+
+ private DocprocChain indexingChain; // The actual docproc chain indexing for this.
+
+ private Tuning tuning;
+ private SearchCoverage searchCoverage;
+
+ // This is the document selector string as derived from the subscription tag.
+ private String routingSelector = null;
+ private DocumentSelectionConverter selectionConverter = null;
+ private List<DocumentDatabase> documentDbs = new LinkedList<>();
+ private final UnionConfiguration unionCfg;
+ private int maxNodesDownPerFixedRow = 0;
+
+ private final SimpleConfigProducer dispatchParent;
+ private final DispatchGroup rootDispatch;
+ private DispatchSpec dispatchSpec;
+ private List<SearchNode> searchNodes = new ArrayList<>();
+
+ /**
+ * Returns the document selector that is able to resolve what documents are to be routed to this search cluster.
+ * This string uses the document selector language as defined in the "document" module.
+ *
+ * @return The document selector.
+ */
+ public String getRoutingSelector() {
+ return routingSelector;
+ }
+
+ public IndexedSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
+ super(parent, clusterName, index);
+ unionCfg = new UnionConfiguration(this, documentDbs);
+ dispatchParent = new SimpleConfigProducer(this, "dispatchers");
+ rootDispatch = new DispatchGroup(this);
+ }
+
+ @Override
+ protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; }
+
+ public final boolean hasExplicitIndexingCluster() {
+ return indexingClusterName != null;
+ }
+
+ public final boolean hasExplicitIndexingChain() {
+ return indexingChainName != null;
+ }
+
+ /**
+ * Returns the name of the docproc cluster running indexing for this search cluster. This is derived from the
+ * services file on initialization, this can NOT be used at runtime to determine indexing chain. When initialization
+ * is done, the {@link #getIndexingServiceName()} method holds the actual indexing docproc chain object.
+ *
+ * @return The name of the docproc cluster associated with this.
+ */
+ public String getIndexingClusterName() {
+ return hasExplicitIndexingCluster() ? indexingClusterName : getClusterName() + ".indexing";
+ }
+
+ public String getIndexingChainName() {
+ return indexingChainName;
+ }
+
+ public void setIndexingChainName(String indexingChainName) {
+ this.indexingChainName = indexingChainName;
+ }
+
+ /**
+ * Sets the name of the docproc cluster running indexing for this search cluster. This is for initial configuration,
+ * and will not reflect the actual indexing chain. See {@link #getIndexingClusterName} for more detail.
+ *
+ * @param name The name of the docproc cluster associated with this.
+ */
+ public void setIndexingClusterName(String name) {
+ indexingClusterName = name;
+ }
+
+ public String getIndexingServiceName() {
+ return indexingChain.getServiceName();
+ }
+
+ /**
+ * Sets the docproc chain that will be running indexing for this search cluster. This is set by the
+ * {@link com.yahoo.vespa.model.content.Content} model during build.
+ *
+ * @param chain the chain that is to run indexing for this cluster.
+ * @return this, to allow chaining.
+ */
+ public AbstractSearchCluster setIndexingChain(DocprocChain chain) {
+ indexingChain = chain;
+ return this;
+ }
+
+ public Dispatch addTld(AbstractConfigProducer tldParent, HostResource hostResource) {
+ int index = rootDispatch.getDispatchers().size();
+ Dispatch tld = Dispatch.createTld(rootDispatch, tldParent, index);
+ tld.setHostResource(hostResource);
+ tld.initService();
+ rootDispatch.addDispatcher(tld);
+ return tld;
+ }
+
+ /**
+ * Make sure to allocate tld with same id as container (i.e if container cluster name is 'foo', with containers
+ * with index 0,1,2 the tlds created will get names ../foo.0.tld.0, ../foo.1.tld.1, ../foo.2.tld.2, so that tld config id is
+ * stable no matter what changes are done to the number of containers in a container cluster
+ * @param tldParent the indexed search cluster the tlds to add should be connected to
+ * @param containerCluster the container cluster that should use the tlds created for searching the indexed search cluster above
+ */
+ public void addTldsWithSameIdsAsContainers(AbstractConfigProducer tldParent, ContainerCluster containerCluster) {
+ for (Container container : containerCluster.getContainers()) {
+ final String containerSubId = container.getSubId();
+ if (!containerSubId.contains(".")) {
+ throw new RuntimeException("Expected container sub id to be of the form string.number");
+ }
+ int containerIndex = Integer.parseInt(containerSubId.split("\\.")[1]);
+ final String containerClusterName = containerCluster.getName();
+ log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() +
+ ", container cluster " + containerClusterName + " (container id " + containerSubId + ") on host " + container.getHostResource().getHostName());
+ rootDispatch.addDispatcher(createTld(tldParent, container.getHostResource(), containerClusterName, containerIndex));
+ }
+ }
+
+ public Dispatch createTld(AbstractConfigProducer tldParent, HostResource hostResource, String containerClusterName, int containerIndex) {
+ Dispatch tld = Dispatch.createTldWithContainerIdInName(rootDispatch, tldParent, containerClusterName, containerIndex);
+ tld.setHostResource(hostResource);
+ tld.initService();
+ return tld;
+ }
+
+ public DispatchGroup getRootDispatch() { return rootDispatch; }
+
+ public void addSearcher(SearchNode searcher) {
+ searchNodes.add(searcher);
+ rootDispatch.addSearcher(searcher);
+ }
+
+ public List<Dispatch> getTLDs() { return rootDispatch.getDispatchers(); }
+
+ public List<SearchNode> getSearchNodes() { return Collections.unmodifiableList(searchNodes); }
+ public int getSearchNodeCount() { return searchNodes.size(); }
+ public SearchNode getSearchNode(int index) { return searchNodes.get(index); }
+ public void setTuning(Tuning tuning) {
+ this.tuning = tuning;
+ }
+ public Tuning getTuning() { return tuning; }
+
+ public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) {
+ for (DocumentDatabase sdoc : documentDbs) {
+ if (sdoc.getName().equals(documentType)) {
+ fillDocumentDBConfig(sdoc, builder);
+ return;
+ }
+ }
+ }
+
+ protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
+ ddbB.inputdoctypename(sdoc.getInputDocType())
+ .configid(sdoc.getConfigId())
+ .visibilitydelay(getVisibilityDelay());
+ }
+
+ @Override
+ public void getConfig(DocumentdbInfoConfig.Builder builder) {
+ for (DocumentDatabase db : documentDbs) {
+ DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder();
+ docDb.name(db.getName());
+ convertSummaryConfig(db, db, docDb);
+ RankProfilesConfig.Builder rpb = new RankProfilesConfig.Builder();
+ db.getConfig(rpb);
+ addRankProfilesConfig(docDb, new RankProfilesConfig(rpb));
+ builder.documentdb(docDb);
+ }
+ }
+
+ public void setRoutingSelector(String sel) {
+ this.routingSelector=sel;
+ if (this.routingSelector != null) {
+ try {
+ this.selectionConverter = new DocumentSelectionConverter(this.routingSelector);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid routing selector: " + e.getMessage());
+ }
+ }
+ }
+ /**
+ * Create default config if not specified by user.
+ * Accept empty strings as user config - it means that all feeds/documents are accepted.
+ */
+ public void defaultDocumentsConfig() {
+ if ((routingSelector == null) && !getDocumentNames().isEmpty()) {
+ Iterator<String> it = getDocumentNames().iterator();
+ routingSelector = it.next();
+ while (it.hasNext()) {
+ routingSelector += " or " + it.next();
+ }
+ }
+ }
+ protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches,
+ List<com.yahoo.searchdefinition.Search> globalSearches) {
+ for (SearchDefinitionSpec spec : localSearches) {
+ com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch();
+ if (!(search instanceof UnproperSearch)) {
+ DocumentDatabase db = new DocumentDatabase(this, search.getName(), new DerivedConfiguration(search, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry()));
+ // TODO: remove explicit adding of user configs when the complete content model is built using builders.
+ db.mergeUserConfigs(spec.getUserConfigs());
+ documentDbs.add(db);
+ }
+ }
+ }
+
+ public List<DocumentDatabase> getDocumentDbs() {
+ return documentDbs;
+ }
+
+ public void setSearchCoverage(SearchCoverage searchCoverage) {
+ this.searchCoverage = searchCoverage;
+ }
+
+ public SearchCoverage getSearchCoverage() {
+ return searchCoverage;
+ }
+
+ @Override
+ public DerivedConfiguration getSdConfig() { return null; }
+
+ @Override
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ unionCfg.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ unionCfg.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ unionCfg.getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ unionCfg.getConfig(builder);
+ }
+
+ @Override
+ protected void exportSdFiles(File toDir) throws IOException { }
+
+ public abstract boolean getAllowFeedingWhenNodesDown();
+
+ public abstract int getMinNodesPerColumn();
+
+ boolean useFixedRowInDispatch() {
+ return false;
+ }
+
+ public abstract boolean isElastic();
+
+ int getMaxNodesDownPerFixedRow() {
+ return maxNodesDownPerFixedRow;
+ }
+
+ public void setMaxNodesDownPerFixedRow(int value) {
+ maxNodesDownPerFixedRow = value;
+ }
+
+ public void setDispatchSpec(DispatchSpec dispatchSpec) {
+ if (dispatchSpec.getNumDispatchGroups() != null) {
+ this.dispatchSpec = new DispatchSpec.Builder().setGroups
+ (DispatchGroupBuilder.createDispatchGroups(getSearchNodes(),
+ dispatchSpec.getNumDispatchGroups())).build();
+ } else {
+ this.dispatchSpec = dispatchSpec;
+ }
+ }
+
+ public DispatchSpec getDispatchSpec() {
+ return dispatchSpec;
+ }
+
+ public boolean useMultilevelDispatchSetup() {
+ return dispatchSpec != null && dispatchSpec.getGroups() != null && !dispatchSpec.getGroups().isEmpty();
+ }
+
+ public void setupDispatchGroups() {
+ if (!useMultilevelDispatchSetup()) {
+ return;
+ }
+ rootDispatch.clearSearchers();
+ new DispatchGroupBuilder(dispatchParent, rootDispatch, this).build(dispatchSpec.getGroups(), getSearchNodes());
+ }
+
+ @Override
+ public void getConfig(DispatchConfig.Builder builder) {
+ for (SearchNode node : getSearchNodes()) {
+ DispatchConfig.Node.Builder nodeBuilder = new DispatchConfig.Node.Builder();
+ nodeBuilder.key(node.getDistributionKey());
+ nodeBuilder.host(node.getHostName());
+ nodeBuilder.port(node.getRpcPort());
+ builder.node(nodeBuilder);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java
new file mode 100644
index 00000000000..c6fb6ef4b71
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.component.chain.model.ChainSpecification;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import com.yahoo.vespa.model.container.docproc.DocprocChain;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexingDocprocChain extends DocprocChain implements SpecialtokensConfig.Producer {
+
+ public static final String NAME = "indexing";
+ private static final List<Phase> phases = new ArrayList<>(2);
+
+ static {
+ phases.add(new Phase("indexingStart", Collections.<String>emptySet(), Collections.<String>emptySet()));
+ phases.add(new Phase("indexingEnd", Collections.<String>emptySet(), Collections.<String>emptySet()));
+ }
+
+ public IndexingDocprocChain() {
+ super(new ChainSpecification(new ComponentId(NAME),
+ new ChainSpecification.Inheritance(Collections.<ComponentSpecification>emptySet(),
+ Collections.<ComponentSpecification>emptySet()),
+ phases,
+ Collections.<ComponentSpecification>emptySet()),
+ new HashMap<>());
+ addInnerComponent(new IndexingProcessor());
+ }
+
+ @Override
+ public void getConfig(SpecialtokensConfig.Builder builder) {
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java
new file mode 100644
index 00000000000..89f2ebbe5eb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.vespa.model.container.docproc.DocumentProcessor;
+import com.yahoo.vespa.model.container.docproc.model.DocumentProcessorModel;
+
+import java.util.Collections;
+import java.util.HashMap;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexingProcessor extends DocumentProcessor {
+
+ public static final String docprocsBundleSpecification = "docprocs";
+
+ public IndexingProcessor() {
+ super(new DocumentProcessorModel(new BundleInstantiationSpecification(new ComponentId(DocumentProcessor.INDEXER),
+ new ComponentSpecification(DocumentProcessor.INDEXER),
+ new ComponentSpecification(docprocsBundleSpecification)),
+ new Dependencies(Collections.<String>emptyList(),
+ Collections.<String>emptyList(),
+ Collections.<String>emptyList()),
+ new HashMap<>()));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java
new file mode 100644
index 00000000000..a068d173ad8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.vespa.model.content.DispatchSpec;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Class used to validate that multilevel dispatch is correctly setup in an indexed content cluster.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class MultilevelDispatchValidator {
+
+ private final String clusterName;
+ private final DispatchSpec dispatchSpec;
+ private final List<SearchNode> searchNodes;
+
+ public MultilevelDispatchValidator(String clusterName,
+ DispatchSpec dispatchSpec,
+ List<SearchNode> searchNodes) {
+ this.clusterName = clusterName;
+ this.dispatchSpec = dispatchSpec;
+ this.searchNodes = searchNodes;
+ }
+
+ public void validate() {
+ validateThatWeReferenceNodesOnlyOnce();
+ validateThatWeReferenceAllNodes();
+ validateThatWeUseValidNodeReferences();
+ }
+
+ private void validateThatWeReferenceNodesOnlyOnce() {
+ Set<Integer> distKeys = new HashSet<>();
+ for (DispatchSpec.Group group : dispatchSpec.getGroups()) {
+ for (DispatchSpec.Node node : group.getNodes()) {
+ int distKey = node.getDistributionKey();
+ if (distKeys.contains(distKey)) {
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected nodes to be referenced only once in dispatch groups, but node with distribution key '" + distKey + "' is referenced multiple times.");
+ }
+ distKeys.add(distKey);
+ }
+ }
+ }
+
+ private void validateThatWeReferenceAllNodes() {
+ Set<Integer> distKeys = createDistributionKeysSet();
+ for (DispatchSpec.Group group : dispatchSpec.getGroups()) {
+ for (DispatchSpec.Node node : group.getNodes()) {
+ distKeys.remove(node.getDistributionKey());
+ }
+ }
+ if (!distKeys.isEmpty()) {
+ Object[] sorted = distKeys.toArray();
+ Arrays.sort(sorted);
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected all nodes to be referenced in dispatch groups, but " + distKeys.size() +
+ " node(s) with distribution keys " + Arrays.toString(sorted) + " are not referenced.");
+ }
+ }
+
+ private void validateThatWeUseValidNodeReferences() {
+ Set<Integer> distKeys = createDistributionKeysSet();
+ for (DispatchSpec.Group group : dispatchSpec.getGroups()) {
+ for (DispatchSpec.Node node : group.getNodes()) {
+ int distKey = node.getDistributionKey();
+ if (!distKeys.contains(distKey)) {
+ throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected all node references in dispatch groups to reference existing nodes, " +
+ "but node with distribution key '" + distKey + "' does not exists.");
+ }
+ }
+ }
+ }
+
+ private Set<Integer> createDistributionKeysSet() {
+ Set<Integer> distKeys = new HashSet<>();
+ for (SearchNode node : searchNodes) {
+ distKeys.add(node.getDistributionKey());
+ }
+ return distKeys;
+ }
+
+ private String getErrorMsgPrefix() {
+ return "In indexed content cluster '" + clusterName + "': ";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java
new file mode 100644
index 00000000000..e5759d8c2e5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+/**
+ * Represents the row id and partition id of a search interface node.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NodeSpec {
+
+ private final int rowId;
+ private final int partitionId;
+
+ public NodeSpec(int rowId, int partitionId) {
+ if (rowId < 0) {
+ throw new IllegalArgumentException("RowId(" + rowId + ") can not be below 0");
+ }
+ if (partitionId < 0) {
+ throw new IllegalArgumentException("PartId(" + partitionId + ") can not be below 0");
+ }
+ this.rowId = rowId;
+ this.partitionId = partitionId;
+ }
+
+ public int rowId() {
+ return rowId;
+ }
+
+ public int partitionId() {
+ return partitionId;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
new file mode 100644
index 00000000000..c88b40c8c3d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.searchdefinition.derived.RawRankProfile;
+import com.yahoo.searchdefinition.derived.SummaryMap;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.vespa.config.search.SummarymapConfig;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Represents a search cluster.
+ *
+ * @author arnej27959
+ */
+public abstract class SearchCluster extends AbstractSearchCluster
+ implements
+ DocumentdbInfoConfig.Producer,
+ IndexInfoConfig.Producer,
+ IlscriptsConfig.Producer {
+
+ private static final long serialVersionUID = 1L;
+
+ protected SearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
+ super(parent, clusterName, index);
+ }
+
+ public void writeFiles(File directory) throws java.io.IOException {
+ if (!directory.isDirectory() && !directory.mkdirs()) {
+ throw new java.io.IOException("Cannot create directory: "+ directory);
+ }
+ writeSdFiles(directory);
+ super.writeFiles(directory);
+ }
+
+ /**
+ * Must be called after cluster is built, to derive SD configs
+ * Derives the search definitions from the application package..
+ * Also stores the document names contained in the search
+ * definitions.
+ */
+ public void deriveSearchDefinitions(List<com.yahoo.searchdefinition.Search> global) {
+ deriveAllSearchDefinitions(getLocalSDS(), global);
+ }
+
+ @Override
+ public void getConfig(IndexInfoConfig.Builder builder) {
+ if (getSdConfig()!=null) getSdConfig().getIndexInfo().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(IlscriptsConfig.Builder builder) {
+ if (getSdConfig()!=null) getSdConfig().getIndexingScript().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ if (getSdConfig()!=null) getSdConfig().getAttributeFields().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) {
+ if (getSdConfig()!=null) getSdConfig().getRankProfileList().getConfig(builder);
+ }
+
+ /**
+ * Converst summary and summary map config to the appropriate information in documentdb
+ *
+ * @param summaryConfigProducer the summary config
+ * @param summarymapConfigProducer the summary map config, or null if none is available
+ * @param docDb the target document dm config
+ */
+ protected void convertSummaryConfig(SummaryConfig.Producer summaryConfigProducer,
+ SummarymapConfig.Producer summarymapConfigProducer,
+ DocumentdbInfoConfig.Documentdb.Builder docDb) {
+
+ SummaryConfig.Builder summaryConfigBuilder = new SummaryConfig.Builder();
+ summaryConfigProducer.getConfig(summaryConfigBuilder);
+ SummaryConfig summaryConfig = new SummaryConfig(summaryConfigBuilder);
+
+ SummarymapConfig summarymapConfig = null;
+ if (summarymapConfigProducer != null) {
+ SummarymapConfig.Builder summarymapConfigBuilder = new SummarymapConfig.Builder();
+ summarymapConfigProducer.getConfig(summarymapConfigBuilder);
+ summarymapConfig = new SummarymapConfig(summarymapConfigBuilder);
+ }
+
+ for (SummaryConfig.Classes sclass : summaryConfig.classes()) {
+ DocumentdbInfoConfig.Documentdb.Summaryclass.Builder sumClassBuilder = new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder();
+ sumClassBuilder.
+ id(sclass.id()).
+ name(sclass.name());
+ for (SummaryConfig.Classes.Fields field : sclass.fields()) {
+ DocumentdbInfoConfig.Documentdb.Summaryclass.Fields.Builder fieldsBuilder = new DocumentdbInfoConfig.Documentdb.Summaryclass.Fields.Builder();
+ fieldsBuilder.name(field.name())
+ .type(field.type())
+ .dynamic(isDynamic(field.name(), summarymapConfig));
+ sumClassBuilder.fields(fieldsBuilder);
+ }
+ docDb.summaryclass(sumClassBuilder);
+ }
+ }
+
+ /** Returns true if this is a dynamic summary field */
+ private boolean isDynamic(String fieldName, SummarymapConfig summarymapConfig) {
+ if (summarymapConfig == null) return false; // not know for streaming, but also not used
+
+ for (SummarymapConfig.Override override : summarymapConfig.override()) {
+ if ( ! fieldName.equals(override.field())) continue;
+ if (SummaryMap.isDynamicCommand(override.command())) return true;
+ }
+ return false;
+ }
+
+ protected void addRankProfilesConfig(DocumentdbInfoConfig.Documentdb.Builder docDbBuilder, RankProfilesConfig rankProfilesCfg) {
+ for (RankProfilesConfig.Rankprofile rankProfile : rankProfilesCfg.rankprofile()) {
+ DocumentdbInfoConfig.Documentdb.Rankprofile.Builder rpB = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder();
+ rpB.name(rankProfile.name());
+ rpB.hasSummaryFeatures(containsPropertiesWithPrefix(RawRankProfile.summaryFeatureFefPropertyPrefix, rankProfile.fef()));
+ rpB.hasRankFeatures(containsPropertiesWithPrefix(RawRankProfile.rankFeatureFefPropertyPrefix, rankProfile.fef()));
+ docDbBuilder.rankprofile(rpB);
+ }
+ }
+
+ private boolean containsPropertiesWithPrefix(String prefix, RankProfilesConfig.Rankprofile.Fef fef) {
+ for (RankProfilesConfig.Rankprofile.Fef.Property p : fef.property()) {
+ if (p.name().startsWith(prefix))
+ return true;
+ }
+ return false;
+ }
+
+ protected abstract void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches,
+ List<com.yahoo.searchdefinition.Search> globalSearches);
+
+ public abstract void defaultDocumentsConfig();
+ public abstract DerivedConfiguration getSdConfig();
+ protected abstract void exportSdFiles(File toDir) throws IOException;
+ protected final void writeSdFiles(File toDir) throws IOException {
+ assureSdConsistent();
+ exportSdFiles(toDir);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java
new file mode 100644
index 00000000000..adeee257317
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SearchColumn extends AbstractConfigProducer {
+
+ // All search nodes contained in this column, these also exist as child config producers.
+ private final List<SearchNode> nodes = new LinkedList<>();
+
+ public SearchColumn(SearchCluster parent, String name, int index) {
+ super(parent, name);
+ }
+
+ /** @return The number of rows in this column. */
+ public int getNumRows() { return nodes.size(); }
+
+ /** @return All search nodes contained in this column. */
+ public List<SearchNode> getSearchNodes() { return nodes; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java
new file mode 100644
index 00000000000..2904c561d71
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.searchdefinition.Search;
+
+import java.util.Collection;
+
+/**
+ * @author tonytv
+ */
+public class SearchDefinition {
+
+ private final Search search;
+ private final String name;
+
+ public static final String fileNameSuffix = ".sd";
+
+ public Search getSearch() {
+ return search;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public SearchDefinition(String name, Search search) {
+ this.name = name;
+ this.search = search;
+ }
+
+ //Find search definition from a collection with the name specified
+ public static SearchDefinition findByName(final String searchDefinitionName, Collection<SearchDefinition> searchDefinitions) {
+ for (SearchDefinition candidate : searchDefinitions) {
+ if (candidate.getName().equals(searchDefinitionName) )
+ return candidate;
+ }
+
+ return null;
+ }
+
+ // Used by admin interface
+ public String getFilename() {
+ return getName() + fileNameSuffix;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java
new file mode 100644
index 00000000000..5ac43b01a5f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
+import org.w3c.dom.Element;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Represents a single searchdefinition file.
+ *
+ * @author arnej27959
+ */
+public class SearchDefinitionXMLHandler implements Serializable {
+
+ private String sdName;
+
+ public SearchDefinitionXMLHandler(ModelElement elem) {
+ sdName = elem.getStringAttribute("name");
+ if (sdName == null) {
+ sdName = elem.getStringAttribute("type");
+ }
+ }
+
+ public String getName() { return sdName; }
+
+ public SearchDefinition getResponsibleSearchDefinition(List<SearchDefinition> searchDefinitions) {
+ return SearchDefinition.findByName( getName(), searchDefinitions );
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java
new file mode 100644
index 00000000000..bedc3684ea9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+/**
+ * This represents an interface for searching.
+ * It can be both a backend search node or a dispatcher.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public interface SearchInterface {
+
+ NodeSpec getNodeSpec();
+ String getDispatcherConnectSpec();
+ String getHostName();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
new file mode 100644
index 00000000000..9bc968bfd1a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -0,0 +1,288 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.metrics.MetricsmanagerConfig;
+import com.yahoo.searchlib.TranslogserverConfig;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.vespa.config.content.StorFilestorConfig;
+import com.yahoo.vespa.config.content.core.StorBucketmoverConfig;
+import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig;
+import com.yahoo.vespa.config.content.core.StorServerConfig;
+import com.yahoo.vespa.config.content.core.StorStatusConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.config.storage.StorDevicesConfig;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.admin.MonitoringSystem;
+import com.yahoo.vespa.model.application.validation.RestartConfigs;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import com.yahoo.vespa.model.content.ContentNode;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.Optional;
+
+/**
+ * Represents a search node (proton).
+ * <p>
+ * Due to the current disconnect between StorageNode and SearchNode, we have to
+ * duplicate the set of RestartConfigs classes from StorageNode here, as SearchNode
+ * runs in a content/storage node context without this being immediately obvious
+ * in the model.
+ *
+ * @author arnej27959
+ * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a>
+ */
+@RestartConfigs({ProtonConfig.class, MetricsmanagerConfig.class, TranslogserverConfig.class,
+ StorDevicesConfig.class, StorFilestorConfig.class, StorBucketmoverConfig.class,
+ StorCommunicationmanagerConfig.class, StorStatusConfig.class,
+ StorServerConfig.class, LoadTypeConfig.class})
+public class SearchNode extends AbstractService implements
+ SearchInterface,
+ ProtonConfig.Producer,
+ MetricsmanagerConfig.Producer,
+ TranslogserverConfig.Producer {
+
+ private static final long serialVersionUID = 1L;
+ private final boolean flushOnShutdown;
+ private NodeSpec nodeSpec;
+ private int distributionKey;
+ private final String clusterName;
+ private TransactionLogServer tls;
+ private AbstractService serviceLayerService;
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> {
+
+ private final String name;
+ private final NodeSpec nodeSpec;
+ private final String clusterName;
+ private final ContentNode contentNode;
+ private final boolean flushOnShutdown;
+ public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown) {
+ this.name = name;
+ this.nodeSpec = nodeSpec;
+ this.clusterName = clusterName;
+ this.contentNode = node;
+ this.flushOnShutdown = flushOnShutdown;
+ }
+
+ @Override
+ protected SearchNode doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown);
+ }
+ }
+
+ /**
+ * Creates a SearchNode in elastic mode.
+ */
+ public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec,
+ String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown) {
+ return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown);
+ }
+
+ private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec,
+ String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown) {
+ this(parent, name, nodeSpec, clusterName, flushOnShutdown);
+ this.distributionKey = distributionKey;
+ this.serviceLayerService = serviceLayerService;
+ setPropertiesElastic(clusterName, distributionKey);
+ }
+
+ private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, boolean flushOnShutdown) {
+ super(parent, name);
+ this.nodeSpec = nodeSpec;
+ this.clusterName = clusterName;
+ this.flushOnShutdown = flushOnShutdown;
+ portsMeta.on(0).tag("rpc").tag("rtc").tag("admin").tag("status");
+ portsMeta.on(1).tag("fs4");
+ portsMeta.on(2).tag("srmp").tag("hack").tag("test");
+ portsMeta.on(3).tag("rpc").tag("engines-provider");
+ portsMeta.on(4).tag("http").tag("json").tag("health").tag("state");
+ // Properties are set in DomSearchBuilder
+ monitorService();
+ }
+
+ private void setPropertiesElastic(String clusterName, int distributionKey) {
+ setProp("index", distributionKey).
+ setProp("clustertype", "search").
+ setProp("clustername", clusterName);
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ private String getClusterConfigId() {
+ return getParent().getConfigId();
+ }
+
+ private String getBaseDir() {
+ return Defaults.getDefaults().vespaHome() + "var/db/vespa/search/cluster." + getClusterName() + "/n" + distributionKey;
+ }
+
+ public void updatePartition(int partitionId) {
+ nodeSpec = new NodeSpec(nodeSpec.rowId(), partitionId);
+ }
+
+ @Override
+ public NodeSpec getNodeSpec() {
+ return nodeSpec;
+ }
+
+ /**
+ * Returns the connection spec string that resolves to this search node.
+ *
+ * @return The connection string.
+ */
+ public String getConnectSpec() {
+ return "tcp/" + getHost().getHostName() + ":" + getRpcPort();
+ }
+
+ /**
+ * Returns the number of ports needed by this service.
+ *
+ * @return The number of ports.
+ */
+ @Override
+ public int getPortCount() {
+ return 5;
+ }
+
+ /**
+ * Returns the RPC port used by this searchnode.
+ *
+ * @return The port.
+ */
+ public int getRpcPort() {
+ return getRelativePort(0);
+ }
+
+ protected int getSlimeMessagingPort() {
+ return getRelativePort(2);
+ }
+
+ /*
+ * Returns the rpc port used for the engines provider interface.
+ * @return The port
+ */
+
+ public int getPersistenceProviderRpcPort() {
+ return getRelativePort(3);
+ }
+
+ @Override
+ public int getHealthPort() {
+ return getHttpPort();
+ }
+
+ @Override
+ public String getServiceType() {
+ return "searchnode";
+ }
+
+ public int getDistributionKey() {
+ return distributionKey;
+ }
+
+ /**
+ * Returns the connection spec string that resolves to the dispatcher service
+ * on this node.
+ *
+ * @return The connection string.
+ */
+ public String getDispatcherConnectSpec() {
+ return "tcp/" + getHost().getHostName() + ":" + getDispatchPort();
+ }
+
+ public int getDispatchPort() {
+ return getRelativePort(1);
+ }
+
+ public int getHttpPort() {
+ return getRelativePort(4);
+ }
+
+ @Override
+ public void getConfig(TranslogserverConfig.Builder builder) {
+ tls.getConfig(builder);
+ }
+
+ @Override
+ public String toString() {
+ return getHostName();
+ }
+
+ public TransactionLogServer getTransactionLogServer() {
+ return tls;
+ }
+
+ public void setTls(TransactionLogServer tls) {
+ this.tls = tls;
+ }
+
+ public AbstractService getServiceLayerService() {
+ return serviceLayerService;
+ }
+
+ @Override
+ public String getStartupCommand() {
+ String startup = getMMapNoCoreEnvVariable() + "exec $ROOT/sbin/proton " + "--identity " + getConfigId();
+ if (serviceLayerService != null) {
+ startup = startup + " --serviceidentity " + serviceLayerService.getConfigId();
+ }
+ return startup;
+ }
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ builder.
+ ptport(getDispatchPort()).
+ rpcport(getRpcPort()).
+ slime_messaging_port(getSlimeMessagingPort()).
+ rtcspec(getConnectSpec()).
+ httpport(getHttpPort()).
+ partition(getNodeSpec().partitionId()).
+ persistenceprovider(new ProtonConfig.Persistenceprovider.Builder().port(getPersistenceProviderRpcPort())).
+ clustername(getClusterName()).
+ basedir(getBaseDir()).
+ tlsspec("tcp/" + getHost().getHostName() + ":" + getTransactionLogServer().getTlsPort()).
+ tlsconfigid(getConfigId()).
+ slobrokconfigid(getClusterConfigId()).
+ routingconfigid(getClusterConfigId()).
+ distributionkey(getDistributionKey());
+ if (isHostedVespa()) {
+ // 4 days, 1 hour, 1 minute due to failed nodes can be in failed for 4 days and we want at least one hour more
+ // to make sure the node failer has done its work
+ builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60);
+ }
+ }
+
+ @Override
+ public HashMap<String, String> getDefaultMetricDimensions() {
+ HashMap<String, String> dimensions = new HashMap<>();
+ if (clusterName != null) {
+ dimensions.put("clustername", clusterName);
+ }
+ return dimensions;
+ }
+
+ @Override
+ public void getConfig(MetricsmanagerConfig.Builder builder) {
+ MonitoringSystem point = getMonitoringService();
+ if (point != null) {
+ builder.snapshot(new MetricsmanagerConfig.Snapshot.Builder().
+ periods(point.getIntervalSeconds()).periods(300));
+ }
+ builder.consumer(
+ new MetricsmanagerConfig.Consumer.Builder().
+ name("log").
+ tags("logdefault"));
+ }
+
+ @Override
+ public Optional<String> getPreShutdownCommand() {
+ return Optional.ofNullable(flushOnShutdown ? Defaults.getDefaults().vespaHome() + "bin/vespa-proton-cmd " + getRpcPort() + " prepareRestart" : null);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java
new file mode 100644
index 00000000000..caa918aab97
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+public class SearchNodeWrapper implements SearchInterface {
+
+ private final NodeSpec nodeSpec;
+ private final SearchNode node;
+
+ public SearchNodeWrapper(NodeSpec nodeSpec, SearchNode node) {
+ this.nodeSpec = nodeSpec;
+ this.node = node;
+ }
+
+ @Override
+ public NodeSpec getNodeSpec() {
+ return nodeSpec;
+ }
+
+ @Override
+ public String getDispatcherConnectSpec() {
+ return node.getDispatcherConnectSpec();
+ }
+
+ @Override
+ public String getHostName() {
+ return node.getHostName();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
new file mode 100644
index 00000000000..2d89eb59b58
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.config.search.SummarymapConfig;
+import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
+import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * A search cluster of type streaming.
+ *
+ * @author balder
+ * @author vegardh
+ */
+public class StreamingSearchCluster extends SearchCluster implements
+ DocumentdbInfoConfig.Producer,
+ RankProfilesConfig.Producer,
+ VsmsummaryConfig.Producer,
+ VsmfieldsConfig.Producer,
+ SummarymapConfig.Producer,
+ SummaryConfig.Producer
+{
+
+ private final String storageRouteSpec;
+ private DerivedConfiguration sdConfig = null;
+
+ public StreamingSearchCluster(AbstractConfigProducer parent, String clusterName, int index, String storageClusterName, String storageRouteSpec) {
+ super(parent, clusterName, index);
+ this.storageRouteSpec = storageRouteSpec;
+ }
+
+ @Override
+ protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; }
+ public final String getStorageRouteSpec() { return storageRouteSpec; }
+ public int getRowBits() { return 0; }
+
+ @Override
+ public void getConfig(DocumentdbInfoConfig.Builder builder) {
+ DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder();
+ String searchName = sdConfig.getSearch().getName();
+ docDb.name(searchName);
+ SummaryConfig.Producer prod = sdConfig.getSummaries();
+ convertSummaryConfig(prod, null, docDb);
+ RankProfilesConfig.Builder rpb = new RankProfilesConfig.Builder();
+ sdConfig.getRankProfileList().getConfig(rpb);
+ addRankProfilesConfig(docDb, new RankProfilesConfig(rpb));
+ builder.documentdb(docDb);
+ }
+
+ @Override
+ protected void assureSdConsistent() {
+ if (sdConfig == null) {
+ throw new IllegalStateException("Search cluster '" + getClusterName() + "' does not have any search definitions");
+ }
+ }
+
+ protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> local,
+ List<com.yahoo.searchdefinition.Search> global) {
+ if (local.size() == 1) {
+ deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), global);
+ } else if (local.size() > 1){
+ throw new IllegalStateException("Logical indexes are not supported: Got " + local.size() + " search definitions, expected 1");
+ }
+ }
+ private void deriveSingleSearchDefinition(com.yahoo.searchdefinition.Search localSearch,
+ List<com.yahoo.searchdefinition.Search> globalSearches) {
+ this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry());
+ }
+ @Override
+ public DerivedConfiguration getSdConfig() {
+ return sdConfig;
+ }
+ @Override
+ protected void exportSdFiles(File toDir) throws IOException {
+ if (sdConfig!=null) {
+ sdConfig.export(toDir.getCanonicalPath());
+ }
+ }
+ @Override
+ public void defaultDocumentsConfig() { }
+
+ @Override
+ public void getConfig(AttributesConfig.Builder builder) {
+ if (getSdConfig()!=null) getSdConfig().getAttributeFields().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(VsmsummaryConfig.Builder builder) {
+ if (getSdConfig()!=null)
+ if (getSdConfig().getVsmSummary()!=null)
+ getSdConfig().getVsmSummary().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(VsmfieldsConfig.Builder builder) {
+ if (getSdConfig()!=null)
+ if (getSdConfig().getVsmFields()!=null)
+ getSdConfig().getVsmFields().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(SummarymapConfig.Builder builder) {
+ if (getSdConfig()!=null)
+ if (getSdConfig().getSummaryMap()!=null)
+ getSdConfig().getSummaryMap().getConfig(builder);
+ }
+
+ @Override
+ public void getConfig(SummaryConfig.Builder builder) {
+ if (getSdConfig()!=null)
+ if (getSdConfig().getSummaries()!=null)
+ getSdConfig().getSummaries().getConfig(builder);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java
new file mode 100644
index 00000000000..08c9d0ff4ce
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.searchlib.TranslogserverConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import org.w3c.dom.Element;
+
+/**
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class TransactionLogServer extends AbstractService {
+
+ private static final long serialVersionUID = 1L;
+
+ public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName) {
+ super(searchNode, "transactionlogserver");
+ portsMeta.on(0).tag("tls");
+ setProp("clustername", clusterName);
+ setProp("clustertype", "search");
+ }
+
+ public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<TransactionLogServer> {
+ private final String clusterName;
+ public Builder(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ @Override
+ protected TransactionLogServer doBuild(AbstractConfigProducer ancestor, Element producerSpec) {
+ return new TransactionLogServer(ancestor, clusterName);
+ }
+ }
+
+ public int getPortCount() {
+ return 1;
+ }
+
+ /**
+ * Returns the port used by the TLS.
+ *
+ * @return The port.
+ */
+ public int getTlsPort() {
+ return getRelativePort(0);
+ }
+
+ /**
+ * Returns the directory used by the TLS.
+ *
+ * @return The directory.
+ */
+ private String getTlsDir() {
+ return "tls";
+ }
+
+ public void getConfig(TranslogserverConfig.Builder builder) {
+ builder.listenport(getTlsPort()).basedir(getTlsDir());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
new file mode 100644
index 00000000000..8493f5414be
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
@@ -0,0 +1,377 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.config.search.core.PartitionsConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.model.content.TuningDispatch;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Class representing the tuning config used for a search cluster.
+ * Take a look at proton.def and vespa doc for detailed explanations.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class Tuning extends AbstractConfigProducer implements PartitionsConfig.Producer, ProtonConfig.Producer {
+
+ public static class Dispatch implements PartitionsConfig.Producer {
+
+ public Integer maxHitsPerPartition = null;
+ public TuningDispatch.DispatchPolicy policy = TuningDispatch.DispatchPolicy.ROUNDROBIN;
+ public boolean useLocalNode = false;
+ public Double minGroupCoverage = null;
+ public Double minActiveDocsCoverage = null;
+
+ @Override
+ public void getConfig(PartitionsConfig.Builder builder) {
+ if (maxHitsPerPartition != null) {
+ for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) {
+ dataset.maxhitspernode(maxHitsPerPartition);
+ }
+ }
+ if (minGroupCoverage != null) {
+ for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) {
+ dataset.min_group_coverage(minGroupCoverage);
+ }
+ }
+ if (minActiveDocsCoverage != null) {
+ for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) {
+ dataset.min_activedocs_coverage(minActiveDocsCoverage);
+ }
+ }
+ for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) {
+ switch (policy) {
+ case RANDOM:
+ dataset.useroundrobinforfixedrow(false);
+ break;
+ case ROUNDROBIN:
+ default:
+ dataset.useroundrobinforfixedrow(true);
+ break;
+ }
+ }
+ }
+ }
+
+ public static class SearchNode implements ProtonConfig.Producer {
+
+ public enum IoType {
+ NORMAL("NORMAL"),
+ DIRECTIO("DIRECTIO"),
+ MMAP("MMAP");
+
+ public final String name;
+
+ IoType(String name) {
+ this.name = name;
+ }
+
+ public static IoType fromString(String name) {
+ for (IoType type : IoType.values()) {
+ if (toLowerCase(name).equals(toLowerCase(type.name))) {
+ return type;
+ }
+ }
+ return NORMAL;
+ }
+ }
+
+ public static class RequestThreads implements ProtonConfig.Producer {
+ public Integer numSearchThreads = null;
+ public Integer numThreadsPerSearch = null;
+ public Integer numSummaryThreads = null;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (numSearchThreads!=null) builder.numsearcherthreads(numSearchThreads);
+ if (numThreadsPerSearch!=null) builder.numthreadspersearch(numThreadsPerSearch);
+ if (numSummaryThreads!=null) builder.numsummarythreads(numSummaryThreads);
+ }
+ }
+
+ public static class FlushStrategy implements ProtonConfig.Producer {
+ public Long totalMaxMemoryGain = null;
+ public Double totalDiskBloatFactor = null;
+ public Long componentMaxMemoryGain = null;
+ public Double componentDiskBloatFactor = null;
+ public Double componentMaxage = null;
+ public Long transactionLogMaxEntries = null;
+ public Long transactionLogMaxSize = null;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ // Here, the config building gets very ugly, because we have to check for null because of autoconversion Long/long etc.
+
+ ProtonConfig.Flush.Memory.Builder memoryBuilder = new ProtonConfig.Flush.Memory.Builder();
+ if (totalMaxMemoryGain != null) memoryBuilder.maxmemory(totalMaxMemoryGain);
+ if (totalDiskBloatFactor != null) memoryBuilder.diskbloatfactor(totalDiskBloatFactor);
+ if (transactionLogMaxSize != null) memoryBuilder.maxtlssize(transactionLogMaxSize);
+
+ ProtonConfig.Flush.Memory.Each.Builder eachBuilder = new ProtonConfig.Flush.Memory.Each.Builder();
+ if (componentMaxMemoryGain != null) eachBuilder.maxmemory(componentMaxMemoryGain);
+ if (componentDiskBloatFactor != null) eachBuilder.diskbloatfactor(componentDiskBloatFactor);
+ memoryBuilder.each(eachBuilder);
+
+ ProtonConfig.Flush.Memory.Maxage.Builder maxageBuilder = new ProtonConfig.Flush.Memory.Maxage.Builder();
+ if (componentMaxage != null) maxageBuilder.time(componentMaxage);
+ if (transactionLogMaxEntries != null) maxageBuilder.serial(transactionLogMaxEntries);
+ memoryBuilder.maxage(maxageBuilder);
+
+ builder.
+ flush(new ProtonConfig.Flush.Builder().
+ memory(memoryBuilder));
+ }
+ }
+
+ public static class Resizing implements ProtonConfig.Producer {
+ public Integer initialDocumentCount = null;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (initialDocumentCount!=null) builder.
+ grow(new ProtonConfig.Grow.Builder().
+ initial(initialDocumentCount));
+ }
+
+ }
+
+ public static class Index implements ProtonConfig.Producer {
+ public static class Io implements ProtonConfig.Producer {
+ public IoType write = null;
+ public IoType read = null;
+ public IoType search = null;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ ProtonConfig.Indexing.Builder indexingB = new ProtonConfig.Indexing.Builder();
+ if (write != null) indexingB.
+ write(new ProtonConfig.Indexing.Write.Builder().
+ io(ProtonConfig.Indexing.Write.Io.Enum.valueOf(write.name)));
+ if (read != null) indexingB.
+ read(new ProtonConfig.Indexing.Read.Builder().
+ io(ProtonConfig.Indexing.Read.Io.Enum.valueOf(read.name)));
+ if (search != null) builder.search(new ProtonConfig.Search.Builder().
+ io(ProtonConfig.Search.Io.Enum.valueOf(search.name)));
+ builder.indexing(indexingB);
+ }
+
+ }
+
+ public Io io;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (io != null) io.getConfig(builder);
+ }
+ }
+
+ public static class Attribute implements ProtonConfig.Producer {
+ public static class Io implements ProtonConfig.Producer {
+ public IoType write = null;
+
+ public Io() {}
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (write != null) builder.attribute(new ProtonConfig.Attribute.Builder().
+ write(new ProtonConfig.Attribute.Write.Builder().
+ io(ProtonConfig.Attribute.Write.Io.Enum.valueOf(write.name))));
+ }
+
+ }
+
+ public Io io;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (io != null) io.getConfig(builder);
+ }
+
+ }
+
+ public static class Summary implements ProtonConfig.Producer {
+ public static class Io {
+ public IoType write = null;
+ public IoType read = null;
+
+ public void getConfig(ProtonConfig.Summary.Builder builder) {
+ if (write != null) builder.
+ write(new ProtonConfig.Summary.Write.Builder().
+ io(ProtonConfig.Summary.Write.Io.Enum.valueOf(write.name)));
+ if (read != null) builder.
+ read(new ProtonConfig.Summary.Read.Builder().
+ io(ProtonConfig.Summary.Read.Io.Enum.valueOf(read.name)));
+ }
+ }
+
+ public static class Store {
+ public static class Compression {
+ public enum Type {
+ NONE("NONE"),
+ LZ4("LZ4");
+
+ public final String name;
+
+ Type(String name) {
+ this.name = name;
+ }
+ public static Type fromString(String name) {
+ for (Type type : Type.values()) {
+ if (toLowerCase(name).equals(toLowerCase(type.name))) {
+ return type;
+ }
+ }
+ return NONE;
+ }
+ }
+ public Type type = null;
+ public Integer level = null;
+
+ public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) {
+ if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name));
+ if (level != null) compression.level(level);
+ }
+
+ public void getConfig(ProtonConfig.Summary.Log.Chunk.Compression.Builder compression) {
+ if (type != null) compression.type(ProtonConfig.Summary.Log.Chunk.Compression.Type.Enum.valueOf(type.name));
+ if (level != null) compression.level(level);
+ }
+ }
+
+ public static class Component {
+ public Long maxSize = null;
+ public Long maxEntries = null;
+ public Long initialEntries = null;
+ public Compression compression = null;
+ private final boolean outputInt;
+
+ public Component() {
+ this.outputInt = false;
+ }
+
+ public Component(boolean outputInt) {
+ this.outputInt = outputInt;
+ }
+
+ public void getConfig(ProtonConfig.Summary.Cache.Builder cache) {
+ if (outputInt) {
+ if (maxSize!=null) cache.maxbytes(maxSize.intValue());
+ if (maxEntries!=null) cache.initialentries(maxEntries.intValue());
+ if (initialEntries!=null) cache.initialentries(initialEntries.intValue());
+ } else {
+ if (maxSize!=null) cache.maxbytes(maxSize);
+ if (maxEntries!=null) cache.initialentries(maxEntries);
+ if (initialEntries!=null) cache.initialentries(initialEntries);
+ }
+ if (compression != null) {
+ ProtonConfig.Summary.Cache.Compression.Builder compressionB = new ProtonConfig.Summary.Cache.Compression.Builder();
+ compression.getConfig(compressionB);
+ cache.compression(compressionB);
+ }
+ }
+
+ public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) {
+ if (outputInt) {
+ if (maxSize!=null) chunk.maxbytes(maxSize.intValue());
+ if (maxEntries!=null) chunk.maxentries(maxEntries.intValue());
+ } else {
+ throw new IllegalStateException("Fix this, chunk does not have long types");
+ }
+ if (compression != null) {
+ ProtonConfig.Summary.Log.Chunk.Compression.Builder compressionB = new ProtonConfig.Summary.Log.Chunk.Compression.Builder();
+ compression.getConfig(compressionB);
+ chunk.compression(compressionB);
+ }
+ }
+ }
+
+ public static class LogStore {
+ public Long maxFileSize = null;
+ public Double maxDiskBloatFactor = null;
+ public Integer numThreads = null;
+ public Component chunk = null;
+ public Double minFileSizeFactor = null;
+
+ public void getConfig(ProtonConfig.Summary.Log.Builder log) {
+ if (maxFileSize!=null) log.maxfilesize(maxFileSize);
+ if (maxDiskBloatFactor!=null) log.maxdiskbloatfactor(maxDiskBloatFactor);
+ if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor);
+ if (numThreads != null) log.numthreads(numThreads);
+ if (chunk != null) {
+ ProtonConfig.Summary.Log.Chunk.Builder chunkB = new ProtonConfig.Summary.Log.Chunk.Builder();
+ chunk.getConfig(chunkB);
+ log.chunk(chunkB);
+ }
+ }
+ }
+
+ public Component cache;
+ public LogStore logStore;
+
+ public void getConfig(ProtonConfig.Summary.Builder builder) {
+ if (cache != null) {
+ ProtonConfig.Summary.Cache.Builder cacheB=new ProtonConfig.Summary.Cache.Builder();
+ cache.getConfig(cacheB);
+ builder.cache(cacheB);
+
+ }
+ if (logStore != null) {
+ ProtonConfig.Summary.Log.Builder logB = new ProtonConfig.Summary.Log.Builder();
+ logStore.getConfig(logB);
+ builder.log(logB);
+ }
+ }
+ }
+
+ public Io io;
+ public Store store;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ ProtonConfig.Summary.Builder summary = new ProtonConfig.Summary.Builder();
+ if (io != null) io.getConfig(summary);
+ if (store != null) store.getConfig(summary);
+ builder.summary(summary);
+ }
+ }
+
+ public RequestThreads threads = null;
+ public FlushStrategy strategy = null;
+ public Resizing resizing = null;
+ public Index index = null;
+ public Attribute attribute = null;
+ public Summary summary = null;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (threads != null) threads.getConfig(builder);
+ if (strategy != null) strategy.getConfig(builder);
+ if (resizing != null) resizing.getConfig(builder);
+ if (index != null) index.getConfig(builder);
+ if (attribute != null) attribute.getConfig(builder);
+ if (summary != null) summary.getConfig(builder);
+ }
+ }
+
+ public Dispatch dispatch;
+ public SearchNode searchNode;
+
+ public Tuning(AbstractConfigProducer parent) {
+ super(parent, "tuning");
+ }
+
+ @Override
+ public void getConfig(PartitionsConfig.Builder builder) {
+ if (dispatch != null) {
+ dispatch.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (searchNode != null) searchNode.getConfig(builder);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java
new file mode 100644
index 00000000000..1c51f021fef
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.utils;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Parses a string on the form:
+ *
+ * [numbers]\s*[unit]?
+ *
+ * Where numbers is a double, and unit is one of:
+ * d - days
+ * m - minutes
+ * s - seconds
+ * ms - milliseconds
+ *
+ * Default is seconds.
+ */
+public class Duration {
+ private static Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?");
+ private static Map<String, Integer> unitMultiplier = new HashMap<>();
+ static {
+ unitMultiplier.put("s", 1000);
+ unitMultiplier.put("d", 1000 * 3600 * 24);
+ unitMultiplier.put("ms", 1);
+ unitMultiplier.put("m", 60 * 1000);
+ unitMultiplier.put("h", 60 * 60 * 1000);
+ }
+
+ long value;
+
+ public Duration(String value) {
+ Matcher matcher = pattern.matcher(value);
+
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Illegal duration format: " + value);
+ }
+
+ double num = Double.parseDouble(matcher.group(1));
+ String unit = matcher.group(2);
+
+ if (unit != null) {
+ Integer multiplier = unitMultiplier.get(unit);
+ if (multiplier == null) {
+ throw new IllegalArgumentException("Unknown time unit: " + unit + " in time value " + value);
+ }
+
+ num *= multiplier;
+ } else {
+ num *= 1000;
+ }
+
+ this.value = (long)num;
+ }
+
+ public double getSeconds() {
+ return value / 1000.0;
+ }
+
+ public long getMilliSeconds() {
+ return value;
+ }
+
+}
+
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
new file mode 100644
index 00000000000..12de5a3fbb0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.utils;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.producer.UserConfigRepo;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.*;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.config.ConfigDefinition.DefaultValued;
+import com.yahoo.vespa.model.AbstractService;
+
+import java.io.Serializable;
+import java.util.*;
+
+/**
+ * Utility methods for sending files to a collection of nodes.
+ *
+ * @author gjoranv
+ * @since 5.1.9
+ */
+public class FileSender implements Serializable {
+
+ /**
+ * Send the given file to all given services.
+ *
+ * @param relativePath The path to the file, relative to the app pkg.
+ * @param services The services to send the file to.
+ * @return The file reference that the file was given, never null.
+ * @throws IllegalStateException if services is empty.
+ */
+ public static FileReference sendFileToServices(String relativePath,
+ Collection<? extends AbstractService> services) {
+ if (services.isEmpty()) {
+ throw new IllegalStateException("'sendFileToServices called for empty services!" +
+ " - This should never happen!");
+ }
+ FileReference fileref = null;
+ for (AbstractService service : services) {
+ // The same reference will be returned from each call.
+ fileref = service.sendFile(relativePath);
+ }
+ return fileref;
+ }
+
+ /**
+ * Sends all user configured files for a producer to all given services.
+ */
+ public static <PRODUCER extends AbstractConfigProducer<?>>
+ void sendUserConfiguredFiles(PRODUCER producer, Collection<? extends AbstractService> services, DeployLogger logger) {
+ if (services.isEmpty())
+ return;
+
+ UserConfigRepo userConfigs = producer.getUserConfigs();
+ Map<String, FileReference> sentFiles = new HashMap<>();
+ for (ConfigDefinitionKey key : userConfigs.configsProduced()) {
+ ConfigPayloadBuilder builder = userConfigs.get(key);
+ try {
+ sendUserConfiguredFiles(builder, sentFiles, services, key, logger);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Unable to send files for " + key, e);
+ }
+ }
+ }
+
+ private static void sendUserConfiguredFiles(ConfigPayloadBuilder builder,
+ Map<String, FileReference> sentFiles,
+ Collection<? extends AbstractService> services,
+ ConfigDefinitionKey key, DeployLogger logger) {
+ ConfigDefinition configDefinition = builder.getConfigDefinition();
+ if (configDefinition == null) {
+ // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder);
+ logger.log(LogLevel.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config");
+ return;
+ }
+ // Inspect fields at this level
+ sendEntries(builder, sentFiles, services, configDefinition.getFileDefs());
+ sendEntries(builder, sentFiles, services, configDefinition.getPathDefs());
+
+ // Inspect arrays
+ for (Map.Entry<String, ConfigDefinition.ArrayDef> entry : configDefinition.getArrayDefs().entrySet()) {
+ if (isFileOrPathArray(entry)) {
+ ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
+ sendFileEntries(array.getElements(), sentFiles, services);
+ }
+ }
+ // Maps
+ for (Map.Entry<String, ConfigDefinition.LeafMapDef> entry : configDefinition.getLeafMapDefs().entrySet()) {
+ if (isFileOrPathMap(entry)) {
+ ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
+ sendFileEntries(map.getElements(), sentFiles, services);
+ }
+ }
+
+ // Inspect inner fields
+ for (String name : configDefinition.getStructDefs().keySet()) {
+ sendUserConfiguredFiles(builder.getObject(name), sentFiles, services, key, logger);
+ }
+ for (String name : configDefinition.getInnerArrayDefs().keySet()) {
+ ConfigPayloadBuilder.Array array = builder.getArray(name);
+ for (ConfigPayloadBuilder element : array.getElements()) {
+ sendUserConfiguredFiles(element, sentFiles, services, key, logger);
+ }
+ }
+ for (String name : configDefinition.getStructMapDefs().keySet()) {
+ ConfigPayloadBuilder.MapBuilder map = builder.getMap(name);
+ for (ConfigPayloadBuilder element : map.getElements()) {
+ sendUserConfiguredFiles(element, sentFiles, services, key, logger);
+ }
+ }
+
+ }
+
+ private static boolean isFileOrPathMap(Map.Entry<String, ConfigDefinition.LeafMapDef> entry) {
+ String mapType = entry.getValue().getTypeSpec().getType();
+ return ("file".equals(mapType) || "path".equals(mapType));
+ }
+
+ private static boolean isFileOrPathArray(Map.Entry<String, ConfigDefinition.ArrayDef> entry) {
+ String arrayType = entry.getValue().getTypeSpec().getType();
+ return ("file".equals(arrayType) || "path".equals(arrayType));
+ }
+
+ private static void sendEntries(ConfigPayloadBuilder builder, Map<String, FileReference> sentFiles, Collection<? extends AbstractService> services, Map<String, ? extends DefaultValued<String>> entries) {
+ for (String name : entries.keySet()) {
+ ConfigPayloadBuilder fileEntry = builder.getObject(name);
+ if (fileEntry.getValue() == null) {
+ throw new IllegalArgumentException("Unable to send file for field '" + name + "'. Invalid config value " + fileEntry.getValue());
+ }
+ sendFileEntry(fileEntry, sentFiles, services);
+ }
+ }
+
+ private static void sendFileEntries(Collection<ConfigPayloadBuilder> builders, Map<String, FileReference> sentFiles, Collection<? extends AbstractService> services) {
+ for (ConfigPayloadBuilder builder : builders) {
+ sendFileEntry(builder, sentFiles, services);
+ }
+ }
+
+ private static void sendFileEntry(ConfigPayloadBuilder builder, Map<String, FileReference> sentFiles, Collection<? extends AbstractService> services) {
+ String path = builder.getValue();
+ FileReference reference = sentFiles.get(path);
+ if (reference == null) {
+ reference = sendFileToServices(path, services);
+ sentFiles.put(path, reference);
+ }
+ builder.setValue(reference.value());
+ }
+} \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java
new file mode 100644
index 00000000000..a05008cc9a0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.utils;
+
+import java.util.*;
+
+/**
+ * Delegates to a map that can be froozen.
+ * Not thread safe.
+ * @author tonytv
+ */
+public class FreezableMap<K, V> implements Map<K, V> {
+ private boolean frozen = false;
+ private Map<K, V> map;
+
+ //TODO: review the use of unchecked.
+ @SuppressWarnings("unchecked")
+ public FreezableMap(Class<LinkedHashMap> mapClass) {
+ try {
+ map = mapClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public int size() {
+ return map.size();
+ }
+
+ public boolean isEmpty() {
+ return map.isEmpty();
+ }
+
+ public boolean containsKey(Object o) {
+ return map.containsKey(o);
+ }
+
+ public boolean containsValue(Object o) {
+ return map.containsValue(o);
+ }
+
+ public V get(Object o) {
+ return map.get(o);
+ }
+
+ public V put(K key, V value) {
+ return map.put(key, value);
+ }
+
+ public V remove(Object o) {
+ return map.remove(o);
+ }
+
+ public void putAll(Map<? extends K, ? extends V> map) {
+ this.map.putAll(map);
+ }
+
+ public void clear() {
+ map.clear();
+ }
+
+ public Set<K> keySet() {
+ return map.keySet();
+ }
+
+ public Collection<V> values() {
+ return map.values();
+ }
+
+ public Set<Entry<K, V>> entrySet() {
+ return map.entrySet();
+ }
+
+ public boolean equals(Object o) {
+ return map.equals(o);
+ }
+
+ public int hashCode() {
+ return map.hashCode();
+ }
+
+ public void freeze() {
+ if (frozen)
+ throw new RuntimeException("The map has already been frozen.");
+ frozen = true;
+ map = Collections.unmodifiableMap(map);
+ }
+
+ public boolean isFrozen() {
+ return frozen;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java
new file mode 100644
index 00000000000..2bc7d2dfbe7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.utils.internal;
+
+import com.yahoo.config.ChangesRequiringRestart;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.vespa.config.ConfigKey;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Utility class containing static methods for retrievinig information about the config producer tree.
+ *
+ * @author lulf
+ * @author bjorncs
+ * @since 5.1
+ */
+public final class ReflectionUtil {
+
+ private ReflectionUtil() {
+ }
+
+ /**
+ * Returns a set of all the configs produced by a given producer.
+ *
+ * @param iface The config producer or interface to check for producers.
+ * @param configId The config id to use when creating keys.
+ * @return A set of config keys.
+ */
+ public static Set<ConfigKey<?>> configsProducedByInterface(Class<?> iface, String configId) {
+ Set<ConfigKey<?>> ret = new LinkedHashSet<>();
+ if (isConcreteProducer(iface)) {
+ ret.add(createConfigKeyFromInstance(iface.getEnclosingClass(), configId));
+ }
+ for (Class<?> parentIface : iface.getInterfaces()) {
+ ret.addAll(configsProducedByInterface(parentIface, configId));
+ }
+ return ret;
+ }
+
+ /**
+ * Determines if the config class contains the methods required for detecting config value changes
+ * between two config instances.
+ */
+ public static boolean hasRestartMethods(Class<? extends ConfigInstance> configClass) {
+ try {
+ configClass.getDeclaredMethod("containsFieldsFlaggedWithRestart");
+ configClass.getDeclaredMethod("getChangesRequiringRestart", configClass);
+ return true;
+ } catch (NoSuchMethodException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Determines if the config definition for the given config class contains key-values flagged with restart.
+ */
+ public static boolean containsFieldsFlaggedWithRestart(Class<? extends ConfigInstance> configClass) {
+ try {
+ Method m = configClass.getDeclaredMethod("containsFieldsFlaggedWithRestart");
+ m.setAccessible(true);
+ return (boolean) m.invoke(null);
+ } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Compares the config instances and lists any differences that will require service restart.
+ * @param from The previous config.
+ * @param to The new config.
+ * @return An object describing the difference.
+ */
+ public static ChangesRequiringRestart getChangesRequiringRestart(ConfigInstance from, ConfigInstance to) {
+ Class<?> clazz = from.getClass();
+ if (!clazz.equals(to.getClass())) {
+ throw new IllegalArgumentException(String.format("%s != %s", clazz, to.getClass()));
+ }
+ try {
+ Method m = clazz.getDeclaredMethod("getChangesRequiringRestart", clazz);
+ m.setAccessible(true);
+ return (ChangesRequiringRestart) m.invoke(from, to);
+ } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static ConfigKey<?> createConfigKeyFromInstance(Class<?> configInstClass, String configId) {
+ try {
+ String defName = ConfigInstance.getDefName(configInstClass);
+ String defNamespace = ConfigInstance.getDefNamespace(configInstClass);
+ return new ConfigKey<>(defName, configId, defNamespace);
+ } catch (IllegalArgumentException | SecurityException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static boolean classIsConfigInstanceProducer(Class<?> clazz) {
+ return clazz.getName().equals(ConfigInstance.Producer.class.getName());
+ }
+
+ private static boolean isConcreteProducer(Class<?> producerInterface) {
+ boolean parentIsConfigInstance = false;
+ for (Class<?> ifaceParent : producerInterface.getInterfaces()) {
+ if (classIsConfigInstanceProducer(ifaceParent)) {
+ parentIsConfigInstance = true;
+ }
+ }
+ return (ConfigInstance.Producer.class.isAssignableFrom(producerInterface) && parentIsConfigInstance && !classIsConfigInstanceProducer(producerInterface));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java
new file mode 100644
index 00000000000..c19058af4fa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.model.utils;
+
+import com.yahoo.osgi.annotation.ExportPackage;