From 5c24dc5c9642a8d9ed70aee4c950fd0678a1ebec Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 19 May 2022 12:03:06 +0200 Subject: Rename the 'searchdefinition' package to 'schema' --- .../application/container/DocumentAccesses.java | 2 +- .../java/ai/vespa/reindexing/ReindexerTest.java | 3 +- .../ai/vespa/reindexing/ReindexingCuratorTest.java | 2 +- .../vespa/reindexing/ReindexingMaintainerTest.java | 2 +- .../vespa/reindexing/http/ReindexingV1ApiTest.java | 2 +- .../com/yahoo/config/model/deploy/DeployState.java | 9 +- .../com/yahoo/documentmodel/NewDocumentType.java | 9 +- .../main/java/com/yahoo/schema/Application.java | 108 ++ .../java/com/yahoo/schema/ApplicationBuilder.java | 513 +++++++ .../java/com/yahoo/schema/DefaultRankProfile.java | 128 ++ .../com/yahoo/schema/DistributableResource.java | 95 ++ .../com/yahoo/schema/DocumentGraphValidator.java | 79 + .../com/yahoo/schema/DocumentModelBuilder.java | 639 ++++++++ .../java/com/yahoo/schema/DocumentOnlySchema.java | 33 + .../java/com/yahoo/schema/DocumentReference.java | 28 + .../yahoo/schema/DocumentReferenceResolver.java | 94 ++ .../java/com/yahoo/schema/DocumentReferences.java | 37 + .../com/yahoo/schema/DocumentsOnlyRankProfile.java | 36 + .../main/java/com/yahoo/schema/FeatureNames.java | 92 ++ .../com/yahoo/schema/FieldOperationApplier.java | 32 + .../schema/FieldOperationApplierForSearch.java | 23 + .../schema/FieldOperationApplierForStructs.java | 20 + .../src/main/java/com/yahoo/schema/FieldSets.java | 77 + .../java/com/yahoo/schema/ImmutableSchema.java | 52 + .../com/yahoo/schema/ImportedFieldsEnumerator.java | 31 + .../src/main/java/com/yahoo/schema/Index.java | 200 +++ .../com/yahoo/schema/LargeRankExpressions.java | 51 + .../com/yahoo/schema/MapEvaluationTypeContext.java | 361 +++++ .../src/main/java/com/yahoo/schema/OnnxModel.java | 120 ++ .../java/com/yahoo/schema/RankExpressionBody.java | 32 + .../main/java/com/yahoo/schema/RankProfile.java | 1526 ++++++++++++++++++++ .../java/com/yahoo/schema/RankProfileRegistry.java | 138 ++ .../com/yahoo/schema/SDDocumentTypeOrderer.java | 136 ++ .../src/main/java/com/yahoo/schema/Schema.java | 754 ++++++++++ .../com/yahoo/schema/TemporarySDTypeResolver.java | 79 + .../java/com/yahoo/schema/UnrankedRankProfile.java | 28 + .../com/yahoo/schema/derived/AttributeFields.java | 324 +++++ .../java/com/yahoo/schema/derived/Derived.java | 141 ++ .../yahoo/schema/derived/DerivedConfiguration.java | 212 +++ .../java/com/yahoo/schema/derived/Deriver.java | 48 + .../java/com/yahoo/schema/derived/Exportable.java | 26 + .../yahoo/schema/derived/FieldRankSettings.java | 75 + .../yahoo/schema/derived/FieldResultTransform.java | 57 + .../schema/derived/FileDistributedConstants.java | 87 ++ .../schema/derived/FileDistributedOnnxModels.java | 60 + .../com/yahoo/schema/derived/ImportedFields.java | 105 ++ .../main/java/com/yahoo/schema/derived/Index.java | 64 + .../java/com/yahoo/schema/derived/IndexInfo.java | 595 ++++++++ .../java/com/yahoo/schema/derived/IndexSchema.java | 245 ++++ .../com/yahoo/schema/derived/IndexingScript.java | 197 +++ .../java/com/yahoo/schema/derived/Juniperrc.java | 62 + .../schema/derived/NativeRankTypeDefinition.java | 44 + .../derived/NativeRankTypeDefinitionSet.java | 93 ++ .../java/com/yahoo/schema/derived/NativeTable.java | 72 + .../com/yahoo/schema/derived/RankProfileList.java | 210 +++ .../com/yahoo/schema/derived/RawRankProfile.java | 524 +++++++ .../java/com/yahoo/schema/derived/SchemaInfo.java | 129 ++ .../com/yahoo/schema/derived/SearchOrderer.java | 123 ++ .../java/com/yahoo/schema/derived/Summaries.java | 51 + .../com/yahoo/schema/derived/SummaryClass.java | 133 ++ .../yahoo/schema/derived/SummaryClassField.java | 132 ++ .../java/com/yahoo/schema/derived/SummaryMap.java | 120 ++ .../java/com/yahoo/schema/derived/VsmFields.java | 313 ++++ .../java/com/yahoo/schema/derived/VsmSummary.java | 109 ++ .../com/yahoo/schema/derived/package-info.java | 5 + .../validation/IndexStructureValidator.java | 50 + .../schema/derived/validation/Validation.java | 12 + .../yahoo/schema/derived/validation/Validator.java | 22 + .../java/com/yahoo/schema/document/Attribute.java | 435 ++++++ .../schema/document/BooleanIndexDefinition.java | 84 ++ .../main/java/com/yahoo/schema/document/Case.java | 15 + .../document/ComplexAttributeFieldUtils.java | 123 ++ .../java/com/yahoo/schema/document/Dictionary.java | 35 + .../java/com/yahoo/schema/document/FieldSet.java | 41 + .../java/com/yahoo/schema/document/GeoPos.java | 26 + .../com/yahoo/schema/document/HnswIndexParams.java | 76 + .../document/ImmutableImportedComplexSDField.java | 29 + .../schema/document/ImmutableImportedSDField.java | 218 +++ .../yahoo/schema/document/ImmutableSDField.java | 91 ++ .../schema/document/ImportedComplexField.java | 49 + .../com/yahoo/schema/document/ImportedField.java | 38 + .../com/yahoo/schema/document/ImportedFields.java | 23 + .../yahoo/schema/document/ImportedSimpleField.java | 18 + .../com/yahoo/schema/document/MatchAlgorithm.java | 16 + .../java/com/yahoo/schema/document/MatchType.java | 14 + .../java/com/yahoo/schema/document/Matching.java | 141 ++ .../com/yahoo/schema/document/NormalizeLevel.java | 87 ++ .../java/com/yahoo/schema/document/RankType.java | 40 + .../java/com/yahoo/schema/document/Ranking.java | 76 + .../com/yahoo/schema/document/SDDocumentType.java | 347 +++++ .../java/com/yahoo/schema/document/SDField.java | 802 ++++++++++ .../java/com/yahoo/schema/document/Sorting.java | 65 + .../java/com/yahoo/schema/document/Stemming.java | 64 + .../schema/document/TemporaryImportedField.java | 37 + .../schema/document/TemporaryImportedFields.java | 44 + .../schema/document/TemporarySDDocumentType.java | 13 + .../yahoo/schema/document/TemporarySDField.java | 19 + .../java/com/yahoo/schema/document/TypedKey.java | 20 + .../document/annotation/SDAnnotationType.java | 42 + .../TemporaryAnnotationReferenceDataType.java | 28 + .../ConstantTensorTransformer.java | 71 + .../expressiontransforms/ExpressionTransforms.java | 47 + .../expressiontransforms/FunctionInliner.java | 33 + .../expressiontransforms/FunctionShadower.java | 59 + .../LightGBMFeatureConverter.java | 59 + .../expressiontransforms/OnnxFeatureConverter.java | 64 + .../expressiontransforms/OnnxModelTransformer.java | 139 ++ .../RankProfileTransformContext.java | 65 + .../TensorFlowFeatureConverter.java | 66 + .../expressiontransforms/TokenTransformer.java | 313 ++++ .../XgboostFeatureConverter.java | 61 + .../schema/fieldoperation/AliasOperation.java | 42 + .../schema/fieldoperation/AttributeOperation.java | 172 +++ .../schema/fieldoperation/BoldingOperation.java | 25 + .../schema/fieldoperation/DictionaryOperation.java | 41 + .../schema/fieldoperation/FieldOperation.java | 22 + .../fieldoperation/FieldOperationContainer.java | 19 + .../yahoo/schema/fieldoperation/IdOperation.java | 35 + .../schema/fieldoperation/IndexOperation.java | 134 ++ .../schema/fieldoperation/IndexingOperation.java | 61 + .../schema/fieldoperation/MatchOperation.java | 65 + .../fieldoperation/NormalizingOperation.java | 34 + .../fieldoperation/QueryCommandOperation.java | 25 + .../yahoo/schema/fieldoperation/RankOperation.java | 36 + .../schema/fieldoperation/RankTypeOperation.java | 43 + .../schema/fieldoperation/SortingOperation.java | 93 ++ .../schema/fieldoperation/StemmingOperation.java | 26 + .../fieldoperation/StructFieldOperation.java | 56 + .../SummaryInFieldLongOperation.java | 70 + .../fieldoperation/SummaryInFieldOperation.java | 46 + .../SummaryInFieldShortOperation.java | 32 + .../schema/fieldoperation/SummaryToOperation.java | 41 + .../schema/fieldoperation/WeightOperation.java | 25 + .../fieldoperation/WeightedSetOperation.java | 70 + .../yahoo/schema/parser/ConvertParsedFields.java | 331 +++++ .../yahoo/schema/parser/ConvertParsedRanking.java | 124 ++ .../yahoo/schema/parser/ConvertParsedSchemas.java | 221 +++ .../yahoo/schema/parser/ConvertParsedTypes.java | 337 +++++ .../schema/parser/ConvertSchemaCollection.java | 212 +++ .../com/yahoo/schema/parser/DictionaryOption.java | 5 + .../yahoo/schema/parser/InheritanceResolver.java | 130 ++ .../schema/parser/IntermediateCollection.java | 159 ++ .../com/yahoo/schema/parser/ParsedAnnotation.java | 57 + .../com/yahoo/schema/parser/ParsedAttribute.java | 69 + .../java/com/yahoo/schema/parser/ParsedBlock.java | 36 + .../com/yahoo/schema/parser/ParsedDocument.java | 127 ++ .../yahoo/schema/parser/ParsedDocumentSummary.java | 49 + .../java/com/yahoo/schema/parser/ParsedField.java | 159 ++ .../com/yahoo/schema/parser/ParsedFieldSet.java | 36 + .../java/com/yahoo/schema/parser/ParsedIndex.java | 79 + .../com/yahoo/schema/parser/ParsedIndexingOp.java | 37 + .../yahoo/schema/parser/ParsedMatchSettings.java | 38 + .../yahoo/schema/parser/ParsedRankFunction.java | 39 + .../com/yahoo/schema/parser/ParsedRankProfile.java | 220 +++ .../java/com/yahoo/schema/parser/ParsedSchema.java | 176 +++ .../com/yahoo/schema/parser/ParsedSorting.java | 48 + .../java/com/yahoo/schema/parser/ParsedStruct.java | 60 + .../yahoo/schema/parser/ParsedSummaryField.java | 51 + .../java/com/yahoo/schema/parser/ParsedType.java | 224 +++ .../com/yahoo/schema/parser/SimpleCharStream.java | 16 + .../main/java/com/yahoo/schema/parser/Utils.java | 15 + ...ttributeTransformToSummaryOfImportedFields.java | 61 + .../processing/AddExtraFieldsToDocument.java | 90 ++ .../processing/AdjustPositionSummaryFields.java | 135 ++ .../schema/processing/AttributeProperties.java | 72 + .../schema/processing/AttributesImplicitWord.java | 57 + .../java/com/yahoo/schema/processing/Bolding.java | 47 + .../schema/processing/BoolAttributeValidator.java | 36 + .../yahoo/schema/processing/BuiltInFieldSets.java | 52 + .../schema/processing/CreatePositionZCurve.java | 216 +++ .../schema/processing/DictionaryProcessor.java | 54 + .../DisallowComplexMapAndWsetKeyTypes.java | 57 + .../processing/DiversitySettingsValidator.java | 65 + .../com/yahoo/schema/processing/ExactMatch.java | 109 ++ .../schema/processing/FastAccessValidator.java | 54 + .../yahoo/schema/processing/FieldSetSettings.java | 107 ++ .../yahoo/schema/processing/FilterFieldNames.java | 72 + .../yahoo/schema/processing/ImplicitSummaries.java | 232 +++ .../schema/processing/ImplicitSummaryFields.java | 40 + .../schema/processing/ImportedFieldsResolver.java | 207 +++ .../yahoo/schema/processing/IndexFieldNames.java | 47 + .../yahoo/schema/processing/IndexingInputs.java | 106 ++ .../yahoo/schema/processing/IndexingOutputs.java | 144 ++ .../schema/processing/IndexingValidation.java | 164 +++ .../yahoo/schema/processing/IndexingValues.java | 71 + .../schema/processing/IntegerIndex2Attribute.java | 88 ++ .../com/yahoo/schema/processing/LiteralBoost.java | 79 + .../com/yahoo/schema/processing/MakeAliases.java | 61 + .../processing/MakeDefaultSummaryTheSuperSet.java | 49 + .../yahoo/schema/processing/MatchConsistency.java | 72 + .../processing/MatchPhaseSettingsValidator.java | 98 ++ .../processing/MatchedElementsOnlyResolver.java | 95 ++ .../processing/MultifieldIndexHarmonizer.java | 76 + .../yahoo/schema/processing/MutableAttributes.java | 29 + .../com/yahoo/schema/processing/NGramMatch.java | 78 + .../processing/OnnxModelConfigGenerator.java | 96 ++ .../schema/processing/OnnxModelTypeResolver.java | 40 + .../yahoo/schema/processing/OptimizeIlscript.java | 38 + .../schema/processing/PagedAttributeValidator.java | 66 + .../schema/processing/PredicateProcessor.java | 144 ++ .../com/yahoo/schema/processing/Processing.java | 152 ++ .../com/yahoo/schema/processing/Processor.java | 157 ++ .../processing/RankingExpressionTypeResolver.java | 135 ++ .../processing/ReferenceFieldsProcessor.java | 65 + .../schema/processing/ReservedDocumentNames.java | 39 + .../schema/processing/ReservedFunctionNames.java | 56 + .../schema/processing/SearchMustHaveDocument.java | 30 + .../processing/SetRankTypeEmptyOnFilters.java | 30 + .../yahoo/schema/processing/SortingSettings.java | 42 + .../StringSettingsOnNonStringFields.java | 43 + .../schema/processing/SummaryConsistency.java | 131 ++ .../processing/SummaryDiskAccessValidator.java | 73 + .../processing/SummaryDynamicStructsArrays.java | 47 + .../SummaryFieldsMustHaveValidSource.java | 81 ++ .../processing/SummaryNamesFieldCollisions.java | 60 + .../java/com/yahoo/schema/processing/TagType.java | 47 + .../schema/processing/TensorFieldProcessor.java | 118 ++ .../com/yahoo/schema/processing/TextMatch.java | 127 ++ .../schema/processing/TypedTransformProvider.java | 62 + .../java/com/yahoo/schema/processing/UriHack.java | 77 + .../yahoo/schema/processing/UrlFieldValidator.java | 34 + .../schema/processing/ValidateFieldTypes.java | 83 ++ .../ValidateFieldTypesDocumentsOnly.java | 26 + ...ValidateFieldWithIndexSettingsCreatesIndex.java | 42 + .../processing/ValidateStructTypeInheritance.java | 71 + .../com/yahoo/schema/processing/WordMatch.java | 51 + .../multifieldresolver/IndexCommandResolver.java | 62 + .../multifieldresolver/MultiFieldResolver.java | 33 + .../RankProfileTypeSettingsProcessor.java | 105 ++ .../multifieldresolver/RankTypeResolver.java | 46 + .../multifieldresolver/StemmingResolver.java | 43 + .../com/yahoo/schema/processing/package-info.java | 14 + .../com/yahoo/searchdefinition/Application.java | 108 -- .../yahoo/searchdefinition/ApplicationBuilder.java | 513 ------- .../yahoo/searchdefinition/DefaultRankProfile.java | 129 -- .../searchdefinition/DistributableResource.java | 95 -- .../searchdefinition/DocumentGraphValidator.java | 79 - .../searchdefinition/DocumentModelBuilder.java | 641 -------- .../yahoo/searchdefinition/DocumentOnlySchema.java | 33 - .../yahoo/searchdefinition/DocumentReference.java | 28 - .../DocumentReferenceResolver.java | 94 -- .../yahoo/searchdefinition/DocumentReferences.java | 37 - .../searchdefinition/DocumentsOnlyRankProfile.java | 36 - .../com/yahoo/searchdefinition/FeatureNames.java | 92 -- .../searchdefinition/FieldOperationApplier.java | 32 - .../FieldOperationApplierForSearch.java | 23 - .../FieldOperationApplierForStructs.java | 27 - .../java/com/yahoo/searchdefinition/FieldSets.java | 78 - .../yahoo/searchdefinition/ImmutableSchema.java | 53 - .../searchdefinition/ImportedFieldsEnumerator.java | 31 - .../java/com/yahoo/searchdefinition/Index.java | 200 --- .../searchdefinition/LargeRankExpressions.java | 51 - .../searchdefinition/MapEvaluationTypeContext.java | 361 ----- .../java/com/yahoo/searchdefinition/OnnxModel.java | 120 -- .../yahoo/searchdefinition/RankExpressionBody.java | 32 - .../com/yahoo/searchdefinition/RankProfile.java | 1526 -------------------- .../searchdefinition/RankProfileRegistry.java | 138 -- .../searchdefinition/SDDocumentTypeOrderer.java | 136 -- .../java/com/yahoo/searchdefinition/Schema.java | 756 ---------- .../searchdefinition/TemporarySDTypeResolver.java | 79 - .../searchdefinition/UnrankedRankProfile.java | 29 - .../searchdefinition/derived/AttributeFields.java | 324 ----- .../yahoo/searchdefinition/derived/Derived.java | 141 -- .../derived/DerivedConfiguration.java | 212 --- .../yahoo/searchdefinition/derived/Deriver.java | 48 - .../yahoo/searchdefinition/derived/Exportable.java | 26 - .../derived/FieldRankSettings.java | 75 - .../derived/FieldResultTransform.java | 57 - .../derived/FileDistributedConstants.java | 87 -- .../derived/FileDistributedOnnxModels.java | 63 - .../searchdefinition/derived/ImportedFields.java | 106 -- .../com/yahoo/searchdefinition/derived/Index.java | 64 - .../yahoo/searchdefinition/derived/IndexInfo.java | 595 -------- .../searchdefinition/derived/IndexSchema.java | 245 ---- .../searchdefinition/derived/IndexingScript.java | 198 --- .../yahoo/searchdefinition/derived/Juniperrc.java | 62 - .../derived/NativeRankTypeDefinition.java | 44 - .../derived/NativeRankTypeDefinitionSet.java | 93 -- .../searchdefinition/derived/NativeTable.java | 72 - .../searchdefinition/derived/RankProfileList.java | 210 --- .../searchdefinition/derived/RawRankProfile.java | 524 ------- .../yahoo/searchdefinition/derived/SchemaInfo.java | 134 -- .../searchdefinition/derived/SearchOrderer.java | 123 -- .../yahoo/searchdefinition/derived/Summaries.java | 51 - .../searchdefinition/derived/SummaryClass.java | 133 -- .../derived/SummaryClassField.java | 132 -- .../yahoo/searchdefinition/derived/SummaryMap.java | 121 -- .../yahoo/searchdefinition/derived/VsmFields.java | 313 ---- .../yahoo/searchdefinition/derived/VsmSummary.java | 109 -- .../searchdefinition/derived/package-info.java | 5 - .../validation/IndexStructureValidator.java | 50 - .../derived/validation/Validation.java | 12 - .../derived/validation/Validator.java | 22 - .../yahoo/searchdefinition/document/Attribute.java | 435 ------ .../document/BooleanIndexDefinition.java | 84 -- .../com/yahoo/searchdefinition/document/Case.java | 15 - .../document/ComplexAttributeFieldUtils.java | 124 -- .../searchdefinition/document/Dictionary.java | 35 - .../yahoo/searchdefinition/document/FieldSet.java | 41 - .../yahoo/searchdefinition/document/GeoPos.java | 26 - .../searchdefinition/document/HnswIndexParams.java | 77 - .../document/ImmutableImportedComplexSDField.java | 29 - .../document/ImmutableImportedSDField.java | 218 --- .../document/ImmutableSDField.java | 91 -- .../document/ImportedComplexField.java | 49 - .../searchdefinition/document/ImportedField.java | 38 - .../searchdefinition/document/ImportedFields.java | 25 - .../document/ImportedSimpleField.java | 18 - .../searchdefinition/document/MatchAlgorithm.java | 16 - .../yahoo/searchdefinition/document/MatchType.java | 14 - .../yahoo/searchdefinition/document/Matching.java | 141 -- .../searchdefinition/document/NormalizeLevel.java | 87 -- .../yahoo/searchdefinition/document/RankType.java | 40 - .../yahoo/searchdefinition/document/Ranking.java | 76 - .../searchdefinition/document/SDDocumentType.java | 347 ----- .../yahoo/searchdefinition/document/SDField.java | 802 ---------- .../yahoo/searchdefinition/document/Sorting.java | 65 - .../yahoo/searchdefinition/document/Stemming.java | 64 - .../document/TemporaryImportedField.java | 37 - .../document/TemporaryImportedFields.java | 44 - .../document/TemporarySDDocumentType.java | 13 - .../document/TemporarySDField.java | 19 - .../yahoo/searchdefinition/document/TypedKey.java | 20 - .../document/annotation/SDAnnotationType.java | 42 - .../TemporaryAnnotationReferenceDataType.java | 28 - .../ConstantTensorTransformer.java | 71 - .../expressiontransforms/ExpressionTransforms.java | 47 - .../expressiontransforms/FunctionInliner.java | 33 - .../expressiontransforms/FunctionShadower.java | 59 - .../LightGBMFeatureConverter.java | 59 - .../expressiontransforms/OnnxFeatureConverter.java | 64 - .../expressiontransforms/OnnxModelTransformer.java | 140 -- .../RankProfileTransformContext.java | 65 - .../TensorFlowFeatureConverter.java | 66 - .../expressiontransforms/TokenTransformer.java | 313 ---- .../XgboostFeatureConverter.java | 61 - .../fieldoperation/AliasOperation.java | 42 - .../fieldoperation/AttributeOperation.java | 172 --- .../fieldoperation/BoldingOperation.java | 25 - .../fieldoperation/DictionaryOperation.java | 41 - .../fieldoperation/FieldOperation.java | 22 - .../fieldoperation/FieldOperationContainer.java | 19 - .../fieldoperation/IdOperation.java | 35 - .../fieldoperation/IndexOperation.java | 134 -- .../fieldoperation/IndexingOperation.java | 61 - .../fieldoperation/MatchOperation.java | 66 - .../fieldoperation/NormalizingOperation.java | 34 - .../fieldoperation/QueryCommandOperation.java | 25 - .../fieldoperation/RankOperation.java | 36 - .../fieldoperation/RankTypeOperation.java | 43 - .../fieldoperation/SortingOperation.java | 93 -- .../fieldoperation/StemmingOperation.java | 26 - .../fieldoperation/StructFieldOperation.java | 56 - .../SummaryInFieldLongOperation.java | 72 - .../fieldoperation/SummaryInFieldOperation.java | 46 - .../SummaryInFieldShortOperation.java | 32 - .../fieldoperation/SummaryToOperation.java | 41 - .../fieldoperation/WeightOperation.java | 25 - .../fieldoperation/WeightedSetOperation.java | 70 - .../parser/ConvertParsedFields.java | 331 ----- .../parser/ConvertParsedRanking.java | 124 -- .../parser/ConvertParsedSchemas.java | 221 --- .../parser/ConvertParsedTypes.java | 337 ----- .../parser/ConvertSchemaCollection.java | 212 --- .../searchdefinition/parser/DictionaryOption.java | 5 - .../parser/InheritanceResolver.java | 130 -- .../parser/IntermediateCollection.java | 161 --- .../searchdefinition/parser/ParsedAnnotation.java | 57 - .../searchdefinition/parser/ParsedAttribute.java | 69 - .../yahoo/searchdefinition/parser/ParsedBlock.java | 36 - .../searchdefinition/parser/ParsedDocument.java | 129 -- .../parser/ParsedDocumentSummary.java | 49 - .../yahoo/searchdefinition/parser/ParsedField.java | 159 -- .../searchdefinition/parser/ParsedFieldSet.java | 36 - .../yahoo/searchdefinition/parser/ParsedIndex.java | 79 - .../searchdefinition/parser/ParsedIndexingOp.java | 37 - .../parser/ParsedMatchSettings.java | 38 - .../parser/ParsedRankFunction.java | 39 - .../searchdefinition/parser/ParsedRankProfile.java | 223 --- .../searchdefinition/parser/ParsedSchema.java | 176 --- .../searchdefinition/parser/ParsedSorting.java | 48 - .../searchdefinition/parser/ParsedStruct.java | 60 - .../parser/ParsedSummaryField.java | 51 - .../yahoo/searchdefinition/parser/ParsedType.java | 224 --- .../searchdefinition/parser/SimpleCharStream.java | 16 - .../com/yahoo/searchdefinition/parser/Utils.java | 15 - ...ttributeTransformToSummaryOfImportedFields.java | 61 - .../processing/AddExtraFieldsToDocument.java | 92 -- .../processing/AdjustPositionSummaryFields.java | 135 -- .../processing/AttributeProperties.java | 72 - .../processing/AttributesImplicitWord.java | 58 - .../yahoo/searchdefinition/processing/Bolding.java | 47 - .../processing/BoolAttributeValidator.java | 36 - .../processing/BuiltInFieldSets.java | 52 - .../processing/CreatePositionZCurve.java | 216 --- .../processing/DictionaryProcessor.java | 54 - .../DisallowComplexMapAndWsetKeyTypes.java | 57 - .../processing/DiversitySettingsValidator.java | 65 - .../searchdefinition/processing/ExactMatch.java | 110 -- .../processing/FastAccessValidator.java | 54 - .../processing/FieldSetSettings.java | 107 -- .../processing/FilterFieldNames.java | 72 - .../processing/ImplicitSummaries.java | 232 --- .../processing/ImplicitSummaryFields.java | 40 - .../processing/ImportedFieldsResolver.java | 207 --- .../processing/IndexFieldNames.java | 47 - .../processing/IndexingInputs.java | 106 -- .../processing/IndexingOutputs.java | 144 -- .../processing/IndexingValidation.java | 164 --- .../processing/IndexingValues.java | 71 - .../processing/IntegerIndex2Attribute.java | 88 -- .../searchdefinition/processing/LiteralBoost.java | 79 - .../searchdefinition/processing/MakeAliases.java | 61 - .../processing/MakeDefaultSummaryTheSuperSet.java | 49 - .../processing/MatchConsistency.java | 73 - .../processing/MatchPhaseSettingsValidator.java | 98 -- .../processing/MatchedElementsOnlyResolver.java | 95 -- .../processing/MultifieldIndexHarmonizer.java | 76 - .../processing/MutableAttributes.java | 29 - .../searchdefinition/processing/NGramMatch.java | 79 - .../processing/OnnxModelConfigGenerator.java | 96 -- .../processing/OnnxModelTypeResolver.java | 40 - .../processing/OptimizeIlscript.java | 38 - .../processing/PagedAttributeValidator.java | 66 - .../processing/PredicateProcessor.java | 144 -- .../searchdefinition/processing/Processing.java | 152 -- .../searchdefinition/processing/Processor.java | 157 -- .../processing/RankingExpressionTypeResolver.java | 135 -- .../processing/ReferenceFieldsProcessor.java | 65 - .../processing/ReservedDocumentNames.java | 39 - .../processing/ReservedFunctionNames.java | 56 - .../processing/SearchMustHaveDocument.java | 30 - .../processing/SetRankTypeEmptyOnFilters.java | 30 - .../processing/SortingSettings.java | 42 - .../StringSettingsOnNonStringFields.java | 43 - .../processing/SummaryConsistency.java | 131 -- .../processing/SummaryDiskAccessValidator.java | 73 - .../processing/SummaryDynamicStructsArrays.java | 47 - .../SummaryFieldsMustHaveValidSource.java | 81 -- .../processing/SummaryNamesFieldCollisions.java | 60 - .../yahoo/searchdefinition/processing/TagType.java | 47 - .../processing/TensorFieldProcessor.java | 118 -- .../searchdefinition/processing/TextMatch.java | 128 -- .../processing/TypedTransformProvider.java | 62 - .../yahoo/searchdefinition/processing/UriHack.java | 77 - .../processing/UrlFieldValidator.java | 34 - .../processing/ValidateFieldTypes.java | 83 -- .../ValidateFieldTypesDocumentsOnly.java | 26 - ...ValidateFieldWithIndexSettingsCreatesIndex.java | 42 - .../processing/ValidateStructTypeInheritance.java | 76 - .../searchdefinition/processing/WordMatch.java | 52 - .../multifieldresolver/IndexCommandResolver.java | 62 - .../multifieldresolver/MultiFieldResolver.java | 33 - .../RankProfileTypeSettingsProcessor.java | 105 -- .../multifieldresolver/RankTypeResolver.java | 46 - .../multifieldresolver/StemmingResolver.java | 43 - .../searchdefinition/processing/package-info.java | 14 - .../configmodel/producers/DataTypeRecognizer.java | 3 +- .../configmodel/producers/DocumentManager.java | 7 +- .../vespa/configmodel/producers/DocumentTypes.java | 3 +- .../yahoo/vespa/documentmodel/DocumentSummary.java | 4 +- .../yahoo/vespa/documentmodel/SummaryField.java | 2 +- .../java/com/yahoo/vespa/model/VespaModel.java | 18 +- .../ComplexAttributeFieldsValidator.java | 10 +- .../application/validation/ConstantValidator.java | 6 +- .../application/validation/NoPrefixForIndexes.java | 10 +- .../application/validation/RankSetupValidator.java | 2 +- .../validation/SearchDataTypeValidator.java | 8 +- .../application/validation/StreamingValidator.java | 6 +- .../StreamingSearchClusterChangeValidator.java | 4 +- .../change/search/AttributeChangeValidator.java | 12 +- .../search/IndexingScriptChangeMessageBuilder.java | 12 +- .../search/IndexingScriptChangeValidator.java | 4 +- .../StructFieldAttributeChangeValidator.java | 6 +- .../model/container/ContainerModelEvaluation.java | 2 +- .../model/container/ml/ModelsEvaluatorTester.java | 2 +- .../model/container/search/ContainerSearch.java | 2 +- .../search/searchchain/LocalProvider.java | 2 +- .../model/container/xml/ContainerModelBuilder.java | 4 +- .../vespa/model/content/ContentSearchCluster.java | 4 +- .../com/yahoo/vespa/model/ml/ConvertedModel.java | 6 +- .../yahoo/vespa/model/search/DocumentDatabase.java | 2 +- .../vespa/model/search/IndexedSearchCluster.java | 6 +- .../model/search/SchemaDefinitionXMLHandler.java | 3 +- .../yahoo/vespa/model/search/SearchCluster.java | 9 +- .../vespa/model/search/StreamingSearchCluster.java | 6 +- config-model/src/main/javacc/SchemaParser.jj | 32 +- .../yahoo/config/model/ApplicationDeployTest.java | 4 +- .../config/model/ApplicationPackageTester.java | 2 +- .../document/test/SDDocumentTypeTestCase.java | 10 +- .../com/yahoo/document/test/SDFieldTestCase.java | 4 +- .../com/yahoo/schema/AbstractSchemaTestCase.java | 82 ++ .../yahoo/schema/AnnotationReferenceTestCase.java | 67 + .../test/java/com/yahoo/schema/ArraysTestCase.java | 35 + .../yahoo/schema/ArraysWeightedSetsTestCase.java | 41 + .../yahoo/schema/AttributeSettingsTestCase.java | 344 +++++ .../test/java/com/yahoo/schema/AttributeUtils.java | 15 + .../java/com/yahoo/schema/CommentTestCase.java | 27 + .../java/com/yahoo/schema/DiversityTestCase.java | 110 ++ .../yahoo/schema/DocumentGraphValidatorTest.java | 166 +++ .../schema/DocumentReferenceResolverTest.java | 103 ++ .../com/yahoo/schema/FeatureNamesTestCase.java | 94 ++ .../yahoo/schema/FieldOfTypeDocumentTestCase.java | 57 + .../yahoo/schema/ImportedFieldsEnumeratorTest.java | 73 + .../IncorrectRankingExpressionFileRefTestCase.java | 36 + .../schema/IncorrectSummaryTypesTestCase.java | 36 + .../com/yahoo/schema/IndexSettingsTestCase.java | 62 + .../com/yahoo/schema/IndexingParsingTestCase.java | 32 + .../yahoo/schema/MultipleSummariesTestCase.java | 24 + .../com/yahoo/schema/NameFieldCheckTestCase.java | 80 + .../java/com/yahoo/schema/OutsideTestCase.java | 32 + .../yahoo/schema/PredicateDataTypeTestCase.java | 199 +++ .../com/yahoo/schema/RankProfileRegistryTest.java | 58 + .../java/com/yahoo/schema/RankProfileTestCase.java | 436 ++++++ .../com/yahoo/schema/RankPropertiesTestCase.java | 163 +++ .../java/com/yahoo/schema/RankingConstantTest.java | 213 +++ .../schema/RankingExpressionConstantsTestCase.java | 229 +++ .../schema/RankingExpressionInliningTestCase.java | 274 ++++ .../RankingExpressionLoopDetectionTestCase.java | 246 ++++ .../schema/RankingExpressionShadowingTestCase.java | 251 ++++ .../RankingExpressionValidationTestCase.java | 52 + .../schema/ReservedWordsAsFieldNamesTestCase.java | 24 + .../schema/SDDocumentTypeOrdererTestCase.java | 78 + .../com/yahoo/schema/SchemaImporterTestCase.java | 189 +++ .../com/yahoo/schema/SchemaParsingTestCase.java | 83 ++ .../test/java/com/yahoo/schema/SchemaTestCase.java | 443 ++++++ .../com/yahoo/schema/StemmingSettingTestCase.java | 51 + .../test/java/com/yahoo/schema/StructTestCase.java | 59 + .../java/com/yahoo/schema/SummaryTestCase.java | 286 ++++ .../yahoo/schema/UrlFieldValidationTestCase.java | 34 + .../schema/derived/AbstractExportingTestCase.java | 165 +++ .../yahoo/schema/derived/AnnotationsTestCase.java | 66 + .../com/yahoo/schema/derived/ArraysTestCase.java | 23 + .../schema/derived/AttributeListTestCase.java | 129 ++ .../yahoo/schema/derived/AttributesTestCase.java | 36 + .../com/yahoo/schema/derived/CasingTestCase.java | 36 + .../CombinedAttributeAndIndexSchemaTestCase.java | 21 + .../com/yahoo/schema/derived/DeriverTestCase.java | 31 + .../schema/derived/DuplicateStructTestCase.java | 19 + .../schema/derived/EmptyRankProfileTestCase.java | 38 + .../yahoo/schema/derived/ExactMatchTestCase.java | 17 + .../yahoo/schema/derived/ExportingTestCase.java | 195 +++ .../schema/derived/ExpressionsAsArgsTestCase.java | 25 + .../com/yahoo/schema/derived/FieldsetTestCase.java | 16 + .../com/yahoo/schema/derived/GeminiTestCase.java | 70 + .../java/com/yahoo/schema/derived/IdTestCase.java | 49 + .../schema/derived/ImportedFieldsTestCase.java | 42 + .../yahoo/schema/derived/IndexSchemaTestCase.java | 208 +++ .../yahoo/schema/derived/InheritanceTestCase.java | 192 +++ .../IntegerAttributeToStringIndexTestCase.java | 17 + .../yahoo/schema/derived/LiteralBoostTestCase.java | 113 ++ .../yahoo/schema/derived/LowercaseTestCase.java | 19 + .../com/yahoo/schema/derived/MailTestCase.java | 24 + .../derived/MatchSettingsResolvingTestCase.java | 63 + .../yahoo/schema/derived/MultiStructTestCase.java | 27 + .../schema/derived/MultipleSummariesTestCase.java | 22 + .../schema/derived/NameCollisionTestCase.java | 27 + .../derived/NativeRankTypeDefinitionsTestCase.java | 92 ++ .../schema/derived/NearestNeighborTestCase.java | 39 + .../yahoo/schema/derived/NeuralNetTestCase.java | 42 + .../com/yahoo/schema/derived/NuwaTestCase.java | 33 + .../schema/derived/OrderIlscriptsTestCase.java | 19 + .../derived/PrefixExactAttributeTestCase.java | 21 + .../yahoo/schema/derived/RankProfilesTestCase.java | 20 + .../schema/derived/RankPropertiesTestCase.java | 19 + .../schema/derived/ReferenceFieldsTestCase.java | 18 + .../derived/ReferenceFromSeveralTestCase.java | 27 + .../schema/derived/SchemaInheritanceTestCase.java | 34 + .../schema/derived/SchemaOrdererTestCase.java | 146 ++ .../schema/derived/SimpleInheritTestCase.java | 47 + .../com/yahoo/schema/derived/SliceTestCase.java | 23 + .../com/yahoo/schema/derived/SortingTestCase.java | 22 + .../schema/derived/StreamingStructTestCase.java | 26 + .../schema/derived/StructAnyOrderTestCase.java | 17 + .../schema/derived/StructInheritanceTestCase.java | 52 + .../yahoo/schema/derived/SummaryMapTestCase.java | 187 +++ .../com/yahoo/schema/derived/SummaryTestCase.java | 182 +++ .../yahoo/schema/derived/TestableDeployLogger.java | 30 + .../yahoo/schema/derived/TokenizationTestCase.java | 19 + .../derived/TwoStreamingStructsTestCase.java | 34 + .../schema/derived/TypeConversionTestCase.java | 44 + .../com/yahoo/schema/derived/TypesTestCase.java | 21 + .../yahoo/schema/derived/VsmFieldsTestCase.java | 42 + .../ComplexAttributeFieldUtilsTestCase.java | 244 ++++ .../schema/document/HnswIndexParamsTestCase.java | 51 + .../schema/parser/ConvertIntermediateTestCase.java | 95 ++ .../parser/IntermediateCollectionTestCase.java | 236 +++ .../schema/parser/ParsedDocumentTestCase.java | 30 + .../yahoo/schema/parser/SchemaParserTestCase.java | 276 ++++ ...buteTransformToSummaryOfImportedFieldsTest.java | 75 + .../AdjustPositionSummaryFieldsTestCase.java | 260 ++++ .../schema/processing/AssertIndexingScript.java | 43 + .../schema/processing/AssertSearchBuilder.java | 29 + .../processing/AttributesExactMatchTestCase.java | 40 + .../yahoo/schema/processing/BoldingTestCase.java | 65 + .../processing/BoolAttributeValidatorTestCase.java | 50 + .../schema/processing/DictionaryTestCase.java | 250 ++++ .../DisallowComplexMapAndWsetKeyTypesTestCase.java | 57 + .../schema/processing/FastAccessValidatorTest.java | 61 + .../processing/ImplicitSchemaFieldsTestCase.java | 94 ++ .../processing/ImplicitStructTypesTestCase.java | 69 + .../processing/ImplicitSummariesTestCase.java | 78 + .../processing/ImplicitSummaryFieldsTestCase.java | 29 + .../processing/ImportedFieldsResolverTestCase.java | 152 ++ .../schema/processing/ImportedFieldsTestCase.java | 529 +++++++ .../schema/processing/IndexingInputsTestCase.java | 45 + .../schema/processing/IndexingOutputsTestCase.java | 30 + .../processing/IndexingScriptRewriterTestCase.java | 200 +++ .../processing/IndexingValidationTestCase.java | 76 + .../schema/processing/IndexingValuesTestCase.java | 30 + .../processing/IntegerIndex2AttributeTestCase.java | 61 + .../MatchPhaseSettingsValidatorTestCase.java | 37 + .../MatchedElementsOnlyResolverTestCase.java | 192 +++ .../com/yahoo/schema/processing/NGramTestCase.java | 88 ++ .../PagedAttributeValidatorTestCase.java | 119 ++ .../schema/processing/ParentChildSearchModel.java | 64 + .../yahoo/schema/processing/PositionTestCase.java | 130 ++ .../schema/processing/RankModifierTestCase.java | 22 + .../processing/RankProfileSearchFixture.java | 128 ++ .../processing/RankPropertyVariablesTestCase.java | 47 + .../RankingExpressionTypeResolverTestCase.java | 521 +++++++ .../RankingExpressionWithLightGBMTestCase.java | 88 ++ .../RankingExpressionWithOnnxModelTestCase.java | 184 +++ .../RankingExpressionWithOnnxTestCase.java | 417 ++++++ .../RankingExpressionWithTensorTestCase.java | 202 +++ ...ingExpressionWithTransformerTokensTestCase.java | 98 ++ .../RankingExpressionWithXGBoostTestCase.java | 90 ++ .../processing/RankingExpressionsTestCase.java | 128 ++ .../schema/processing/ReferenceFieldTestCase.java | 92 ++ .../processing/ReservedDocumentNamesTestCase.java | 27 + ...rvedRankingExpressionFunctionNamesTestCase.java | 71 + .../processing/SchemaMustHaveDocumentTest.java | 30 + .../processing/SummaryConsistencyTestCase.java | 45 + .../SummaryFieldsMustHaveValidSourceTestCase.java | 60 + .../schema/processing/TensorFieldTestCase.java | 172 +++ .../schema/processing/TensorTransformTestCase.java | 234 +++ .../schema/processing/ValidateFieldTypesTest.java | 80 + .../schema/processing/VespaMlModelTestCase.java | 77 + .../processing/WeightedSetSummaryToTestCase.java | 23 + .../searchdefinition/AbstractSchemaTestCase.java | 82 -- .../AnnotationReferenceTestCase.java | 67 - .../com/yahoo/searchdefinition/ArraysTestCase.java | 35 - .../ArraysWeightedSetsTestCase.java | 41 - .../AttributeSettingsTestCase.java | 344 ----- .../com/yahoo/searchdefinition/AttributeUtils.java | 15 - .../yahoo/searchdefinition/CommentTestCase.java | 27 - .../yahoo/searchdefinition/DiversityTestCase.java | 110 -- .../DocumentGraphValidatorTest.java | 166 --- .../DocumentReferenceResolverTest.java | 103 -- .../searchdefinition/FeatureNamesTestCase.java | 94 -- .../FieldOfTypeDocumentTestCase.java | 57 - .../ImportedFieldsEnumeratorTest.java | 74 - .../IncorrectRankingExpressionFileRefTestCase.java | 36 - .../IncorrectSummaryTypesTestCase.java | 36 - .../searchdefinition/IndexSettingsTestCase.java | 62 - .../searchdefinition/IndexingParsingTestCase.java | 32 - .../MultipleSummariesTestCase.java | 24 - .../searchdefinition/NameFieldCheckTestCase.java | 80 - .../yahoo/searchdefinition/OutsideTestCase.java | 32 - .../PredicateDataTypeTestCase.java | 199 --- .../searchdefinition/RankProfileRegistryTest.java | 58 - .../searchdefinition/RankProfileTestCase.java | 436 ------ .../searchdefinition/RankPropertiesTestCase.java | 163 --- .../searchdefinition/RankingConstantTest.java | 213 --- .../RankingExpressionConstantsTestCase.java | 229 --- .../RankingExpressionInliningTestCase.java | 274 ---- .../RankingExpressionLoopDetectionTestCase.java | 246 ---- .../RankingExpressionShadowingTestCase.java | 251 ---- .../RankingExpressionValidationTestCase.java | 52 - .../ReservedWordsAsFieldNamesTestCase.java | 24 - .../SDDocumentTypeOrdererTestCase.java | 78 - .../searchdefinition/SchemaImporterTestCase.java | 189 --- .../searchdefinition/SchemaParsingTestCase.java | 83 -- .../com/yahoo/searchdefinition/SchemaTestCase.java | 443 ------ .../searchdefinition/StemmingSettingTestCase.java | 51 - .../com/yahoo/searchdefinition/StructTestCase.java | 59 - .../yahoo/searchdefinition/SummaryTestCase.java | 286 ---- .../UrlFieldValidationTestCase.java | 34 - .../derived/AbstractExportingTestCase.java | 167 --- .../derived/AnnotationsTestCase.java | 66 - .../searchdefinition/derived/ArraysTestCase.java | 23 - .../derived/AttributeListTestCase.java | 129 -- .../derived/AttributesTestCase.java | 36 - .../searchdefinition/derived/CasingTestCase.java | 36 - .../CombinedAttributeAndIndexSchemaTestCase.java | 21 - .../searchdefinition/derived/DeriverTestCase.java | 31 - .../derived/DuplicateStructTestCase.java | 19 - .../derived/EmptyRankProfileTestCase.java | 38 - .../derived/ExactMatchTestCase.java | 17 - .../derived/ExportingTestCase.java | 195 --- .../derived/ExpressionsAsArgsTestCase.java | 25 - .../searchdefinition/derived/FieldsetTestCase.java | 16 - .../searchdefinition/derived/GeminiTestCase.java | 70 - .../yahoo/searchdefinition/derived/IdTestCase.java | 49 - .../derived/ImportedFieldsTestCase.java | 42 - .../derived/IndexSchemaTestCase.java | 208 --- .../derived/InheritanceTestCase.java | 192 --- .../IntegerAttributeToStringIndexTestCase.java | 17 - .../derived/LiteralBoostTestCase.java | 113 -- .../derived/LowercaseTestCase.java | 19 - .../searchdefinition/derived/MailTestCase.java | 24 - .../derived/MatchSettingsResolvingTestCase.java | 63 - .../derived/MultiStructTestCase.java | 28 - .../derived/MultipleSummariesTestCase.java | 22 - .../derived/NameCollisionTestCase.java | 28 - .../derived/NativeRankTypeDefinitionsTestCase.java | 92 -- .../derived/NearestNeighborTestCase.java | 39 - .../derived/NeuralNetTestCase.java | 43 - .../searchdefinition/derived/NuwaTestCase.java | 33 - .../derived/OrderIlscriptsTestCase.java | 19 - .../derived/PrefixExactAttributeTestCase.java | 21 - .../derived/RankProfilesTestCase.java | 20 - .../derived/RankPropertiesTestCase.java | 19 - .../derived/ReferenceFieldsTestCase.java | 18 - .../derived/ReferenceFromSeveralTestCase.java | 28 - .../derived/SchemaInheritanceTestCase.java | 34 - .../derived/SchemaOrdererTestCase.java | 146 -- .../derived/SimpleInheritTestCase.java | 47 - .../searchdefinition/derived/SliceTestCase.java | 27 - .../searchdefinition/derived/SortingTestCase.java | 22 - .../derived/StreamingStructTestCase.java | 26 - .../derived/StructAnyOrderTestCase.java | 17 - .../derived/StructInheritanceTestCase.java | 52 - .../derived/SummaryMapTestCase.java | 187 --- .../searchdefinition/derived/SummaryTestCase.java | 182 --- .../derived/TestableDeployLogger.java | 30 - .../derived/TokenizationTestCase.java | 19 - .../derived/TwoStreamingStructsTestCase.java | 34 - .../derived/TypeConversionTestCase.java | 44 - .../searchdefinition/derived/TypesTestCase.java | 21 - .../derived/VsmFieldsTestCase.java | 43 - .../ComplexAttributeFieldUtilsTestCase.java | 244 ---- .../document/HnswIndexParamsTestCase.java | 51 - .../parser/ConvertIntermediateTestCase.java | 97 -- .../parser/IntermediateCollectionTestCase.java | 237 --- .../parser/ParsedDocumentTestCase.java | 30 - .../parser/SchemaParserTestCase.java | 276 ---- ...buteTransformToSummaryOfImportedFieldsTest.java | 76 - .../AdjustPositionSummaryFieldsTestCase.java | 260 ---- .../processing/AssertIndexingScript.java | 43 - .../processing/AssertSearchBuilder.java | 29 - .../processing/AttributesExactMatchTestCase.java | 41 - .../processing/BoldingTestCase.java | 65 - .../processing/BoolAttributeValidatorTestCase.java | 50 - .../processing/DictionaryTestCase.java | 250 ---- .../DisallowComplexMapAndWsetKeyTypesTestCase.java | 57 - .../processing/FastAccessValidatorTest.java | 61 - .../processing/ImplicitSchemaFieldsTestCase.java | 94 -- .../processing/ImplicitStructTypesTestCase.java | 69 - .../processing/ImplicitSummariesTestCase.java | 78 - .../processing/ImplicitSummaryFieldsTestCase.java | 29 - .../processing/ImportedFieldsResolverTestCase.java | 152 -- .../processing/ImportedFieldsTestCase.java | 529 ------- .../processing/IndexingInputsTestCase.java | 45 - .../processing/IndexingOutputsTestCase.java | 30 - .../processing/IndexingScriptRewriterTestCase.java | 200 --- .../processing/IndexingValidationTestCase.java | 76 - .../processing/IndexingValuesTestCase.java | 30 - .../processing/IntegerIndex2AttributeTestCase.java | 61 - .../MatchPhaseSettingsValidatorTestCase.java | 37 - .../MatchedElementsOnlyResolverTestCase.java | 192 --- .../searchdefinition/processing/NGramTestCase.java | 89 -- .../PagedAttributeValidatorTestCase.java | 119 -- .../processing/ParentChildSearchModel.java | 65 - .../processing/PositionTestCase.java | 130 -- .../processing/RankModifierTestCase.java | 22 - .../processing/RankProfileSearchFixture.java | 128 -- .../processing/RankPropertyVariablesTestCase.java | 47 - .../RankingExpressionTypeResolverTestCase.java | 521 ------- .../RankingExpressionWithLightGBMTestCase.java | 88 -- .../RankingExpressionWithOnnxModelTestCase.java | 184 --- .../RankingExpressionWithOnnxTestCase.java | 417 ------ .../RankingExpressionWithTensorTestCase.java | 202 --- ...ingExpressionWithTransformerTokensTestCase.java | 98 -- .../RankingExpressionWithXGBoostTestCase.java | 90 -- .../processing/RankingExpressionsTestCase.java | 128 -- .../processing/ReferenceFieldTestCase.java | 92 -- .../processing/ReservedDocumentNamesTestCase.java | 27 - ...rvedRankingExpressionFunctionNamesTestCase.java | 71 - .../processing/SchemaMustHaveDocumentTest.java | 30 - .../processing/SummaryConsistencyTestCase.java | 45 - .../SummaryFieldsMustHaveValidSourceTestCase.java | 60 - .../processing/TensorFieldTestCase.java | 172 --- .../processing/TensorTransformTestCase.java | 234 --- .../processing/ValidateFieldTypesTest.java | 81 -- .../processing/VespaMlModelTestCase.java | 77 - .../processing/WeightedSetSummaryToTestCase.java | 23 - .../AbstractReferenceFieldTestCase.java | 2 +- ...DocumentModelBuilderImportedFieldsTestCase.java | 4 +- .../DocumentModelBuilderReferenceTypeTestCase.java | 4 +- .../DocumentModelBuilderTestCase.java | 6 +- .../search/DocumentTypeChangeValidatorTest.java | 2 +- .../search/searchchain/MockSearchClusters.java | 2 +- .../search/test/QueryProfilesTestCase.java | 3 +- .../vespa/model/search/test/SchemaClusterTest.java | 10 +- .../yahoo/vespa/config/DocumentGenPluginTest.java | 3 +- .../main/java/com/yahoo/vespa/DocumentGenMojo.java | 8 +- .../test/java/com/yahoo/vespa/DocumentGenTest.java | 2 +- .../restapi/resource/DocumentV1ApiTest.java | 2 +- 799 files changed, 38657 insertions(+), 38751 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/schema/Application.java create mode 100644 config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DefaultRankProfile.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DistributableResource.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentGraphValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentOnlySchema.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentReference.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentReferenceResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentReferences.java create mode 100644 config-model/src/main/java/com/yahoo/schema/DocumentsOnlyRankProfile.java create mode 100644 config-model/src/main/java/com/yahoo/schema/FeatureNames.java create mode 100644 config-model/src/main/java/com/yahoo/schema/FieldOperationApplier.java create mode 100644 config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForSearch.java create mode 100644 config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForStructs.java create mode 100644 config-model/src/main/java/com/yahoo/schema/FieldSets.java create mode 100644 config-model/src/main/java/com/yahoo/schema/ImmutableSchema.java create mode 100644 config-model/src/main/java/com/yahoo/schema/ImportedFieldsEnumerator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/Index.java create mode 100644 config-model/src/main/java/com/yahoo/schema/LargeRankExpressions.java create mode 100644 config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java create mode 100644 config-model/src/main/java/com/yahoo/schema/OnnxModel.java create mode 100644 config-model/src/main/java/com/yahoo/schema/RankExpressionBody.java create mode 100644 config-model/src/main/java/com/yahoo/schema/RankProfile.java create mode 100644 config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java create mode 100644 config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/Schema.java create mode 100644 config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/UnrankedRankProfile.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/Derived.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/Deriver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/Exportable.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/FieldRankSettings.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/FieldResultTransform.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/Index.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinition.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinitionSet.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/NativeTable.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/Summaries.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/package-info.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/validation/IndexStructureValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/validation/Validation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/derived/validation/Validator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Attribute.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Case.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Dictionary.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/FieldSet.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/GeoPos.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImportedField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/MatchType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Matching.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/RankType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Ranking.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/SDField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Sorting.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/Stemming.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/TypedKey.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/ConstantTensorTransformer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionInliner.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionShadower.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/LightGBMFeatureConverter.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxFeatureConverter.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/TokenTransformer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/expressiontransforms/XgboostFeatureConverter.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/AliasOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/AttributeOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/BoldingOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/DictionaryOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperationContainer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/IdOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/MatchOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/NormalizingOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/QueryCommandOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/RankOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/RankTypeOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/SortingOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/StemmingOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/StructFieldOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldLongOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldShortOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryToOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightedSetOperation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/DictionaryOption.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/InheritanceResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedAnnotation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedBlock.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedDocument.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedFieldSet.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedIndex.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedIndexingOp.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedRankFunction.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedSchema.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedSorting.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/SimpleCharStream.java create mode 100644 config-model/src/main/java/com/yahoo/schema/parser/Utils.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/AttributeProperties.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/Bolding.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/BuiltInFieldSets.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypes.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/FastAccessValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/FieldSetSettings.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/FilterFieldNames.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaryFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/IndexingValues.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/LiteralBoost.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MakeAliases.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MatchConsistency.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MatchedElementsOnlyResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MultifieldIndexHarmonizer.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/MutableAttributes.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/OnnxModelConfigGenerator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/OnnxModelTypeResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/Processing.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/Processor.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ReservedDocumentNames.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SearchMustHaveDocument.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SetRankTypeEmptyOnFilters.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SortingSettings.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/StringSettingsOnNonStringFields.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SummaryDiskAccessValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSource.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SummaryNamesFieldCollisions.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/TagType.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/UriHack.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/UrlFieldValidator.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypesDocumentsOnly.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldWithIndexSettingsCreatesIndex.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/ValidateStructTypeInheritance.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/WordMatch.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/package-info.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/Application.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentGraphValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentReference.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/Index.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/Schema.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/TemporarySDTypeResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedOnnxModels.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/XgboostFeatureConverter.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/DictionaryOption.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndexingOp.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSummaryField.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelConfigGenerator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelTypeResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateStructTypeInheritance.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java create mode 100644 config-model/src/test/java/com/yahoo/schema/AbstractSchemaTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/AnnotationReferenceTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/ArraysTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/ArraysWeightedSetsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/AttributeUtils.java create mode 100644 config-model/src/test/java/com/yahoo/schema/CommentTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/FeatureNamesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/FieldOfTypeDocumentTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/IncorrectRankingExpressionFileRefTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/IncorrectSummaryTypesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/IndexSettingsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/IndexingParsingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/MultipleSummariesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/NameFieldCheckTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/OutsideTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/PredicateDataTypeTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankProfileRegistryTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankingConstantTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankingExpressionConstantsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankingExpressionLoopDetectionTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankingExpressionShadowingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/RankingExpressionValidationTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/ReservedWordsAsFieldNamesTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/SchemaParsingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/StemmingSettingTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/StructTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/UrlFieldValidationTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/derived/AnnotationsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ArraysTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/AttributesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/CasingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/CombinedAttributeAndIndexSchemaTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/DeriverTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/DuplicateStructTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/EmptyRankProfileTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ExactMatchTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ExpressionsAsArgsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/FieldsetTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/GeminiTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ImportedFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/IntegerAttributeToStringIndexTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/LowercaseTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/MailTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/derived/MatchSettingsResolvingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/MultiStructTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/MultipleSummariesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/NameCollisionTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/NativeRankTypeDefinitionsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/NuwaTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/derived/OrderIlscriptsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/PrefixExactAttributeTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/RankPropertiesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ReferenceFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/ReferenceFromSeveralTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SchemaInheritanceTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SimpleInheritTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SliceTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SortingTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/derived/StreamingStructTestCase.java create mode 100755 config-model/src/test/java/com/yahoo/schema/derived/StructAnyOrderTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/StructInheritanceTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/TestableDeployLogger.java create mode 100755 config-model/src/test/java/com/yahoo/schema/derived/TokenizationTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/TwoStreamingStructsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/TypesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/document/HnswIndexParamsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/parser/ConvertIntermediateTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/parser/ParsedDocumentTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/AnnotationReferenceTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSchemaTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/DuplicateStructTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ExpressionsAsArgsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/NuwaTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java delete mode 100755 config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXGBoostTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java diff --git a/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java b/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java index 1b2226c7a17..90f87d0b37e 100644 --- a/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java +++ b/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java @@ -5,7 +5,7 @@ import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.DocumentAccessParams; import com.yahoo.documentapi.local.LocalDocumentAccess; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import java.io.File; import java.util.stream.Stream; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java index 5737f038a17..1bab8f6ee27 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java @@ -14,7 +14,7 @@ import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.messagebus.DynamicThrottlePolicy; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.jupiter.api.BeforeEach; @@ -31,7 +31,6 @@ import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java index c5212bdbfff..60dd0a1589d 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.ProgressToken; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.jupiter.api.Test; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java index c5043f0fac3..e92d1e2ce1a 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java @@ -6,7 +6,7 @@ import ai.vespa.reindexing.Reindexing.Trigger; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; import org.junit.jupiter.api.Test; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java index 6ab41f4a7f7..2e3f4942b7d 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java @@ -10,7 +10,7 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.ProgressToken; import com.yahoo.jdisc.test.MockMetric; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.jupiter.api.Test; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 3cd2499f968..11d84884194 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -29,10 +29,10 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.schema.Application; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -41,7 +41,6 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; import com.yahoo.vespa.model.container.search.SemanticRules; -import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.FileNotFoundException; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index ff5930f8f6a..7f15da3179f 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -4,16 +4,15 @@ package com.yahoo.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.Field; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.processing.BuiltInFieldSets; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.processing.BuiltInFieldSets; import java.util.Collection; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/schema/Application.java b/config-model/src/main/java/com/yahoo/schema/Application.java new file mode 100644 index 00000000000..aa47818ff99 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/Application.java @@ -0,0 +1,108 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.schema.derived.SearchOrderer; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.processing.Processing; +import com.yahoo.schema.processing.Processor; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A collection of objects representing the content of an application package. + * This is created, then added to, and lastly validated when there is no more content to add. + * At that point it is ready to use for deriving configuration. + * + * @author bratseth + */ +public class Application { + + private final ApplicationPackage applicationPackage; + private final Map schemas; + private final DocumentModel documentModel; + + public Application(ApplicationPackage applicationPackage, + List schemas, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, + ModelContext.Properties properties, + boolean documentsOnly, + boolean validate, + Set> processorsToSkip, + DeployLogger logger) { + this.applicationPackage = applicationPackage; + + Map schemaMap = new LinkedHashMap<>(); + for (Schema schema : schemas) { + if (schemaMap.containsKey(schema.getName())) + throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); + schemaMap.put(schema.getName(), schema); + } + this.schemas = Collections.unmodifiableMap(schemaMap); + + schemas.forEach(schema -> schema.setOwner(this)); + if (validate) + schemas.forEach(schema -> schema.validate(logger)); + + new TemporarySDTypeResolver(schemas, logger).process(); + + List sdocs = new ArrayList<>(); + sdocs.add(SDDocumentType.VESPA_DOCUMENT); + for (Schema schema : schemas) { + if (schema.hasDocument()) { + sdocs.add(schema.getDocument()); + } + } + + var orderer = new SDDocumentTypeOrderer(sdocs, logger); + orderer.process(); + for (SDDocumentType sdoc : orderer.getOrdered()) { + new FieldOperationApplierForStructs().process(sdoc); + new FieldOperationApplier().process(sdoc); + } + + var resolver = new DocumentReferenceResolver(schemas); + sdocs.forEach(resolver::resolveReferences); + sdocs.forEach(resolver::resolveInheritedReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(schemas); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); + + if (validate) + new DocumentGraphValidator().validateDocumentGraph(sdocs); + + List schemasSomewhatOrdered = new ArrayList<>(schemas); + for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { + new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? + new Processing(properties).process(schema, + logger, + rankProfileRegistry, + queryProfiles, + validate, + documentsOnly, + processorsToSkip); + } + + this.documentModel = new DocumentModelBuilder().build(schemasSomewhatOrdered); + } + + public ApplicationPackage applicationPackage() { return applicationPackage; } + + /** Returns an unmodifiable list of the schemas of this application */ + public Map schemas() { return schemas; } + + public DocumentModel documentModel() { return documentModel; } + + @Override + public String toString() { return "application " + applicationPackage.getApplicationId(); } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java new file mode 100644 index 00000000000..d2c7035bac6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -0,0 +1,513 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.schema.parser.ConvertSchemaCollection; +import com.yahoo.schema.parser.IntermediateCollection; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.Processor; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Application builder. Usage: + * 1) Add all schemas, using the addXXX() methods, + * 2) provide the available rank types and rank expressions, using the setRankXXX() methods, + * 3) invoke the {@link #build} method + * + * @author bratseth + */ +public class ApplicationBuilder { + + private final IntermediateCollection mediator; + private final ApplicationPackage applicationPackage; + private final List schemas = new ArrayList<>(); + private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); + private final RankProfileRegistry rankProfileRegistry; + private final QueryProfileRegistry queryProfileRegistry; + private final FileRegistry fileRegistry; + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + /** True to build the document aspect only, skipping instantiation of rank profiles */ + private final boolean documentsOnly; + + private Application application; + + private final Set> processorsToSkip = new HashSet<>(); + + /** For testing only */ + public ApplicationBuilder() { + this(new RankProfileRegistry(), new QueryProfileRegistry()); + } + + /** For testing only */ + public ApplicationBuilder(DeployLogger deployLogger) { + this(MockApplicationPackage.createEmpty(), deployLogger); + } + + /** For testing only */ + public ApplicationBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); + } + + /** Used for generating documents for typed access to document fields in Java */ + public ApplicationBuilder(boolean documentsOnly) { + this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); + } + + /** For testing only */ + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger) { + this(app, new MockFileRegistry(), deployLogger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); + } + + /** For testing only */ + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + this(app, new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, new QueryProfileRegistry()); + } + + /** For testing only */ + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry) { + this(rankProfileRegistry, new QueryProfileRegistry()); + } + + /** For testing only */ + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); + } + + /** For testing only */ + public ApplicationBuilder(ModelContext.Properties properties) { + this(new RankProfileRegistry(), new QueryProfileRegistry(), properties); + } + + /** For testing only */ + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { + this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); + } + + /** normal constructor */ + public ApplicationBuilder(ApplicationPackage app, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + this(app, fileRegistry, deployLogger, properties, rankProfileRegistry, queryProfileRegistry, false); + } + + private ApplicationBuilder(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { + this.mediator = new IntermediateCollection(deployLogger, properties); + this.applicationPackage = applicationPackage; + this.rankProfileRegistry = rankProfileRegistry; + this.queryProfileRegistry = queryProfileRegistry; + this.fileRegistry = fileRegistry; + this.deployLogger = deployLogger; + this.properties = properties; + this.documentsOnly = documentsOnly; + for (NamedReader reader : applicationPackage.getSchemas()) + addSchema(reader); + } + + /** + * Adds a schema to this application. + * + * @param fileName the name of the file to import + * @throws IOException thrown if the file can not be read for some reason + * @throws ParseException thrown if the file does not contain a valid search definition + */ + public void addSchemaFile(String fileName) throws IOException, ParseException { + var parsedName = mediator.addSchemaFromFile(fileName); + addRankProfileFiles(parsedName); + } + + /** + * Reads and parses the schema string provided by the given reader. Once all schemas have been + * imported, call {@link #build}. + * + * @param reader the reader whose content to import + */ + public void addSchema(NamedReader reader) { + try { + var parsedName = mediator.addSchemaFromReader(reader); + addRankProfileFiles(parsedName); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); + } + } + + /** + * Adds a schema to this + * + * @param schemaString the content of the schema + */ + public void addSchema(String schemaString) throws ParseException { + var parsed = mediator.addSchemaFromString(schemaString); + addRankProfileFiles(parsed.name()); + } + + /** + * Registers the given schema to the application to be built during {@link #build}. A + * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most + * programmatically constructed schemas used in unit tests. + * + * @param schema the object to import + * @throws IllegalArgumentException if the given search object has already been processed + */ + public Schema add(Schema schema) { + if (schema.getName() == null) + throw new IllegalArgumentException("Schema has no name"); + schemas.add(schema); + return schema; + } + + private void addRankProfileFiles(String schemaName) throws ParseException { + if (applicationPackage == null) return; + + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schemaName); + for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) { + mediator.addRankProfileFile(schemaName, reader); + } + + Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schemaName); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile", true)) { + System.out.println("Got " + reader); + mediator.addRankProfileFile(schemaName, reader); + } + } + + /** + * Processes and finalizes the schemas of this. + * + * @throws IllegalStateException thrown if this method has already been called + */ + public Application build(boolean validate) { + if (application != null) throw new IllegalStateException("Application already built"); + var converter = new ConvertSchemaCollection(mediator, + documentTypeManager, + applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + documentsOnly); + for (var schema : converter.convertToSchemas()) + add(schema); + application = new Application(applicationPackage, + schemas, + rankProfileRegistry, + new QueryProfiles(queryProfileRegistry, deployLogger), + properties, + documentsOnly, + validate, + processorsToSkip, + deployLogger); + return application; + } + + /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ + public Set> processorsToSkip() { return processorsToSkip; } + + /** + * Convenience method to call {@link #getSchema(String)} when there is only a single {@link Schema} object + * built. This method will never return null. + * + * @return the built object + * @throws IllegalStateException if there is not exactly one search. + */ + public Schema getSchema() { + if (application == null) throw new IllegalStateException("Application not built"); + if (application.schemas().size() != 1) + throw new IllegalStateException("This call only works if we have 1 schema. Schemas: " + + application.schemas().values()); + + return application.schemas().values().stream().findAny().get(); + } + + public DocumentModel getModel() { return application.documentModel(); } + + /** + * Returns the built {@link Schema} object that has the given name. If the name is unknown, this method will simply + * return null. + * + * @param name the name of the schema to return, + * or null to return the only one or throw an exception if there are multiple to choose from + * @return the built object, or null if none with this name + * @throws IllegalStateException if {@link #build} has not been called. + */ + public Schema getSchema(String name) { + if (application == null) throw new IllegalStateException("Application not built"); + if (name == null) return getSchema(); + return application.schemas().get(name); + } + + public Application application() { return application; } + + /** + * Convenience method to return a list of all built {@link Schema} objects. + * + * @return the list of built searches + */ + public List getSchemaList() { + return new ArrayList<>(application.schemas().values()); + } + + /** + * Convenience factory method to import and build a {@link Schema} object from a string. + * + * @param sd the string to build from + * @return the built {@link ApplicationBuilder} object + * @throws ParseException thrown if there is a problem parsing the string + */ + public static ApplicationBuilder createFromString(String sd) throws ParseException { + return createFromString(sd, new BaseDeployLogger()); + } + + public static ApplicationBuilder createFromString(String sd, DeployLogger logger) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(sd); + builder.build(true); + return builder; + } + + public static ApplicationBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); + for (var schema : schemas) + builder.addSchema(schema); + builder.build(true); + return builder; + } + + /** + * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. + * + * @param fileName the file to build from + * @return the built {@link ApplicationBuilder} object + * @throws IOException if there was a problem reading the file. + * @throws ParseException if there was a problem parsing the file content. + */ + public static ApplicationBuilder createFromFile(String fileName) throws IOException, ParseException { + return createFromFile(fileName, new BaseDeployLogger()); + } + + /** + * Convenience factory methods to create a SearchBuilder from multiple SD files. Only for testing. + */ + public static ApplicationBuilder createFromFiles(Collection fileNames) throws IOException, ParseException { + return createFromFiles(fileNames, new BaseDeployLogger()); + } + + public static ApplicationBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { + return createFromFile(fileName, logger, new RankProfileRegistry(), new QueryProfileRegistry()); + } + + private static ApplicationBuilder createFromFiles(Collection fileNames, DeployLogger logger) throws IOException, ParseException { + return createFromFiles(fileNames, new MockFileRegistry(), logger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); + } + + /** + * Convenience factory method to import and build a {@link Schema} object from a file. + * + * @param fileName the file to build from. + * @param deployLogger logger for deploy messages. + * @param rankProfileRegistry registry for rank profiles. + * @return the built {@link ApplicationBuilder} object. + * @throws IOException if there was a problem reading the file. + * @throws ParseException if there was a problem parsing the file content. + */ + private static ApplicationBuilder createFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) + throws IOException, ParseException { + return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), + rankProfileRegistry, queryprofileRegistry); + } + + /** + * Convenience factory methdd to create a SearchBuilder from multiple SD files.. + */ + private static ApplicationBuilder createFromFiles(Collection fileNames, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) + throws IOException, ParseException { + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryprofileRegistry); + for (String fileName : fileNames) { + builder.addSchemaFile(fileName); + } + builder.build(true); + return builder; + } + + + public static ApplicationBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { + return createFromDirectory(dir, fileRegistry, logger, properties, new RankProfileRegistry()); + } + public static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + return createFromDirectory(dir, fileRegistry, logger, properties, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); + } + private static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + return createFromDirectory(dir, MockApplicationPackage.fromSearchDefinitionAndRootDirectory(dir), fileRegistry, logger, properties, + rankProfileRegistry, queryProfileRegistry); + } + + private static ApplicationBuilder createFromDirectory(String dir, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + ApplicationBuilder builder = new ApplicationBuilder(applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryProfileRegistry); + for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next().toString()); + } + builder.build(true); + return builder; + } + + private static QueryProfileRegistry createQueryProfileRegistryFromDirectory(String dir) { + File queryProfilesDir = new File(dir, "query-profiles"); + if ( ! queryProfilesDir.exists()) return new QueryProfileRegistry(); + return new QueryProfileXMLReader().read(queryProfilesDir.toString()); + } + + // TODO: The build methods below just call the create methods above - remove + + /** + * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. + * + * @param fileName the file to build from + * @return the built {@link Schema} object + * @throws IOException thrown if there was a problem reading the file + * @throws ParseException thrown if there was a problem parsing the file content + */ + public static Schema buildFromFile(String fileName) throws IOException, ParseException { + return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); + } + + /** + * Convenience factory method to import and build a {@link Schema} object from a file. + * + * @param fileName the file to build from + * @param rankProfileRegistry registry for rank profiles + * @return the built {@link Schema} object + * @throws IOException thrown if there was a problem reading the file + * @throws ParseException thrown if there was a problem parsing the file content + */ + public static Schema buildFromFile(String fileName, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) + throws IOException, ParseException { + return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry); + } + + /** + * Convenience factory method to import and build a {@link Schema} from a file. + * + * @param fileName the file to build from + * @param deployLogger logger for deploy messages + * @param rankProfileRegistry registry for rank profiles + * @return the built {@link Schema} object + * @throws IOException thrown if there was a problem reading the file + * @throws ParseException thrown if there was a problem parsing the file content + */ + public static Schema buildFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) + throws IOException, ParseException { + return createFromFile(fileName, deployLogger, rankProfileRegistry, queryProfileRegistry).getSchema(); + } + + /** + * Convenience factory method to import and build a {@link Schema} object from a raw object. + * + * @param rawSchema the raw object to build from + * @return the built {@link ApplicationBuilder} object + * @see #add(Schema) + */ + public static ApplicationBuilder createFromRawSchema(Schema rawSchema, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry); + builder.add(rawSchema); + builder.build(true); + return builder; + } + + /** + * Convenience factory method to import and build a {@link Schema} object from a raw object. + * + * @param rawSchema the raw object to build from + * @return the built {@link Schema} object + * @see #add(Schema) + */ + public static Schema buildFromRawSchema(Schema rawSchema, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + return createFromRawSchema(rawSchema, rankProfileRegistry, queryProfileRegistry).getSchema(); + } + + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + + public ModelContext.Properties getProperties() { return properties; } + + public DeployLogger getDeployLogger() { return deployLogger; } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/schema/DefaultRankProfile.java new file mode 100644 index 00000000000..9ab03b8c4a1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DefaultRankProfile.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.ImmutableSDField; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * The rank profile containing default settings. This is derived from the fields + * whenever this is accessed. + * + * @author bratseth + */ +public class DefaultRankProfile extends RankProfile { + + /** + * Creates a new rank profile + * + * @param rankProfileRegistry the {@link com.yahoo.schema.RankProfileRegistry} + * to use for storing and looking up rank profiles + */ + public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { + super("default", schema, rankProfileRegistry); + } + + /** Ignore self inheriting of default as some applications may use that for historical reasons. */ + public void inherit(String inheritedName) { + if (inheritedName.equals("default")) return; + super.inherit(inheritedName); + } + + @Override + public RankSetting getRankSetting(String fieldOrIndex, RankSetting.Type type) { + RankSetting setting = super.getRankSetting(fieldOrIndex, type); + if (setting != null) return setting; + + ImmutableSDField field = schema().getConcreteField(fieldOrIndex); + if (field != null) { + setting = toRankSetting(field, type); + if (setting != null) + return setting; + } + + Index index = schema().getIndex(fieldOrIndex); + if (index != null) { + setting = toRankSetting(index, type); + if (setting != null) + return setting; + } + + return null; + } + + private RankSetting toRankSetting(ImmutableSDField field, RankSetting.Type type) { + if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight() > 0 && field.getWeight() != 100) + return new RankSetting(field.getName(), type, field.getWeight()); + if (type.equals(RankSetting.Type.RANKTYPE)) + return new RankSetting(field.getName(), type, field.getRankType()); + if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost() > 0) + return new RankSetting(field.getName(), type, field.getLiteralBoost()); + + // Index level setting really + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { + return new RankSetting(field.getName(), type, true); + } + + return null; + } + + private RankSetting toRankSetting(Index index, RankSetting.Type type) { + /* TODO: Add support for indexes by adding a ranking object to the index + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && index.isPreferBitVector()) { + return new RankSetting(index.getName(), type, new Boolean(true)); + } + */ + return null; + } + + /** + * Returns the names of the fields which have a rank boost setting + * explicitly in this profile or in fields + */ + @Override + public Set rankSettings() { + Set settings = new LinkedHashSet<>(20); + settings.addAll(this.rankSettings); + for (ImmutableSDField field : schema().allConcreteFields() ) { + addSetting(field, RankSetting.Type.WEIGHT, settings); + addSetting(field, RankSetting.Type.RANKTYPE, settings); + addSetting(field, RankSetting.Type.LITERALBOOST, settings); + addSetting(field, RankSetting.Type.PREFERBITVECTOR, settings); + } + + // For settings that really pertains to indexes do the explicit indexes too + for (Index index : schema().getExplicitIndices()) { + addSetting(index, RankSetting.Type.PREFERBITVECTOR, settings); + } + return settings; + } + + private void addSetting(ImmutableSDField field, RankSetting.Type type, Set settings) { + if (type.isIndexLevel()) { + addIndexSettings(field, type, settings); + } + else { + RankSetting setting = toRankSetting(field, type); + if (setting == null) return; + settings.add(setting); + } + } + + private void addIndexSettings(ImmutableSDField field, RankSetting.Type type, Set settings) { + String indexName = field.getName(); + + // TODO: Make a ranking object in the index override the field level ranking object + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { + settings.add(new RankSetting(indexName, type, true)); + } + } + + private void addSetting(Index index, RankSetting.Type type, Set settings) { + RankSetting setting = toRankSetting(index, type); + if (setting == null) return; + settings.add(setting); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DistributableResource.java b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java new file mode 100644 index 00000000000..7a8a3963ba4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.path.Path; + +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DistributableResource implements Comparable { + + public enum PathType { FILE, URI, BLOB } + + /** The search definition-unique name of this constant */ + private final String name; + // TODO: Make path/pathType final + private PathType pathType; + private String path; + private FileReference fileReference = new FileReference(""); + + public PathType getPathType() { + return pathType; + } + + public DistributableResource(String name) { + this(name, null, PathType.FILE); + } + public DistributableResource(String name, String path) { + this(name, path, PathType.FILE); + } + public DistributableResource(String name, String path, PathType type) { + this.name = name; + this.path = path; + this.pathType = type; + } + + // TODO: Remove and make path/pathType final + public void setFileName(String fileName) { + Objects.requireNonNull(fileName, "Filename cannot be null"); + this.path = fileName; + this.pathType = PathType.FILE; + } + + // TODO: Remove and make path/pathType final + public void setUri(String uri) { + Objects.requireNonNull(uri, "uri cannot be null"); + this.path = uri; + this.pathType = PathType.URI; + } + + public String getName() { return name; } + public String getFileName() { return path; } + public Path getFilePath() { return Path.fromString(path); } + public String getUri() { return path; } + public String getFileReference() { return fileReference.value(); } + + public void validate() { + switch (pathType) { + case FILE: + case URI: + if (path == null || path.isEmpty()) + throw new IllegalArgumentException("Distributable URI/FILE resource must have a file or uri."); + break; + } + } + + public void register(FileRegistry fileRegistry) { + switch (pathType) { + case FILE: + fileReference = fileRegistry.addFile(path); + break; + case URI: + fileReference = fileRegistry.addUri(path); + break; + default: + throw new IllegalArgumentException("Unknown path type " + pathType); + } + } + + protected void register(FileRegistry fileRegistry, ByteBuffer blob) { + fileReference = fileRegistry.addBlob(path, blob); + } + + @Override + public String toString() { + return "resource '" + name + " of type '" + pathType + "' with ref '" + fileReference + "'"; + } + + @Override + public int compareTo(DistributableResource o) { + return name.compareTo(o.getName()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentGraphValidator.java b/config-model/src/main/java/com/yahoo/schema/DocumentGraphValidator.java new file mode 100644 index 00000000000..648cdf18c5b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentGraphValidator.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDDocumentType; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +import static java.util.stream.Collectors.joining; + +/** + * Validates that there are no cycles between document types (exception: self-reference is allowed). + * Example: if document B inherits A, then A cannot have a document reference to B. + * + * @author bjorncs + */ +public class DocumentGraphValidator { + + public void validateDocumentGraph(List documents) { + for (SDDocumentType document : documents) { + validateRoot(document); + } + } + + private static void validateRoot(SDDocumentType root) { + validateChildren(root, root); + } + + private static void validateChildren(SDDocumentType root, SDDocumentType currentDocument) { + try { + currentDocument.getDocumentReferences().get() + .forEach(entry -> { + SDDocumentType referencedDocument = entry.getValue().targetSearch().getDocument(); + validateDocument(root, referencedDocument); + }); + currentDocument.getInheritedTypes() + .forEach(inheritedDocument -> { + if (!isRootDocument(inheritedDocument)) { + validateDocument(root, inheritedDocument); + } + }); + } catch (DocumentGraphException e) { + e.addParentDocument(currentDocument); + throw e; + } + } + + private static void validateDocument(SDDocumentType root, SDDocumentType currentDocument) { + if (root.equals(currentDocument)) { + throw new DocumentGraphException(currentDocument); + } + validateChildren(root, currentDocument); + } + + private static boolean isRootDocument(SDDocumentType doc) { + return doc.getName().equals("document"); + } + + public static class DocumentGraphException extends IllegalArgumentException { + private final Deque deque = new ArrayDeque<>(); + + public DocumentGraphException(SDDocumentType document) { + deque.addLast(document); + } + + public void addParentDocument(SDDocumentType document) { + deque.addFirst(document); + } + + @Override + public String getMessage() { + return deque.stream() + .map(SDDocumentType::getName) + .collect(joining("->", "Document dependency cycle detected: ", ".")); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java new file mode 100644 index 00000000000..1a6134b410f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java @@ -0,0 +1,639 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.TemporaryStructuredDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.documentmodel.DataTypeCollection; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.OwnedStructDataType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedFields; +import com.yahoo.schema.document.annotation.SDAnnotationType; +import com.yahoo.schema.document.annotation.TemporaryAnnotationReferenceDataType; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.documentmodel.FieldView; +import com.yahoo.vespa.documentmodel.SearchDef; +import com.yahoo.vespa.documentmodel.SearchField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @author baldersheim + */ +public class DocumentModelBuilder { + + private final DocumentModel model; + + public DocumentModelBuilder() { + this.model = new DocumentModel(); + this.model.getDocumentManager().add(VespaDocumentType.INSTANCE); + } + + public DocumentModel build(Collection schemaList) { + List docList = new LinkedList<>(); + for (Schema schema : schemaList) { + docList.add(schema.getDocument()); + } + docList = sortDocumentTypes(docList); + addDocumentTypes(docList); + for (Collection toAdd = tryAdd(schemaList); + ! toAdd.isEmpty() && (toAdd.size() < schemaList.size()); + toAdd = tryAdd(schemaList)) { + schemaList = toAdd; + } + return model; + } + + private List sortDocumentTypes(List docList) { + Set doneNames = new HashSet<>(); + doneNames.add(SDDocumentType.VESPA_DOCUMENT.getName()); + List doneList = new LinkedList<>(); + List prevList = null; + List nextList = docList; + while (prevList == null || nextList.size() < prevList.size()) { + prevList = nextList; + nextList = new LinkedList<>(); + for (SDDocumentType doc : prevList) { + boolean isDone = true; + for (SDDocumentType inherited : doc.getInheritedTypes()) { + if (!doneNames.contains(inherited.getName())) { + isDone = false; + break; + } + } + if (isDone) { + doneNames.add(doc.getName()); + doneList.add(doc); + } else { + nextList.add(doc); + } + } + } + if (!nextList.isEmpty()) { + throw new IllegalArgumentException("Could not resolve inheritance of document types " + + toString(prevList) + "."); + } + return doneList; + } + + private static String toString(List lst) { + StringBuilder out = new StringBuilder(); + for (int i = 0, len = lst.size(); i < len; ++i) { + out.append("'").append(lst.get(i).getName()).append("'"); + if (i < len - 2) { + out.append(", "); + } else if (i < len - 1) { + out.append(" and "); + } + } + return out.toString(); + } + + private Collection tryAdd(Collection schemaList) { + Collection left = new ArrayList<>(); + for (Schema schema : schemaList) { + try { + addToModel(schema); + } catch (RetryLaterException e) { + left.add(schema); + } + } + return left; + } + + private void addToModel(Schema schema) { + // Then we add the search specific stuff + SearchDef searchDef = new SearchDef(schema.getName()); + addSearchFields(schema.extraFieldList(), searchDef); + for (Field f : schema.getDocument().fieldSet()) { + addSearchField((SDField) f, searchDef); + } + for (SDField field : schema.allConcreteFields()) { + for (Attribute attribute : field.getAttributes().values()) { + if ( ! searchDef.getFields().containsKey(attribute.getName())) { + searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true)); + } + } + } + + for (Field f : schema.getDocument().fieldSet()) { + addAlias((SDField) f, searchDef); + } + model.getSearchManager().add(searchDef); + } + + private static void addSearchFields(Collection fields, SearchDef searchDef) { + for (SDField field : fields) { + addSearchField(field, searchDef); + } + } + + private static void addSearchField(SDField field, SearchDef searchDef) { + SearchField searchField = + new SearchField(field, + field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), + field.getAttributes().containsKey(field.getName())); + searchDef.add(searchField); + + // Add field to views + addToView(field.getIndices().keySet(), searchField, searchDef); + } + + private static void addAlias(SDField field, SearchDef searchDef) { + for (Map.Entry entry : field.getAliasToName().entrySet()) { + searchDef.addAlias(entry.getKey(), entry.getValue()); + } + } + + private static void addToView(Collection views, Field field, SearchDef searchDef) { + for (String viewName : views) { + addToView(viewName, field, searchDef); + } + } + + private static void addToView(String viewName, Field field, SearchDef searchDef) { + if (searchDef.getViews().containsKey(viewName)) { + searchDef.getViews().get(viewName).add(field); + } else { + if (!searchDef.getFields().containsKey(viewName)) { + FieldView view = new FieldView(viewName); + view.add(field); + searchDef.add(view); + } + } + } + + private static String descT(DataType type) { + if (type == null) { return ""; } + return "'" + type.getName() + "' [" + type.getId() + "] {"+type.getClass() + "}"; + } + + private void addDocumentTypes(List docList) { + LinkedList lst = new LinkedList<>(); + for (SDDocumentType doc : docList) { + lst.add(convert(doc)); + model.getDocumentManager().add(lst.getLast()); + } + Map replacements = new IdentityHashMap<>(); + for(NewDocumentType doc : lst) { + resolveTemporaries(doc.getAllTypes(), lst, replacements); + resolveTemporariesRecurse(doc.getContentStruct(), doc.getAllTypes(), lst, replacements); + } + for(NewDocumentType doc : lst) { + for (var entry : replacements.entrySet()) { + var old = entry.getKey(); + if (doc.getDataType(old.getId()) == old) { + doc.replace(entry.getValue()); + } + } + } + } + + private static void resolveTemporaries(DataTypeCollection dtc, + Collection docs, + Map replacements) { + for (DataType type : dtc.getTypes()) { + resolveTemporariesRecurse(type, dtc, docs, replacements); + } + } + + @SuppressWarnings("deprecation") + private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, + Collection docs, + Map replacements) { + if (replacements.containsKey(type)) { + return replacements.get(type); + } + DataType original = type; + if (type instanceof TemporaryStructuredDataType) { + throw new IllegalArgumentException("Cannot handle temporary: " + type); + } + if (type instanceof TemporaryUnknownType) { + // must be a known struct or document type + DataType other = repo.getDataType(type.getId()); + if (other == null || other == type) { + // maybe it is the name of a document type: + other = getDocumentType(docs, type.getName()); + } + if (other == null) { + throw new IllegalArgumentException("No replacement found for temporary type: " + type); + } + type = other; + } else if (type instanceof OwnedTemporaryType) { + // must be replaced with the real struct type + DataType other = repo.getDataType(type.getId()); + if (other == null || other == type) { + throw new IllegalArgumentException("No replacement found for temporary type: " + type); + } + if (other instanceof OwnedStructDataType) { + var owned = (OwnedTemporaryType) type; + String ownedBy = owned.getOwnerName(); + var otherOwned = (OwnedStructDataType) other; + String otherOwnedBy = otherOwned.getOwnerName(); + if (! ownedBy.equals(otherOwnedBy)) { + throw new IllegalArgumentException("Wrong document for type: " + otherOwnedBy + " but expected " + ownedBy); + } + } else { + throw new IllegalArgumentException("Found wrong sort of type: " + other + " [" + other.getClass() + "]"); + } + type = other; + } else if (type instanceof DocumentType) { + DataType other = getDocumentType(docs, type.getName()); + if (other != null) { + type = other; + } else if (type != DataType.DOCUMENT) { + throw new IllegalArgumentException + ("Can not handle nested document definitions. Undefined document type: " + type.toString()); + } + } else if (type instanceof NewDocumentType) { + DataType other = getDocumentType(docs, type.getName()); + if (other != null) { + type = other; + } + } else if (type instanceof StructDataType) { + // trick avoids infinite recursion: + var old = replacements.put(original, type); + assert(old == null); + StructDataType dt = (StructDataType) type; + for (com.yahoo.document.Field field : dt.getFields()) { + var ft = field.getDataType(); + var newft = resolveTemporariesRecurse(ft, repo, docs, replacements); + if (ft != newft) { + // XXX deprecated: + field.setDataType(newft); + } + } + old = replacements.remove(original); + assert(old == type); + } + else if (type instanceof MapDataType) { + MapDataType t = (MapDataType) type; + var old_kt = t.getKeyType(); + var old_vt = t.getValueType(); + var kt = resolveTemporariesRecurse(old_kt, repo, docs, replacements); + var vt = resolveTemporariesRecurse(old_vt, repo, docs, replacements); + if (kt != old_kt || vt != old_vt) { + type = new MapDataType(kt, vt, t.getId()); + } + } + else if (type instanceof ArrayDataType) { + ArrayDataType t = (ArrayDataType) type; + var old_nt = t.getNestedType(); + var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); + if (nt != old_nt) { + type = new ArrayDataType(nt, t.getId()); + } + } + else if (type instanceof WeightedSetDataType) { + WeightedSetDataType t = (WeightedSetDataType) type; + var old_nt = t.getNestedType(); + var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); + if (nt != old_nt) { + boolean c = t.createIfNonExistent(); + boolean r = t.removeIfZero(); + type = new WeightedSetDataType(nt, c, r, t.getId()); + } + } + else if (type instanceof NewDocumentReferenceDataType) { + var t = (NewDocumentReferenceDataType) type; + var doc = getDocumentType(docs, t.getTargetTypeName()); + type = doc.getReferenceDataType(); + } + if (type != original) { + replacements.put(original, type); + } + return type; + } + + private static NewDocumentType getDocumentType(Collection docs, String name) { + for (NewDocumentType doc : docs) { + if (doc.getName().equals(name)) { + return doc; + } + } + return null; + } + + private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { + if (sa.getInherits() != null) { + AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); + SDAnnotationType inherited = (SDAnnotationType) tmp; + return ((inherited.getSdDocType() != null) || anyParentsHavePayLoad(inherited, sdoc)); + } + return false; + } + + private NewDocumentType convert(SDDocumentType sdoc) { + NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), + sdoc.getDocumentType().contentStruct(), + sdoc.getFieldSets(), + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); + for (SDDocumentType n : sdoc.getInheritedTypes()) { + NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); + NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); + if (inherited != null) { + dt.inherit(inherited); + } + } + var extractor = new TypeExtractor(dt); + extractor.extract(sdoc); + return dt; + } + + static class TypeExtractor { + private final NewDocumentType targetDt; + Map annotationInheritance = new LinkedHashMap<>(); + Map structInheritance = new LinkedHashMap<>(); + private final Map inProgress = new IdentityHashMap<>(); + TypeExtractor(NewDocumentType target) { + this.targetDt = target; + } + + void extract(SDDocumentType sdoc) { + for (SDDocumentType type : sdoc.getTypes()) { + if (type.isStruct()) { + handleStruct(type); + } else { + throw new IllegalArgumentException("Data type '" + type.getName() + "' is not a struct => tostring='" + type.toString() + "'."); + } + } + for (SDDocumentType type : sdoc.getTypes()) { + for (SDDocumentType proxy : type.getInheritedTypes()) { + var inherited = (StructDataType) targetDt.getDataTypeRecursive(proxy.getName()); + var converted = (StructDataType) targetDt.getDataType(type.getName()); + assert(converted instanceof OwnedStructDataType); + assert(inherited instanceof OwnedStructDataType); + if (! converted.inherits(inherited)) { + converted.inherit(inherited); + } + } + } + for (AnnotationType annotation : sdoc.getAnnotations().values()) { + targetDt.add(annotation); + } + for (AnnotationType annotation : sdoc.getAnnotations().values()) { + SDAnnotationType sa = (SDAnnotationType) annotation; + if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { + annotationInheritance.put(annotation, sa.getInherits()); + } + if (annotation.getDataType() == null) { + if (sa.getSdDocType() != null) { + StructDataType s = handleStruct(sa.getSdDocType()); + annotation.setDataType(s); + if ((sa.getInherits() != null)) { + structInheritance.put(s, "annotation." + sa.getInherits()); + } + } else if (sa.getInherits() != null) { + StructDataType s = new OwnedStructDataType("annotation." + annotation.getName(), sdoc.getName()); + if (anyParentsHavePayLoad(sa, sdoc)) { + annotation.setDataType(s); + addType(s); + } + structInheritance.put(s, "annotation." + sa.getInherits()); + } + } else { + var dt = annotation.getDataType(); + if (dt instanceof StructDataType) { + handleStruct((StructDataType) dt); + } + } + } + for (Map.Entry e : annotationInheritance.entrySet()) { + e.getKey().inherit(targetDt.getAnnotationType(e.getValue())); + } + for (Map.Entry e : structInheritance.entrySet()) { + StructDataType s = (StructDataType)targetDt.getDataType(e.getValue()); + if (s != null) { + e.getKey().inherit(s); + } + } + handleStruct(sdoc.getDocumentType().contentStruct()); + extractDataTypesFromFields(sdoc.fieldSet()); + } + + private void extractDataTypesFromFields(Collection fields) { + for (Field f : fields) { + DataType type = f.getDataType(); + if (testAddType(type)) { + extractNestedTypes(type); + addType(type); + } + } + } + + private void extractNestedTypes(DataType type) { + if (inProgress.containsKey(type)) { + return; + } + inProgress.put(type, this); + if (type instanceof StructDataType) { + StructDataType tmp = (StructDataType) type; + extractDataTypesFromFields(tmp.getFieldsThisTypeOnly()); + } else if (type instanceof CollectionDataType) { + CollectionDataType tmp = (CollectionDataType) type; + extractNestedTypes(tmp.getNestedType()); + addType(tmp.getNestedType()); + } else if (type instanceof MapDataType) { + MapDataType tmp = (MapDataType) type; + extractNestedTypes(tmp.getKeyType()); + extractNestedTypes(tmp.getValueType()); + addType(tmp.getKeyType()); + addType(tmp.getValueType()); + } else if (type instanceof TemporaryAnnotationReferenceDataType) { + throw new IllegalArgumentException(type.toString()); + } + } + + private boolean testAddType(DataType type) { return internalAddType(type, true); } + + private boolean addType(DataType type) { return internalAddType(type, false); } + + private boolean internalAddType(DataType type, boolean dryRun) { + DataType oldType = targetDt.getDataTypeRecursive(type.getId()); + if (oldType == null) { + if ( ! dryRun) { + targetDt.add(type); + } + return true; + } + if (oldType == type) { + return false; + } + if (targetDt.getDataType(type.getId()) == null) { + if ((oldType instanceof OwnedStructDataType) + && (type instanceof OwnedStructDataType)) + { + var oldOwned = (OwnedStructDataType) oldType; + var newOwned = (OwnedStructDataType) type; + if (newOwned.getOwnerName().equals(targetDt.getName()) && + ! oldOwned.getOwnerName().equals(targetDt.getName())) + { + if ( ! dryRun) { + targetDt.add(type); + } + return true; + } + } + } + if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { + StructDataType s = (StructDataType) type; + StructDataType os = (StructDataType) oldType; + if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { + if ( ! dryRun) { + targetDt.replace(type); + } + return true; + } + } + return false; + } + + + @SuppressWarnings("deprecation") + private void specialHandleAnnotationReference(Field field) { + DataType fieldType = specialHandleAnnotationReferenceRecurse(field.getName(), field.getDataType()); + if (fieldType == null) { + return; + } + field.setDataType(fieldType); // XXX deprecated + } + + private DataType specialHandleAnnotationReferenceRecurse(String fieldName, + DataType dataType) { + if (dataType instanceof TemporaryAnnotationReferenceDataType) { + TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; + if (refType.getId() != 0) { + return null; + } + AnnotationType target = targetDt.getAnnotationType(refType.getTarget()); + if (target == null) { + throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + + "' does not exist."); + } + dataType = new AnnotationReferenceDataType(target); + addType(dataType); + return dataType; + } + else if (dataType instanceof MapDataType) { + MapDataType t = (MapDataType)dataType; + DataType valueType = specialHandleAnnotationReferenceRecurse(fieldName, t.getValueType()); + if (valueType == null) { + return null; + } + var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); + addType(mapType); + return mapType; + } + else if (dataType instanceof ArrayDataType) { + ArrayDataType t = (ArrayDataType) dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + if (nestedType == null) { + return null; + } + var lstType = new ArrayDataType(nestedType, t.getId()); + addType(lstType); + return lstType; + } + else if (dataType instanceof WeightedSetDataType) { + WeightedSetDataType t = (WeightedSetDataType) dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + if (nestedType == null) { + return null; + } + boolean c = t.createIfNonExistent(); + boolean r = t.removeIfZero(); + var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); + addType(lstType); + return lstType; + } + return null; + } + + @SuppressWarnings("deprecation") + private StructDataType handleStruct(SDDocumentType type) { + if (type.isStruct()) { + var st = type.getStruct(); + if (st.getName().equals(type.getName()) && + (st instanceof StructDataType) && + (! (st instanceof TemporaryUnknownType)) && + (! (st instanceof OwnedTemporaryType))) + { + return handleStruct((StructDataType) st); + } + } + StructDataType s = new OwnedStructDataType(type.getName(), targetDt.getName()); + for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(f); + s.addField(f); + } + for (StructDataType inherited : type.getDocumentType().contentStruct().getInheritedTypes()) { + s.inherit(inherited); + } + extractNestedTypes(s); + addType(s); + return s; + } + + private StructDataType handleStruct(StructDataType s) { + for (Field f : s.getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(f); + } + extractNestedTypes(s); + addType(s); + return s; + } + + } + + private static Set convertDocumentReferencesToNames(Optional documentReferences) { + if (!documentReferences.isPresent()) { + return Set.of(); + } + return documentReferences.get().referenceMap().values().stream() + .map(documentReference -> documentReference.targetSearch().getDocument()) + .map(documentType -> new NewDocumentType.Name(documentType.getName())) + .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + } + + private static Set convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return Set.of(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + + public static class RetryLaterException extends IllegalArgumentException { + public RetryLaterException(String message) { + super(message); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentOnlySchema.java b/config-model/src/main/java/com/yahoo/schema/DocumentOnlySchema.java new file mode 100644 index 00000000000..f9005c7b775 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentOnlySchema.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.schema.document.SDDocumentType; + +/** + * A search that was derived from an sd file containing no search element(s), only + * document specifications, so the name of this is decided by parsing and adding the document instance. + * + * @author vegardh + */ +public class DocumentOnlySchema extends Schema { + + public DocumentOnlySchema(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties) { + super(applicationPackage, fileRegistry, deployLogger, properties); + } + + @Override + public void addDocument(SDDocumentType docType) { + if (getName() == null) { + setName(docType.getName()); + } + super.addDocument(docType); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentReference.java b/config-model/src/main/java/com/yahoo/schema/DocumentReference.java new file mode 100644 index 00000000000..048035ffef8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentReference.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.Field; + +/** + * Represents a document reference. Contains the document reference field and the search instance of the referred document. + * + * @author bjorncs + */ +public class DocumentReference { + + private final Field referenceField; + private final Schema targetSchema; + + public DocumentReference(Field referenceField, Schema targetSchema) { + this.referenceField = referenceField; + this.targetSchema = targetSchema; + } + + public Field referenceField() { + return referenceField; + } + + public Schema targetSearch() { + return targetSchema; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/schema/DocumentReferenceResolver.java new file mode 100644 index 00000000000..b3e06fd5e02 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentReferenceResolver.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.Field; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +/** + * Resolves all document references in the search definitions + * + * Iterates through all document fields having a {@link NewDocumentReferenceDataType} and uses {@link NewDocumentReferenceDataType#getTargetTypeName()} + * to determine the referenced document. This information is aggregated into a {@link DocumentReferences} object. + * + * @author bjorncs + */ +public class DocumentReferenceResolver { + + private final Map schemaMapping; + + public DocumentReferenceResolver(Collection schemas) { + this.schemaMapping = createDocumentNameToSearchMapping(schemas); + } + + public void resolveReferences(SDDocumentType documentType) { + var references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType)); + documentType.setDocumentReferences(references); + } + + public void resolveInheritedReferences(SDDocumentType documentType) { + resolveInheritedReferencesRecursive(documentType, documentType.getInheritedTypes()); + } + + private void resolveInheritedReferencesRecursive(SDDocumentType documentType, + Collection inheritedTypes) { + for (var inheritedType : inheritedTypes) { + documentType.getDocumentReferences().get().mergeFrom(inheritedType.getDocumentReferences().get()); + } + for (var inheritedType : inheritedTypes) { + resolveInheritedReferencesRecursive(documentType, inheritedType.getInheritedTypes()); + } + } + + private Map createFieldToDocumentReferenceMapping(SDDocumentType documentType) { + return fieldStream(documentType) + .filter(field -> field.getDataType() instanceof NewDocumentReferenceDataType) + .collect(toMap(Field::getName, this::createDocumentReference)); + } + + private DocumentReference createDocumentReference(Field field) { + if (!isAttribute(field)) { + throw new IllegalArgumentException( + String.format( + "The field '%s' is an invalid document reference. The field must be an attribute.", + field.getName())); + } + NewDocumentReferenceDataType reference = (NewDocumentReferenceDataType) field.getDataType(); + String targetDocumentName = getTargetDocumentName(reference); + Schema schema = schemaMapping.get(targetDocumentName); + if (schema == null) { + throw new IllegalArgumentException( + String.format("Invalid document reference '%s': " + + "Could not find document type '%s'", field.getName(), targetDocumentName)); + } + return new DocumentReference(field, schema); + } + + private static boolean isAttribute(Field field) { + SDField sdField = (SDField) field; // Ugly, but SDDocumentType only expose the fields as the super class Field + return sdField.doesAttributing(); + } + + private static Map createDocumentNameToSearchMapping(Collection schemaDefintions) { + return schemaDefintions.stream() + .filter(search -> search.getDocument() != null) + .collect(toMap(search -> search.getDocument().getName(), identity())); + } + + private static Stream fieldStream(SDDocumentType documentType) { + return documentType.getDocumentType().getFields().stream(); + } + + private static String getTargetDocumentName(NewDocumentReferenceDataType reference) { + return reference.getTargetTypeName(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentReferences.java b/config-model/src/main/java/com/yahoo/schema/DocumentReferences.java new file mode 100644 index 00000000000..3583a5134e0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentReferences.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Contains all document references for a document mapped by field name + * + * @author bjorncs + */ +public class DocumentReferences implements Iterable> { + private final Map references; + + public DocumentReferences(Map references) { + this.references = references; + } + + public void mergeFrom(DocumentReferences other) { + references.putAll(other.references); + } + + @Override + public Iterator> iterator() { + return Collections.unmodifiableSet(references.entrySet()).iterator(); + } + + public Map referenceMap() { + return Collections.unmodifiableMap(references); + } + + public Stream> stream() { + return references.entrySet().stream(); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/schema/DocumentsOnlyRankProfile.java new file mode 100644 index 00000000000..ffd517cf241 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/DocumentsOnlyRankProfile.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import java.util.List; + +/** + * A rank profile which ignores all calls made to it which may fail in a document only setting. + * This is used by the search definition parser when it is requested to parse documents only, + * to avoid having to check for this in every method which adds to the rank profile. + * (And why do we ever want to parse documents only? Because it is used when generating Java classes + * from documents, where the full application package may not be available.) + * + * @author bratseth + */ +public class DocumentsOnlyRankProfile extends RankProfile { + + public DocumentsOnlyRankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { + super(name, schema, rankProfileRegistry); + } + + @Override + public void setFirstPhaseRanking(String expression) { + // Ignore + } + + @Override + public void setSecondPhaseRanking(String expression) { + // Ignore + } + + @Override + public void addFunction(String name, List arguments, String expression, boolean inline) { + // Ignore + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/FeatureNames.java b/config-model/src/main/java/com/yahoo/schema/FeatureNames.java new file mode 100644 index 00000000000..0671903194f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/FeatureNames.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.searchlib.rankingexpression.Reference; + +import java.util.Optional; + +/** + * Utility methods for query, document and constant rank feature names + * + * @author bratseth + */ +public class FeatureNames { + + public static Reference asConstantFeature(String constantName) { + return Reference.simple("constant", quoteIfNecessary(constantName)); + } + + public static Reference asAttributeFeature(String attributeName) { + return Reference.simple("attribute", attributeName); + } + + public static Reference asQueryFeature(String propertyName) { + return Reference.simple("query", quoteIfNecessary(propertyName)); + } + + /** Returns true if the given reference is an attribute, constant or query feature */ + public static boolean isSimpleFeature(Reference reference) { + if ( ! reference.isSimple()) return false; + String name = reference.name(); + return name.equals("attribute") || name.equals("constant") || name.equals("query"); + } + + /** Returns true if this is a constant */ + public static boolean isConstantFeature(Reference reference) { + if ( ! isSimpleFeature(reference)) return false; + return reference.name().equals("constant"); + } + + /** Returns true if this is a query feature */ + public static boolean isQueryFeature(Reference reference) { + if ( ! isSimpleFeature(reference)) return false; + return reference.name().equals("query"); + } + + /** Returns true if this is an attribute feature */ + public static boolean isAttributeFeature(Reference reference) { + if ( ! isSimpleFeature(reference)) return false; + return reference.name().equals("attribute"); + } + + /** + * Returns the single argument of the given feature name, without any quotes, + * or empty if it is not a valid query, attribute or constant feature name + */ + public static Optional argumentOf(String feature) { + Optional reference = Reference.simple(feature); + if ( reference.isEmpty()) return Optional.empty(); + if ( ! ( reference.get().name().equals("attribute") || + reference.get().name().equals("constant") || + reference.get().name().equals("query"))) + return Optional.empty(); + + return Optional.of(reference.get().arguments().expressions().get(0).toString()); + } + + private static String quoteIfNecessary(String s) { + if (notNeedQuotes(s)) + return s; + else + return "\"" + s + "\""; + } + + static boolean notNeedQuotes(String s) { + // Faster version of the regexp [A-Za-z0-9_][A-Za-z0-9_-]* + if (s.isEmpty()) return false; + if ( ! isValidFirst(s.charAt(0))) return false; + for (int i = 1; i < s.length(); i++) { + if (!isValidAny(s.charAt(i))) return false; + } + return true; + } + private static boolean isValidFirst(char c) { + // [A-Za-z0-9_] + return (c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || ((c >= '0') && (c <= '9')); + } + private static boolean isValidAny(char c) { + // [A-Za-z0-9_-]* + return c == '-' || isValidFirst(c); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/FieldOperationApplier.java b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplier.java new file mode 100644 index 00000000000..b2f40b045ea --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplier.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.Field; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class FieldOperationApplier { + + public void process(SDDocumentType sdoc) { + if (!sdoc.isStruct()) { + apply(sdoc); + } + } + + protected void apply(SDDocumentType type) { + for (Field field : type.fieldSet()) { + apply(field); + } + } + + protected void apply(Field field) { + if (field instanceof SDField) { + SDField sdField = (SDField) field; + sdField.applyOperations(); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForSearch.java new file mode 100644 index 00000000000..b107dbaea59 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForSearch.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.Field; +import com.yahoo.schema.document.SDDocumentType; + +/** + * @author Einar M R Rosenvinge + */ +public class FieldOperationApplierForSearch extends FieldOperationApplier { + + @Override + public void process(SDDocumentType sdoc) { + //Do nothing + } + + public void process(Schema schema) { + for (Field field : schema.extraFieldList()) { + apply(field); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForStructs.java new file mode 100644 index 00000000000..1ec1de6a9c6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForStructs.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDDocumentType; + +/** + * @author Einar M R Rosenvinge + */ +public class FieldOperationApplierForStructs extends FieldOperationApplier { + + @Override + public void process(SDDocumentType sdoc) { + for (SDDocumentType type : sdoc.getAllTypes()) { + if (type.isStruct()) { + apply(type); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/FieldSets.java b/config-model/src/main/java/com/yahoo/schema/FieldSets.java new file mode 100644 index 00000000000..0594056150c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/FieldSets.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +import com.yahoo.schema.document.FieldSet; + +/** + * The field sets owned by a {@link Schema} + * Both built in and user defined. + * + * @author vegardh + */ +public class FieldSets { + + private final Optional owner; + private final Map userFieldSets; + private final Map builtInFieldSets; + + public FieldSets(Optional owner) { + this.owner = owner; + userFieldSets = new LinkedHashMap<>(); + builtInFieldSets = new LinkedHashMap<>(); + } + + /** + * Adds an entry to user field sets, creating entries as needed + * + * @param setName name of a field set + * @param field field to add to field set + */ + public void addUserFieldSetItem(String setName, String field) { + if (userFieldSets.get(setName) == null) { + // First entry in this set + userFieldSets.put(setName, new FieldSet(setName)); + } + userFieldSets.get(setName).addFieldName(field); + } + + /** + * Adds an entry to built in field sets, creating entries as needed + * + * @param setName name of a field set + * @param field field to add to field set + */ + public void addBuiltInFieldSetItem(String setName, String field) { + if (builtInFieldSets.get(setName) == null) { + // First entry in this set + builtInFieldSets.put(setName, new FieldSet(setName)); + } + builtInFieldSets.get(setName).addFieldName(field); + } + + /** Returns the built in field sets, unmodifiable */ + public Map builtInFieldSets() { + if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(builtInFieldSets); + if (builtInFieldSets.isEmpty()) return owner.get().inherited().get().fieldSets().builtInFieldSets(); + + var fieldSets = new LinkedHashMap<>(owner.get().inherited().get().fieldSets().builtInFieldSets()); + fieldSets.putAll(builtInFieldSets); + return Collections.unmodifiableMap(fieldSets); + } + + /** Returns the user defined field sets, unmodifiable */ + public Map userFieldSets() { + if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(userFieldSets); + if (userFieldSets.isEmpty()) return owner.get().inherited().get().fieldSets().userFieldSets(); + + var fieldSets = new LinkedHashMap<>(owner.get().inherited().get().fieldSets().userFieldSets()); + fieldSets.putAll(userFieldSets); + return Collections.unmodifiableMap(fieldSets); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/ImmutableSchema.java b/config-model/src/main/java/com/yahoo/schema/ImmutableSchema.java new file mode 100644 index 00000000000..c352d9c417f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/ImmutableSchema.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.io.Reader; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +/** + * An interface containing the non-mutating methods of {@link Schema}. + * For description of the methods see {@link Schema}. + * + * @author bjorncs + */ +public interface ImmutableSchema { + + String getName(); + Optional inherited(); + Index getIndex(String name); + ImmutableSDField getConcreteField(String name); + //TODO split in mutating/immutable by returning List + List allConcreteFields(); + List getExplicitIndices(); + Reader getRankingExpression(String fileName); + ApplicationPackage applicationPackage(); + DeployLogger getDeployLogger(); + ModelContext.Properties getDeployProperties(); + Map constants(); + LargeRankExpressions rankExpressionFiles(); + Map onnxModels(); + Stream allImportedFields(); + SDDocumentType getDocument(); + ImmutableSDField getField(String name); + + default Stream allFields() { + return allFieldsList().stream(); + } + List allFieldsList(); + + List getSummaryFields(ImmutableSDField field); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/schema/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..0df79b30298 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/ImportedFieldsEnumerator.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDDocumentType; + +import java.util.Collection; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final Collection schemas; + + public ImportedFieldsEnumerator(Collection schemas) { + this.schemas = schemas; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.schemas.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/Index.java b/config-model/src/main/java/com/yahoo/schema/Index.java new file mode 100644 index 00000000000..190081cf80f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/Index.java @@ -0,0 +1,200 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.Stemming; + +import java.io.Serializable; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * An index definition in a search definition. + * Two indices are equal if they have the same name and the same settings, except + * alias settings (which are excluded). + * + * @author bratseth + */ +public class Index implements Cloneable, Serializable { + + public enum Type { + + VESPA("vespa"); + private final String name; + Type(String name) { this.name = name; } + public String getName() { return name; } + + } + + // Please see hashCode, equals and copy when adding attributes to this + + /** The search definition-unique name of this index */ + private String name; + + /** The rank type of this index */ + private RankType rankType = null; + + /** Whether this index supports prefix search */ + private boolean prefix; + + /** The list of aliases (Strings) to this index name */ + private Set aliases = new java.util.LinkedHashSet<>(1); + + /** + * The stemming setting of this field, or null to use the default. + * Default is determined by the owning search definition. + */ + private Stemming stemming = null; + + private Type type = Type.VESPA; + + /** The boolean index definition, if set */ + private BooleanIndexDefinition boolIndex; + + private Optional hnswIndexParams = Optional.empty(); + + /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ + private boolean interleavedFeatures = false; + + public Index(String name) { + this(name, false); + } + + public Index(String name, boolean prefix) { + this.name = name; + this.prefix = prefix; + } + + public void setName(String name) { this.name = name; } + + public String getName() { return name; } + + /** Sets the rank type of this field */ + public void setRankType(RankType rankType) { this.rankType = rankType; } + + /** Returns the rank type of this field, or null if nothing is set */ + public RankType getRankType() { return rankType; } + + /** Return the stemming setting of this index, may be null */ + public Stemming getStemming() { return stemming; } + + /** + * Whether this field should be stemmed in this search definition, + * this is never null + */ + public Stemming getStemming(Schema schema) { + if (stemming != null) + return stemming; + else + return schema.getStemming(); + } + + /** + * Sets how this field should be stemmed, or set to null to use the default. + */ + public void setStemming(Stemming stemming) { this.stemming = stemming; } + + /** Returns whether this index supports prefix search, default is false */ + public boolean isPrefix() { return prefix; } + + /** Sets whether this index supports prefix search */ + public void setPrefix(boolean prefix) { this.prefix=prefix; } + + /** Adds an alias to this index name */ + public void addAlias(String alias) { + aliases.add(alias); + } + + /** Returns a read-only iterator of the aliases (Strings) to this index name */ + public Iterator aliasIterator() { + return Collections.unmodifiableSet(aliases).iterator(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Index index = (Index) o; + return prefix == index.prefix && + interleavedFeatures == index.interleavedFeatures && + Objects.equals(name, index.name) && + rankType == index.rankType && + Objects.equals(aliases, index.aliases) && + stemming == index.stemming && + type == index.type && + Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(hnswIndexParams, index.hnswIndexParams); + } + + @Override + public int hashCode() { + return Objects.hash(name, rankType, prefix, aliases, stemming, type, boolIndex, hnswIndexParams, interleavedFeatures); + } + + public String toString() { + String rankTypeName = rankType == null ? "(none)" : rankType.name(); + return "index '" + name + + "' [ranktype: " + rankTypeName + + ", prefix: " + prefix + "]"; + } + + /** Makes a deep copy of this index */ + @Override + public Object clone() { + try { + Index copy = (Index)super.clone(); + copy.aliases = new LinkedHashSet<>(this.aliases); + return copy; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error",e); + } + } + + public Index copy() { + return (Index)clone(); + } + + /** Returns the index engine type */ + public Type getType() { + return type; + } + + /** Sets the index engine type */ + public void setType(Type type) { + this.type = type; + } + + /** Returns the boolean index definition */ + public BooleanIndexDefinition getBooleanIndexDefiniton() { + return boolIndex; + } + + /** Sets the boolean index definition */ + public void setBooleanIndexDefiniton(BooleanIndexDefinition def) { + boolIndex = def; + } + + public Optional getHnswIndexParams() { + return hnswIndexParams; + } + + public void setHnswIndexParams(HnswIndexParams params) { + hnswIndexParams = Optional.of(params); + } + + public void setInterleavedFeatures(boolean value) { + interleavedFeatures = value; + } + + public boolean useInterleavedFeatures() { + return interleavedFeatures; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/schema/LargeRankExpressions.java new file mode 100644 index 00000000000..cfdd4729b97 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/LargeRankExpressions.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.FileRegistry; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class LargeRankExpressions { + private final Map expressions = new ConcurrentHashMap<>(); + private final FileRegistry fileRegistry; + private final int limit; + + public LargeRankExpressions(FileRegistry fileRegistry) { + this(fileRegistry, 8192); + } + public LargeRankExpressions(FileRegistry fileRegistry, int limit) { + this.fileRegistry = fileRegistry; + this.limit = limit; + } + + public void add(RankExpressionBody expression) { + String name = expression.getName(); + RankExpressionBody prev = expressions.putIfAbsent(name, expression); + if (prev == null) { + expression.validate(); + expression.register(fileRegistry); + } else { + if ( ! prev.getBlob().equals(expression.getBlob())) { + throw new IllegalArgumentException("Rank expression '" + name + + "' defined twice. Previous blob with " + prev.getBlob().remaining() + + " bytes, while current has " + expression.getBlob().remaining() + " bytes"); + } + } + } + public int limit() { return limit; } + + /** Returns a read-only list of ranking constants ordered by name */ + public Collection expressions() { + return expressions.values().stream().sorted().collect(Collectors.toUnmodifiableList()); + } + + // Note: Use by integration tests in internal repo + /** Returns a read-only map of the ranking constants in this indexed by name */ + public Map asMap() { + return Collections.unmodifiableMap(expressions); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java new file mode 100644 index 00000000000..c6c807f2dbb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java @@ -0,0 +1,361 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.schema.expressiontransforms.OnnxModelTransformer; +import com.yahoo.schema.expressiontransforms.TokenTransformer; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; +import com.yahoo.searchlib.rankingexpression.rule.NameNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; + +/** + * A context which only contains type information. + * This returns empty tensor types (double) for unknown features which are not + * query, attribute or constant features, as we do not have information about which such + * features exist (but we know those that exist are doubles). + * + * This is not multithread safe. + * + * @author bratseth + */ +public class MapEvaluationTypeContext extends FunctionReferenceContext implements TypeContext { + + private final Optional parent; + + private final Map featureTypes = new HashMap<>(); + + private final Map resolvedTypes = new HashMap<>(); + + /** To avoid re-resolving diamond-shaped dependencies */ + private final Map globallyResolvedTypes; + + /** For invocation loop detection */ + private final Deque currentResolutionCallStack; + + private final SortedSet queryFeaturesNotDeclared; + private boolean tensorsAreUsed; + + MapEvaluationTypeContext(ImmutableMap functions, Map featureTypes) { + super(functions); + this.parent = Optional.empty(); + this.featureTypes.putAll(featureTypes); + this.currentResolutionCallStack = new ArrayDeque<>(); + this.queryFeaturesNotDeclared = new TreeSet<>(); + tensorsAreUsed = false; + globallyResolvedTypes = new HashMap<>(); + } + + private MapEvaluationTypeContext(Map functions, + Map bindings, + Optional parent, + Map featureTypes, + Deque currentResolutionCallStack, + SortedSet queryFeaturesNotDeclared, + boolean tensorsAreUsed, + Map globallyResolvedTypes) { + super(functions, bindings); + this.parent = parent; + this.featureTypes.putAll(featureTypes); + this.currentResolutionCallStack = currentResolutionCallStack; + this.queryFeaturesNotDeclared = queryFeaturesNotDeclared; + this.tensorsAreUsed = tensorsAreUsed; + this.globallyResolvedTypes = globallyResolvedTypes; + } + + public void setType(Reference reference, TensorType type) { + featureTypes.put(reference, type); + queryFeaturesNotDeclared.remove(reference); + } + + public Map featureTypes() { return Collections.unmodifiableMap(featureTypes); } + + @Override + public TensorType getType(String reference) { + throw new UnsupportedOperationException("Not able to parse general references from string form"); + } + + public void forgetResolvedTypes() { + resolvedTypes.clear(); + } + + private boolean referenceCanBeResolvedGlobally(Reference reference) { + Optional function = functionInvocation(reference); + return function.isPresent() && function.get().arguments().size() == 0; + // are there other cases we would like to resolve globally? + } + + @Override + public TensorType getType(Reference reference) { + // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: + boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); + + TensorType resolvedType = resolvedTypes.get(reference); + if (resolvedType == null && canBeResolvedGlobally) { + resolvedType = globallyResolvedTypes.get(reference); + } + if (resolvedType != null) { + return resolvedType; + } + + resolvedType = resolveType(reference); + if (resolvedType == null) + return defaultTypeOf(reference); // Don't store fallback to default as we may know more later + resolvedTypes.put(reference, resolvedType); + if (resolvedType.rank() > 0) + tensorsAreUsed = true; + + if (canBeResolvedGlobally) { + globallyResolvedTypes.put(reference, resolvedType); + } + + return resolvedType; + } + + MapEvaluationTypeContext getParent(String forArgument, String boundTo) { + return parent.orElseThrow( + () -> new IllegalArgumentException("argument "+forArgument+" is bound to "+boundTo+" but there is no parent context")); + } + + String resolveBinding(String argument) { + String bound = getBinding(argument); + if (bound == null) { + return argument; + } + return getParent(argument, bound).resolveBinding(bound); + } + + private TensorType resolveType(Reference reference) { + if (currentResolutionCallStack.contains(reference)) + throw new IllegalArgumentException("Invocation loop: " + + currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) + + " -> " + reference); + + // Bound to a function argument? + Optional binding = boundIdentifier(reference); + if (binding.isPresent()) { + try { + // This is not pretty, but changing to bind expressions rather + // than their string values requires deeper changes + var expr = new RankingExpression(binding.get()); + var type = expr.type(getParent(reference.name(), binding.get())); + return type; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + try { + currentResolutionCallStack.addLast(reference); + + // A reference to an attribute, query or constant feature? + if (FeatureNames.isSimpleFeature(reference)) { + // The argument may be a local identifier bound to the actual value + String argument = reference.simpleArgument().get(); + String argumentBinding = resolveBinding(argument); + reference = Reference.simple(reference.name(), argumentBinding); + return featureTypes.get(reference); + } + + // A reference to a function? + Optional function = functionInvocation(reference); + if (function.isPresent()) { + var body = function.get().getBody(); + var child = this.withBindings(bind(function.get().arguments(), reference.arguments())); + var type = body.type(child); + return type; + } + + // A reference to an ONNX model? + Optional onnxFeatureType = onnxFeatureType(reference); + if (onnxFeatureType.isPresent()) { + return onnxFeatureType.get(); + } + + // A reference to a feature for transformer token input? + Optional transformerTokensFeatureType = transformerTokensFeatureType(reference); + if (transformerTokensFeatureType.isPresent()) { + return transformerTokensFeatureType.get(); + } + + // A reference to a feature which returns a tensor? + Optional featureTensorType = tensorFeatureType(reference); + if (featureTensorType.isPresent()) { + return featureTensorType.get(); + } + + // A directly injected identifier? (Useful for stateless model evaluation) + if (reference.isIdentifier() && featureTypes.containsKey(reference)) { + return featureTypes.get(reference); + } + + // the name of a constant feature? + if (reference.isIdentifier()) { + Reference asConst = FeatureNames.asConstantFeature(reference.name()); + if (featureTypes.containsKey(asConst)) { + return featureTypes.get(asConst); + } + } + + // We do not know what this is - since we do not have complete knowledge about the match features + // in Java we must assume this is a match feature and return the double type - which is the type of + // all match features + return TensorType.empty; + } + finally { + currentResolutionCallStack.removeLast(); + } + } + + /** + * Returns the default type for this simple feature, or null if it does not have a default + */ + public TensorType defaultTypeOf(Reference reference) { + if ( ! FeatureNames.isSimpleFeature(reference)) + throw new IllegalArgumentException("This can only be called for simple references, not " + reference); + if (reference.name().equals("query")) { // we do not require all query features to be declared, only non-doubles + queryFeaturesNotDeclared.add(reference); + return TensorType.empty; + } + return null; + } + + /** + * Returns the binding if this reference is a simple identifier which is bound in this context. + * Returns empty otherwise. + */ + private Optional boundIdentifier(Reference reference) { + if ( ! reference.arguments().isEmpty()) return Optional.empty(); + if ( reference.output() != null) return Optional.empty(); + return Optional.ofNullable(getBinding(reference.name())); + } + + private Optional functionInvocation(Reference reference) { + if (reference.output() != null) return Optional.empty(); + ExpressionFunction function = getFunctions().get(reference.name()); + if (function == null) return Optional.empty(); + if (function.arguments().size() != reference.arguments().size()) return Optional.empty(); + return Optional.of(function); + } + + private Optional onnxFeatureType(Reference reference) { + if ( ! reference.name().equals("onnxModel") && ! reference.name().equals("onnx")) + return Optional.empty(); + + if ( ! featureTypes.containsKey(reference)) { + String configOrFileName = reference.arguments().expressions().get(0).toString(); + + // Look up standardized format as added in RankProfile + String modelConfigName = OnnxModelTransformer.getModelConfigName(reference); + String modelOutput = OnnxModelTransformer.getModelOutput(reference, null); + + reference = new Reference("onnxModel", new Arguments(new ReferenceNode(modelConfigName)), modelOutput); + if ( ! featureTypes.containsKey(reference)) { + throw new IllegalArgumentException("Missing onnx-model config for '" + configOrFileName + "'"); + } + } + + return Optional.of(featureTypes.get(reference)); + } + + private Optional transformerTokensFeatureType(Reference reference) { + if ( ! reference.name().equals("tokenTypeIds") && + ! reference.name().equals("tokenInputIds") && + ! reference.name().equals("tokenAttentionMask")) + return Optional.empty(); + + if ( ! (reference.arguments().size() > 1)) + throw new IllegalArgumentException(reference.name() + " must have at least 2 arguments"); + + ExpressionNode size = reference.arguments().expressions().get(0); + return Optional.of(TokenTransformer.createTensorType(reference.name(), size)); + } + + /** + * There are two features which returns the (non-empty) tensor type: tensorFromLabels and tensorFromWeightedSet. + * This returns the type of those features if this is a reference to either of them, or empty otherwise. + */ + private Optional tensorFeatureType(Reference reference) { + if ( ! reference.name().equals("tensorFromLabels") && ! reference.name().equals("tensorFromWeightedSet")) + return Optional.empty(); + + if (reference.arguments().size() != 1 && reference.arguments().size() != 2) + throw new IllegalArgumentException(reference.name() + " must have one or two arguments"); + + ExpressionNode arg0 = reference.arguments().expressions().get(0); + if ( ! ( arg0 instanceof ReferenceNode) || ! FeatureNames.isSimpleFeature(((ReferenceNode)arg0).reference())) + throw new IllegalArgumentException("The first argument of " + reference.name() + + " must be a simple feature, not " + arg0); + + String dimension; + if (reference.arguments().size() > 1) { + ExpressionNode arg1 = reference.arguments().expressions().get(1); + if ( ( ! (arg1 instanceof ReferenceNode) || ! (((ReferenceNode)arg1).reference().isIdentifier())) + && + ( ! (arg1 instanceof NameNode))) + throw new IllegalArgumentException("The second argument of " + reference.name() + + " must be a dimension name, not " + arg1); + dimension = reference.arguments().expressions().get(1).toString(); + } + else { // default + dimension = ((ReferenceNode)arg0).reference().arguments().expressions().get(0).toString(); + } + + // TODO: Determine the type of the weighted set/vector and use that as value type + return Optional.of(new TensorType.Builder().mapped(dimension).build()); + } + + /** Binds the given list of formal arguments to their actual values */ + private Map bind(List formalArguments, + Arguments invocationArguments) { + Map bindings = new HashMap<>(formalArguments.size()); + for (int i = 0; i < formalArguments.size(); i++) { + String identifier = invocationArguments.expressions().get(i).toString(); + bindings.put(formalArguments.get(i), identifier); + } + return bindings; + } + + /** + * Returns an unmodifiable view of the query features which was requested but for which we have no type info + * (such that they default to TensorType.empty), shared between all instances of this + * involved in resolving a particular rank profile. + */ + public SortedSet queryFeaturesNotDeclared() { + return Collections.unmodifiableSortedSet(queryFeaturesNotDeclared); + } + + /** Returns true if any feature across all instances involved in resolving this rank profile resolves to a tensor */ + public boolean tensorsAreUsed() { return tensorsAreUsed; } + + @Override + public MapEvaluationTypeContext withBindings(Map bindings) { + return new MapEvaluationTypeContext(getFunctions(), + bindings, + Optional.of(this), + featureTypes, + currentResolutionCallStack, + queryFeaturesNotDeclared, + tensorsAreUsed, + globallyResolvedTypes); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java new file mode 100644 index 00000000000..26a0b3e595d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.model.ml.OnnxModelInfo; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * A global ONNX model distributed using file distribution, similar to ranking constants. + * + * @author lesters + */ +public class OnnxModel extends DistributableResource { + + private OnnxModelInfo modelInfo = null; + private final Map inputMap = new HashMap<>(); + private final Map outputMap = new HashMap<>(); + + private String statelessExecutionMode = null; + private Integer statelessInterOpThreads = null; + private Integer statelessIntraOpThreads = null; + + public OnnxModel(String name) { + super(name); + } + + public OnnxModel(String name, String fileName) { + super(name, fileName); + validate(); + } + + @Override + public void setUri(String uri) { + throw new IllegalArgumentException("URI for ONNX models are not currently supported"); + } + + public void addInputNameMapping(String onnxName, String vespaName) { + addInputNameMapping(onnxName, vespaName, true); + } + + public void addInputNameMapping(String onnxName, String vespaName, boolean overwrite) { + Objects.requireNonNull(onnxName, "Onnx name cannot be null"); + Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + if (overwrite || ! inputMap.containsKey(onnxName)) { + inputMap.put(onnxName, vespaName); + } + } + + public void addOutputNameMapping(String onnxName, String vespaName) { + addOutputNameMapping(onnxName, vespaName, true); + } + + public void addOutputNameMapping(String onnxName, String vespaName, boolean overwrite) { + Objects.requireNonNull(onnxName, "Onnx name cannot be null"); + Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + if (overwrite || ! outputMap.containsKey(onnxName)) { + outputMap.put(onnxName, vespaName); + } + } + + public void setModelInfo(OnnxModelInfo modelInfo) { + Objects.requireNonNull(modelInfo, "Onnx model info cannot be null"); + for (String onnxName : modelInfo.getInputs()) { + addInputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false); + } + for (String onnxName : modelInfo.getOutputs()) { + addOutputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false); + } + this.modelInfo = modelInfo; + } + + public Map getInputMap() { return Collections.unmodifiableMap(inputMap); } + public Map getOutputMap() { return Collections.unmodifiableMap(outputMap); } + + public String getDefaultOutput() { + return modelInfo != null ? modelInfo.getDefaultOutput() : ""; + } + + TensorType getTensorType(String onnxName, Map inputTypes) { + return modelInfo != null ? modelInfo.getTensorType(onnxName, inputTypes) : TensorType.empty; + } + + public void setStatelessExecutionMode(String executionMode) { + if ("parallel".equalsIgnoreCase(executionMode)) { + this.statelessExecutionMode = "parallel"; + } else if ("sequential".equalsIgnoreCase(executionMode)) { + this.statelessExecutionMode = "sequential"; + } + } + + public Optional getStatelessExecutionMode() { + return Optional.ofNullable(statelessExecutionMode); + } + + public void setStatelessInterOpThreads(int interOpThreads) { + if (interOpThreads >= 0) { + this.statelessInterOpThreads = interOpThreads; + } + } + + public Optional getStatelessInterOpThreads() { + return Optional.ofNullable(statelessInterOpThreads); + } + + public void setStatelessIntraOpThreads(int intraOpThreads) { + if (intraOpThreads >= 0) { + this.statelessIntraOpThreads = intraOpThreads; + } + } + + public Optional getStatelessIntraOpThreads() { + return Optional.ofNullable(statelessIntraOpThreads); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/schema/RankExpressionBody.java new file mode 100644 index 00000000000..d383a25aecb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/RankExpressionBody.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.FileRegistry; + +import java.nio.ByteBuffer; + +import static java.util.Objects.requireNonNull; + +public class RankExpressionBody extends DistributableResource { + + private final ByteBuffer blob; + + public RankExpressionBody(String name, ByteBuffer blob) { + super(name, name + ".lz4", PathType.BLOB); + this.blob = requireNonNull(blob, "Blob cannot be null"); + } + + public ByteBuffer getBlob() { return blob; } + + public void validate() { + // Remove once pathType is final + if (getPathType() != PathType.BLOB) { + throw new IllegalArgumentException("PathType must be BLOB."); + } + } + + public void register(FileRegistry fileRegistry) { + register(fileRegistry, blob); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java new file mode 100644 index 00000000000..5479ecf323f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -0,0 +1,1526 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.ranking.Diversity; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.expressiontransforms.ExpressionTransforms; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.FeatureList; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.io.IOException; +import java.io.Reader; +import java.io.Serializable; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Represents a rank profile - a named set of ranking settings + * + * @author bratseth + */ +public class RankProfile implements Cloneable { + + public final static String FIRST_PHASE = "firstphase"; + public final static String SECOND_PHASE = "secondphase"; + + /** The schema-unique name of this rank profile */ + private final String name; + + /** The schema owning this profile, or null if global (owned by a model) */ + private final ImmutableSchema schema; + + private final List inheritedNames = new ArrayList<>(); + + /** The resolved inherited profiles, or null when not resolved. */ + private List inherited; + + private MatchPhaseSettings matchPhaseSettings = null; + + protected Set rankSettings = new java.util.LinkedHashSet<>(); + + /** The ranking expression to be used for first phase */ + private RankingExpressionFunction firstPhaseRanking = null; + + /** The ranking expression to be used for second phase */ + private RankingExpressionFunction secondPhaseRanking = null; + + /** Number of hits to be reranked in second phase, -1 means use default */ + private int rerankCount = -1; + + /** Mysterious attribute */ + private int keepRankCount = -1; + + private int numThreadsPerSearch = -1; + private int minHitsPerThread = -1; + private int numSearchPartitions = -1; + + private Double termwiseLimit = null; + private Double postFilterThreshold = null; + private Double approximateThreshold = null; + + /** The drop limit used to drop hits with rank score less than or equal to this value */ + private double rankScoreDropLimit = -Double.MAX_VALUE; + + private Set summaryFeatures; + private String inheritedSummaryFeaturesProfileName; + + private Set matchFeatures; + private String inheritedMatchFeaturesProfileName; + + private Set rankFeatures; + + /** The properties of this - a multimap */ + private Map> rankProperties = new LinkedHashMap<>(); + + private Boolean ignoreDefaultRankFeatures = null; + + private Map functions = new LinkedHashMap<>(); + // This cache must be invalidated every time modifications are done to 'functions'. + private CachedFunctions allFunctionsCached = null; + + private Map inputs = new LinkedHashMap<>(); + + private Map constants = new LinkedHashMap<>(); + + private Map onnxModels = new LinkedHashMap<>(); + + private Set filterFields = new HashSet<>(); + + private final RankProfileRegistry rankProfileRegistry; + + private final TypeSettings attributeTypes = new TypeSettings(); + + private List allFieldsList; + + private Boolean strict; + + private final ApplicationPackage applicationPackage; + private final DeployLogger deployLogger; + + /** + * Creates a new rank profile for a particular schema + * + * @param name the name of the new profile + * @param schema the schema owning this profile + * @param rankProfileRegistry the {@link com.yahoo.schema.RankProfileRegistry} to use for storing + * and looking up rank profiles. + */ + public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { + this.name = Objects.requireNonNull(name, "name cannot be null"); + this.schema = Objects.requireNonNull(schema, "schema cannot be null"); + this.rankProfileRegistry = rankProfileRegistry; + this.applicationPackage = schema.applicationPackage(); + this.deployLogger = schema.getDeployLogger(); + } + + /** + * Creates a global rank profile + * + * @param name the name of the new profile + */ + public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry) { + this.name = Objects.requireNonNull(name, "name cannot be null"); + this.schema = null; + this.rankProfileRegistry = rankProfileRegistry; + this.applicationPackage = applicationPackage; + this.deployLogger = deployLogger; + } + + public String name() { return name; } + + /** Returns the search definition owning this, or null if it is global */ + public ImmutableSchema schema() { return schema; } + + /** Returns the application this is part of */ + public ApplicationPackage applicationPackage() { + return applicationPackage; + } + + private Stream allFields() { + if (schema == null) return Stream.empty(); + if (allFieldsList == null) { + allFieldsList = schema.allFieldsList(); + } + return allFieldsList.stream(); + } + + private Stream allImportedFields() { + return schema != null ? schema.allImportedFields() : Stream.empty(); + } + + /** + * Returns whether type checking should fail if this profile accesses query features that are + * not defined in query profile types. + * + * Default is false. + */ + public boolean isStrict() { + Boolean declaredStrict = declaredStrict(); + if (declaredStrict != null) return declaredStrict; + return false; + } + + /** Returns the strict value declared in this or any parent profile. */ + public Boolean declaredStrict() { + if (strict != null) return strict; + return uniquelyInherited(p -> p.declaredStrict(), "strict").orElse(null); + } + + public void setStrict(Boolean strict) { + this.strict = strict; + } + + /** + * Adds a profile to those inherited by this. + * The profile must belong to this schema (directly or by inheritance). + */ + public void inherit(String inheritedName) { + inherited = null; + inheritedNames.add(inheritedName); + } + + /** Returns the names of the profiles this inherits, if any. */ + public List inheritedNames() { return Collections.unmodifiableList(inheritedNames); } + + /** Returns the rank profiles inherited by this. */ + private List inherited() { + if (inheritedNames.isEmpty()) return List.of(); + if (inherited != null) return inherited; + + inherited = resolveInheritedProfiles(schema); + List children = new ArrayList<>(); + children.add(createFullyQualifiedName()); + inherited.forEach(profile -> verifyNoInheritanceCycle(children, profile)); + return inherited; + } + + private String createFullyQualifiedName() { + return (schema != null) + ? (schema.getName() + "." + name()) + : name(); + } + + private void verifyNoInheritanceCycle(List children, RankProfile parent) { + children.add(parent.createFullyQualifiedName()); + String root = children.get(0); + if (root.equals(parent.createFullyQualifiedName())) + throw new IllegalArgumentException("There is a cycle in the inheritance for rank-profile '" + root + "' = " + children); + for (RankProfile parentInherited : parent.inherited()) + verifyNoInheritanceCycle(children, parentInherited); + } + + private List resolveInheritedProfiles(ImmutableSchema schema) { + List inherited = new ArrayList<>(); + for (String inheritedName : inheritedNames) { + RankProfile inheritedProfile = schema == null + ? rankProfileRegistry.getGlobal(inheritedName) + : resolveInheritedProfile(schema, inheritedName); + if (inheritedProfile == null) + throw new IllegalArgumentException("rank-profile '" + name() + "' inherits '" + inheritedName + + "', but this is not found in " + + ((schema() != null) ? schema() : " global rank profiles")); + inherited.add(inheritedProfile); + } + return inherited; + } + + private RankProfile resolveInheritedProfile(ImmutableSchema schema, String inheritedName) { + SDDocumentType documentType = schema.getDocument(); + if (documentType != null) { + if (name.equals(inheritedName)) { + // If you seemingly inherit yourself, you are actually referencing a rank-profile in one of your inherited schemas + for (SDDocumentType baseType : documentType.getInheritedTypes()) { + RankProfile resolvedFromBase = rankProfileRegistry.resolve(baseType, inheritedName); + if (resolvedFromBase != null) return resolvedFromBase; + } + } + return rankProfileRegistry.resolve(documentType, inheritedName); + } + return rankProfileRegistry.get(schema.getName(), inheritedName); + } + + /** Returns whether this profile inherits (directly or indirectly) the given profile name. */ + public boolean inherits(String name) { + for (RankProfile inheritedProfile : inherited()) { + if (inheritedProfile.name().equals(name)) return true; + if (inheritedProfile.inherits(name)) return true; + } + return false; + } + + public void setMatchPhaseSettings(MatchPhaseSettings settings) { + settings.checkValid(); + this.matchPhaseSettings = settings; + } + + public MatchPhaseSettings getMatchPhaseSettings() { + if (matchPhaseSettings != null) return matchPhaseSettings; + return uniquelyInherited(p -> p.getMatchPhaseSettings(), "match phase settings").orElse(null); + } + + /** Returns the uniquely determined property, where non-empty is defined as non-null */ + private Optional uniquelyInherited(Function propertyRetriever, + String propertyDescription) { + return uniquelyInherited(propertyRetriever, p -> p != null, propertyDescription); + } + + /** + * Returns the property retrieved by the given function, if it is only present in a single unique variant + * among all profiled inherited by this, or empty if not present. + * Note that for properties that don't implement a values-based equals this reverts to the stricter condition that + * only one inherited profile can define a non-empty value at all. + * + * @throws IllegalArgumentException if the inherited profiles defines multiple different values of the property + */ + private Optional uniquelyInherited(Function propertyRetriever, + Predicate nonEmptyValueFilter, + String propertyDescription) { + Set uniqueProperties = inherited().stream() + .map(p -> propertyRetriever.apply(p)) + .filter(p -> nonEmptyValueFilter.test(p)) + .collect(Collectors.toSet()); + if (uniqueProperties.isEmpty()) return Optional.empty(); + if (uniqueProperties.size() == 1) return Optional.of(uniqueProperties.stream().findAny().get()); + throw new IllegalArgumentException("Only one of the profiles inherited by " + this + " can contain " + + propertyDescription + ", but it is present in multiple"); + } + + public void addRankSetting(RankSetting rankSetting) { + rankSettings.add(rankSetting); + } + + public void addRankSetting(String fieldName, RankSetting.Type type, Object value) { + addRankSetting(new RankSetting(fieldName, type, value)); + } + + /** + * Returns the a rank setting of a field, or null if there is no such rank setting in this profile + * + * @param field the field whose settings to return + * @param type the type that the field is required to be + * @return the rank setting found, or null + */ + RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { + for (Iterator i = declaredRankSettingIterator(); i.hasNext(); ) { + RankSetting setting = i.next(); + if (setting.getFieldName().equals(field) && setting.getType() == type) { + return setting; + } + } + return null; + } + + /** + * Returns a rank setting of field or index, or null if there is no such rank setting in this profile or one it + * inherits + * + * @param field the field whose settings to return + * @param type the type that the field is required to be + * @return the rank setting found, or null + */ + public RankSetting getRankSetting(String field, RankSetting.Type type) { + RankSetting rankSetting = getDeclaredRankSetting(field, type); + if (rankSetting != null) return rankSetting; + + return uniquelyInherited(p -> p.getRankSetting(field, type), "rank setting " + type + " on " + field).orElse(null); + } + + /** + * Returns the rank settings in this rank profile + * + * @return an iterator for the declared rank setting + */ + public Iterator declaredRankSettingIterator() { + return Collections.unmodifiableSet(rankSettings).iterator(); + } + + /** + * Returns all settings in this profile or any profile it inherits + * + * @return an iterator for all rank settings of this + */ + public Iterator rankSettingIterator() { + return rankSettings().iterator(); + } + + /** + * Returns a snapshot of the rank settings of this and everything it inherits. + * Changes to the returned set will not be reflected in this rank profile. + */ + public Set rankSettings() { + Set settings = new LinkedHashSet<>(); + for (RankProfile inheritedProfile : inherited()) { + for (RankSetting setting : inheritedProfile.rankSettings()) { + if (settings.contains(setting)) + throw new IllegalArgumentException(setting + " is present in " + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + settings.add(setting); + } + } + + // TODO: Here we do things in the wrong order to not break tests. Reverse this. + Set finalSettings = new LinkedHashSet<>(rankSettings); + finalSettings.addAll(settings); + return finalSettings; + } + + public void add(Constant constant) { + constants.put(constant.name(), constant); + } + + /** Returns an unmodifiable view of the constants declared in this */ + public Map declaredConstants() { return Collections.unmodifiableMap(constants); } + + /** Returns an unmodifiable view of the constants available in this */ + public Map constants() { + Map allConstants = new HashMap<>(); + for (var inheritedProfile : inherited()) { + for (var constant : inheritedProfile.constants().values()) { + if (allConstants.containsKey(constant.name())) + throw new IllegalArgumentException(constant + "' is present in " + + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + allConstants.put(constant.name(), constant); + } + } + + if (schema != null) + allConstants.putAll(schema.constants()); + allConstants.putAll(constants); + return allConstants; + } + + public void add(OnnxModel model) { + onnxModels.put(model.getName(), model); + } + + /** Returns an unmodifiable map of the onnx models declared in this. */ + public Map declaredOnnxModels() { return onnxModels; } + + /** Returns an unmodifiable map of the onnx models available in this. */ + public Map onnxModels() { + Map allModels = new HashMap<>(); + for (var inheritedProfile : inherited()) { + for (var model : inheritedProfile.onnxModels().values()) { + if (allModels.containsKey(model.getName())) + throw new IllegalArgumentException(model + "' is present in " + + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + allModels.put(model.getName(), model); + } + } + + if (schema != null) + allModels.putAll(schema.onnxModels()); + allModels.putAll(onnxModels); + return allModels; + } + + public void addAttributeType(String attributeName, String attributeType) { + attributeTypes.addType(attributeName, attributeType); + } + + public Map getAttributeTypes() { + return attributeTypes.getTypes(); + } + + /** + * Returns the ranking expression to use by this. This expression must not be edited. + * Returns null if no expression is set. + */ + public RankingExpression getFirstPhaseRanking() { + RankingExpressionFunction function = getFirstPhase(); + if (function == null) return null; + return function.function.getBody(); + } + + public RankingExpressionFunction getFirstPhase() { + if (firstPhaseRanking != null) return firstPhaseRanking; + return uniquelyInherited(p -> p.getFirstPhase(), "first-phase expression").orElse(null); + } + + void setFirstPhaseRanking(RankingExpression rankingExpression) { + this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); + } + + public void setFirstPhaseRanking(String expression) { + try { + firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); + } catch (ParseException e) { + throw new IllegalArgumentException("Illegal first phase ranking function", e); + } + } + + /** + * Returns the ranking expression to use by this. This expression must not be edited. + * Returns null if no expression is set. + */ + public RankingExpression getSecondPhaseRanking() { + RankingExpressionFunction function = getSecondPhase(); + if (function == null) return null; + return function.function().getBody(); + } + + public RankingExpressionFunction getSecondPhase() { + if (secondPhaseRanking != null) return secondPhaseRanking; + return uniquelyInherited(p -> p.getSecondPhase(), "second-phase expression").orElse(null); + } + + public void setSecondPhaseRanking(String expression) { + try { + secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal second phase ranking function", e); + } + } + + // TODO: Below we have duplicate methods for summary and match features: Encapsulate this in a single parametrized + // class instead (and probably make rank features work the same). + + /** + * Sets the name this should inherit the summary features of. + * Without setting this, this will either have the summary features of the single parent setting them, + * or if summary features are set in this, only have the summary features in this. + * With this set the resulting summary features of this will be the superset of those defined in this and + * the final (with inheritance included) summary features of the given parent. + * The profile must be one which is directly inherited by this. + */ + public void setInheritedSummaryFeatures(String parentProfile) { + if ( ! inheritedNames().contains(parentProfile)) + throw new IllegalArgumentException("This can only inherit the summary features of a directly inherited profile, '" + + ", but attempting to inherit '" + parentProfile); + this.inheritedSummaryFeaturesProfileName = parentProfile; + } + + /** + * Sets the name of a profile this should inherit the match features of. + * Without setting this, this will either have the match features of the single parent setting them, + * or if match features are set in this, only have the match features in this. + * With this set the resulting match features of this will be the superset of those defined in this and + * the final (with inheritance included) match features of the given parent. + * The profile must be one which which is directly inherited by this. + * + */ + public void setInheritedMatchFeatures(String parentProfile) { + if ( ! inheritedNames().contains(parentProfile)) + throw new IllegalArgumentException("This can only inherit the match features of a directly inherited profile, '" + + ", but attempting to inherit '" + parentProfile); + this.inheritedMatchFeaturesProfileName = parentProfile; + } + + /** Returns a read-only view of the summary features to use in this profile. This is never null */ + public Set getSummaryFeatures() { + if (inheritedSummaryFeaturesProfileName != null && summaryFeatures != null) { + Set combined = new HashSet<>(); + RankProfile inherited = inherited().stream() + .filter(p -> p.name().equals(inheritedSummaryFeaturesProfileName)) + .findAny() + .orElseThrow(); + combined.addAll(inherited.getSummaryFeatures()); + combined.addAll(summaryFeatures); + return Collections.unmodifiableSet(combined); + } + if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); + return uniquelyInherited(p -> p.getSummaryFeatures(), f -> ! f.isEmpty(), "summary features") + .orElse(Set.of()); + } + + /** Returns a read-only view of the match features to use in this profile. This is never null */ + public Set getMatchFeatures() { + if (inheritedMatchFeaturesProfileName != null && matchFeatures != null) { + Set combined = new HashSet<>(); + RankProfile inherited = inherited().stream() + .filter(p -> p.name().equals(inheritedMatchFeaturesProfileName)) + .findAny() + .orElseThrow(); + combined.addAll(inherited.getMatchFeatures()); + combined.addAll(matchFeatures); + return Collections.unmodifiableSet(combined); + } + if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); + return uniquelyInherited(p -> p.getMatchFeatures(), f -> ! f.isEmpty(), "match features") + .orElse(Set.of()); + } + + private void addSummaryFeature(ReferenceNode feature) { + if (summaryFeatures == null) + summaryFeatures = new LinkedHashSet<>(); + summaryFeatures.add(feature); + } + + private void addMatchFeature(ReferenceNode feature) { + if (matchFeatures == null) + matchFeatures = new LinkedHashSet<>(); + matchFeatures.add(feature); + } + + /** Adds the content of the given feature list to the internal list of summary features. */ + public void addSummaryFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addSummaryFeature(feature); + } + } + + /** Adds the content of the given feature list to the internal list of match features. */ + public void addMatchFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addMatchFeature(feature); + } + } + + /** Returns a read-only view of the rank features to use in this profile. This is never null */ + public Set getRankFeatures() { + if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); + return uniquelyInherited(p -> p.getRankFeatures(), f -> ! f.isEmpty(), "summary-features") + .orElse(Set.of()); + } + + private void addRankFeature(ReferenceNode feature) { + if (rankFeatures == null) + rankFeatures = new LinkedHashSet<>(); + rankFeatures.add(feature); + } + + /** + * Adds the content of the given feature list to the internal list of rank features. + * + * @param features The features to add. + */ + public void addRankFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addRankFeature(feature); + } + } + + /** Returns a read only flattened list view of the rank properties to use in this profile. This is never null. */ + public List getRankProperties() { + List properties = new ArrayList<>(); + for (List propertyList : getRankPropertyMap().values()) { + properties.addAll(propertyList); + } + return Collections.unmodifiableList(properties); + } + + /** Returns a read only map view of the rank properties to use in this profile. This is never null. */ + public Map> getRankPropertyMap() { + if (rankProperties.size() == 0 && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); + + var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") + .orElse(Map.of()); + if (rankProperties.isEmpty()) return inheritedProperties; + + // Neither is null + Map> combined = new LinkedHashMap<>(inheritedProperties); + combined.putAll(rankProperties); // Don't combine values across inherited properties + return Collections.unmodifiableMap(combined); + } + + public void addRankProperty(String name, String parameter) { + addRankProperty(new RankProperty(name, parameter)); + } + + private void addRankProperty(RankProperty rankProperty) { + // Just the usual multimap semantics here + rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty); + } + + public void setRerankCount(int rerankCount) { this.rerankCount = rerankCount; } + + public int getRerankCount() { + if (rerankCount >= 0) return rerankCount; + return uniquelyInherited(p -> p.getRerankCount(), c -> c >= 0, "rerank-count").orElse(-1); + } + + public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } + + public int getNumThreadsPerSearch() { + if (numThreadsPerSearch >= 0) return numThreadsPerSearch; + return uniquelyInherited(p -> p.getNumThreadsPerSearch(), n -> n >= 0, "num-threads-per-search") + .orElse(-1); + } + + public void setMinHitsPerThread(int minHits) { this.minHitsPerThread = minHits; } + + public int getMinHitsPerThread() { + if (minHitsPerThread >= 0) return minHitsPerThread; + return uniquelyInherited(p -> p.getMinHitsPerThread(), n -> n >= 0, "min-hits-per-search").orElse(-1); + } + + public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } + + public int getNumSearchPartitions() { + if (numSearchPartitions >= 0) return numSearchPartitions; + return uniquelyInherited(p -> p.getNumSearchPartitions(), n -> n >= 0, "num-search-partitions").orElse(-1); + } + + public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } + public void setPostFilterThreshold(double threshold) { this.postFilterThreshold = threshold; } + public void setApproximateThreshold(double threshold) { this.approximateThreshold = threshold; } + + public OptionalDouble getTermwiseLimit() { + if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); + return uniquelyInherited(p -> p.getTermwiseLimit(), l -> l.isPresent(), "termwise-limit") + .orElse(OptionalDouble.empty()); + } + + public OptionalDouble getPostFilterThreshold() { + if (postFilterThreshold != null) { + return OptionalDouble.of(postFilterThreshold); + } + return uniquelyInherited(p -> p.getPostFilterThreshold(), l -> l.isPresent(), "post-filter-threshold").orElse(OptionalDouble.empty()); + } + + public OptionalDouble getApproximateThreshold() { + if (approximateThreshold != null) { + return OptionalDouble.of(approximateThreshold); + } + return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); + } + + /** Whether we should ignore the default rank features. Set to null to use inherited */ + public void setIgnoreDefaultRankFeatures(Boolean ignoreDefaultRankFeatures) { + this.ignoreDefaultRankFeatures = ignoreDefaultRankFeatures; + } + + public Boolean getIgnoreDefaultRankFeatures() { + if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; + return uniquelyInherited(p -> p.getIgnoreDefaultRankFeatures(), "ignore-default-rank-features").orElse(false); + } + + public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } + + public int getKeepRankCount() { + if (keepRankCount >= 0) return keepRankCount; + return uniquelyInherited(p -> p.getKeepRankCount(), c -> c >= 0, "keep-rank-count").orElse(-1); + } + + public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } + + public double getRankScoreDropLimit() { + if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; + return uniquelyInherited(p -> p.getRankScoreDropLimit(), c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") + .orElse(rankScoreDropLimit); + } + + public void addFunction(String name, List arguments, String expression, boolean inline) { + try { + addFunction(parseRankingExpression(name, arguments, expression), inline); + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse function '" + name + "'", e); + } + } + + /** Adds a function and returns it */ + public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { + RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); + if (functions.containsKey(function.getName())) { + deployLogger.log(Level.WARNING, "Function '" + function.getName() + "' is defined twice " + + "in rank profile '" + this.name + "'"); + } + functions.put(function.getName(), rankingExpressionFunction); + allFunctionsCached = null; + return rankingExpressionFunction; + } + + /** + * Adds the type of an input feature consumed by this profile. + * All inputs must either be declared through this or in query profile types, + * otherwise they are assumes to be scalars. + */ + public void addInput(Reference reference, Input input) { + if (inputs.containsKey(reference)) { + Input existing = inputs().get(reference); + if (! input.equals(existing)) + throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); + } + inputs.put(reference, input); + } + + /** Returns the inputs of this, which also includes all inputs of the parents of this. */ + // This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles + // because inputs are tied closer to functions than the profile itself. + public Map inputs() { + if (inputs.isEmpty() && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs); + + // Combine + Map allInputs = new LinkedHashMap<>(); + for (var inheritedProfile : inherited()) { + for (var input : inheritedProfile.inputs().entrySet()) { + Input existing = allInputs.get(input.getKey()); + if (existing != null && ! existing.equals(input.getValue())) + throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + + input.getValue() + ", but this input is already defined as " + + existing + " in another profile this inherits"); + allInputs.put(input.getKey(), input.getValue()); + } + } + allInputs.putAll(inputs); + return Collections.unmodifiableMap(allInputs); + } + + public static class MutateOperation { + public enum Phase { on_match, on_first_phase, on_second_phase, on_summary} + final Phase phase; + final String attribute; + final String operation; + public MutateOperation(Phase phase, String attribute, String operation) { + this.phase = phase; + this.attribute = attribute; + this.operation = operation; + } + } + private final List mutateOperations = new ArrayList<>(); + + public void addMutateOperation(MutateOperation op) { + mutateOperations.add(op); + String prefix = "vespa.mutate." + op.phase.toString(); + addRankProperty(prefix + ".attribute", op.attribute); + addRankProperty(prefix + ".operation", op.operation); + } + public void addMutateOperation(MutateOperation.Phase phase, String attribute, String operation) { + addMutateOperation(new MutateOperation(phase, attribute, operation)); + } + public List getMutateOperations() { return mutateOperations; } + + public RankingExpressionFunction findFunction(String name) { + RankingExpressionFunction function = functions.get(name); + if (function != null) return function; + return uniquelyInherited(p -> p.findFunction(name), "function '" + name + "'").orElse(null); + } + + /** Returns an unmodifiable snapshot of the functions in this */ + public Map getFunctions() { + updateCachedFunctions(); + return allFunctionsCached.allRankingExpressionFunctions; + } + private ImmutableMap getExpressionFunctions() { + updateCachedFunctions(); + return allFunctionsCached.allExpressionFunctions; + } + private void updateCachedFunctions() { + if (needToUpdateFunctionCache()) { + allFunctionsCached = new CachedFunctions(gatherAllFunctions()); + } + } + + private Map gatherAllFunctions() { + if (functions.isEmpty() && inherited().isEmpty()) return Map.of(); + if (inherited().isEmpty()) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); + + // Combine + Map allFunctions = new LinkedHashMap<>(); + for (var inheritedProfile : inherited()) { + for (var function : inheritedProfile.getFunctions().entrySet()) { + if (allFunctions.containsKey(function.getKey())) + throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + + function.getValue() + ", but this function is already " + + "defined in another profile this inherits"); + allFunctions.put(function.getKey(), function.getValue()); + } + } + allFunctions.putAll(functions); + return Collections.unmodifiableMap(allFunctions); + } + + private boolean needToUpdateFunctionCache() { + if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; + return allFunctionsCached == null; + } + + public Set filterFields() { return filterFields; } + + /** Returns all filter fields in this profile and any profile it inherits. */ + public Set allFilterFields() { + Set inheritedFilterFields = uniquelyInherited(p -> p.allFilterFields(), fields -> ! fields.isEmpty(), + "filter fields").orElse(Set.of()); + + if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); + + Set combined = new LinkedHashSet<>(inheritedFilterFields); + combined.addAll(filterFields()); + return combined; + } + + private ExpressionFunction parseRankingExpression(String name, List arguments, String expression) throws ParseException { + if (expression.trim().length() == 0) + throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); + + try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { + return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); + } + catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { + ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + + "' in " + name() + ", " + name + "."); + throw (ParseException)exception.initCause(e); + } + catch (IOException e) { + throw new RuntimeException("IOException parsing ranking expression '" + name + "'", e); + } + } + + private static String extractFileName(String expression) { + String fileName = expression.substring("file:".length()).trim(); + if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX)) + fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX; + + return fileName; + } + + private Reader openRankingExpressionReader(String expName, String expression) { + if (!expression.startsWith("file:")) return new StringReader(expression); + + String fileName = extractFileName(expression); + Path.fromString(fileName); // No ".." + if (fileName.contains("/")) // See ticket 4102122 + throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + + fileName + "' in a different directory, which is not supported."); + + return schema.getRankingExpression(fileName); + } + + /** Shallow clones this */ + @Override + public RankProfile clone() { + try { + RankProfile clone = (RankProfile)super.clone(); + clone.rankSettings = new LinkedHashSet<>(this.rankSettings); + clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? + clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; + clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null; + clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; + clone.rankProperties = new LinkedHashMap<>(this.rankProperties); + clone.inputs = new LinkedHashMap<>(this.inputs); + clone.functions = new LinkedHashMap<>(this.functions); + clone.allFunctionsCached = null; + clone.filterFields = new HashSet<>(this.filterFields); + clone.constants = new HashMap<>(this.constants); + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + + /** + * Returns a copy of this where the content is optimized for execution. + * Compiled profiles should never be modified. + */ + public RankProfile compile(QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { + try { + RankProfile compiled = this.clone(); + compiled.compileThis(queryProfiles, importedModels); + return compiled; + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Rank profile '" + name() + "' is invalid", e); + } + } + + private void compileThis(QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { + checkNameCollisions(getFunctions(), constants()); + ExpressionTransforms expressionTransforms = new ExpressionTransforms(); + + Map featureTypes = featureTypes(); + // Function compiling first pass: compile inline functions without resolving other functions + Map inlineFunctions = + compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); + + firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); + + // Function compiling second pass: compile all functions and insert previously compiled inline functions + // TODO: This merges all functions from inherited profiles too and erases inheritance information. Not good. + functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); + allFunctionsCached = null; + } + + private void checkNameCollisions(Map functions, Map constants) { + for (var functionEntry : functions.entrySet()) { + if (constants.containsKey(FeatureNames.asConstantFeature(functionEntry.getKey()))) + throw new IllegalArgumentException("Cannot have both a constant and function named '" + + functionEntry.getKey() + "'"); + } + } + + private Map getInlineFunctions() { + return getFunctions().entrySet().stream().filter(x -> x.getValue().inline()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Map compileFunctions(Supplier> functions, + QueryProfileRegistry queryProfiles, + Map featureTypes, + ImportedMlModels importedModels, + Map inlineFunctions, + ExpressionTransforms expressionTransforms) { + Map compiledFunctions = new LinkedHashMap<>(); + Map.Entry entry; + // Compile all functions. Why iterate in such a complicated way? + // Because some functions (imported models adding generated functions) may add other functions during compiling. + // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException + while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { + RankingExpressionFunction rankingExpressionFunction = entry.getValue(); + RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, + importedModels, constants(), inlineFunctions, + expressionTransforms); + compiledFunctions.put(entry.getKey(), compiled); + } + return compiledFunctions; + } + + private static Map.Entry findUncompiledFunction(Map functions, + Set compiledFunctionNames) { + for (Map.Entry entry : functions.entrySet()) { + if ( ! compiledFunctionNames.contains(entry.getKey())) + return entry; + } + return null; + } + + private RankingExpressionFunction compile(RankingExpressionFunction function, + QueryProfileRegistry queryProfiles, + Map featureTypes, + ImportedMlModels importedModels, + Map constants, + Map inlineFunctions, + ExpressionTransforms expressionTransforms) { + if (function == null) return null; + + RankProfileTransformContext context = new RankProfileTransformContext(this, + queryProfiles, + featureTypes, + importedModels, + constants, + inlineFunctions); + RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context); + for (Map.Entry rankProperty : context.rankProperties().entrySet()) { + addRankProperty(rankProperty.getKey(), rankProperty.getValue()); + } + return function.withExpression(expression); + } + + /** + * Creates a context containing the type information of all constants, attributes and query profiles + * referable from this rank profile. + */ + public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles) { + return typeContext(queryProfiles, featureTypes()); + } + + public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); } + + private Map featureTypes() { + Map featureTypes = inputs().values().stream() + .collect(Collectors.toMap(input -> input.name(), input -> input.type())); + allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); + allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); + return featureTypes; + } + + public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, + Map featureTypes) { + MapEvaluationTypeContext context = new MapEvaluationTypeContext(getExpressionFunctions(), featureTypes); + + constants().forEach((k, v) -> context.setType(k, v.type())); + + // Add query features from all rank profile types + for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { + for (FieldDescription field : queryProfileType.declaredFields().values()) { + TensorType type = field.getType().asTensorType(); + Optional feature = Reference.simple(field.getName()); + if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; + if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides + + TensorType existingType = context.getType(feature.get()); + if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) + type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> + new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + + " with type " + field.getType().asTensorType() + + ", but this is already defined in another query profile with type " + + context.getType(feature.get()))); + context.setType(feature.get(), type); + } + } + + // Add output types for ONNX models + for (var model : onnxModels().values()) { + Arguments args = new Arguments(new ReferenceNode(model.getName())); + Map inputTypes = resolveOnnxInputTypes(model, context); + + TensorType defaultOutputType = model.getTensorType(model.getDefaultOutput(), inputTypes); + context.setType(new Reference("onnxModel", args, null), defaultOutputType); + + for (Map.Entry mapping : model.getOutputMap().entrySet()) { + TensorType type = model.getTensorType(mapping.getKey(), inputTypes); + context.setType(new Reference("onnxModel", args, mapping.getValue()), type); + } + } + return context; + } + + private Map resolveOnnxInputTypes(OnnxModel model, MapEvaluationTypeContext context) { + Map inputTypes = new HashMap<>(); + for (String onnxInputName : model.getInputMap().keySet()) { + resolveOnnxInputType(onnxInputName, model, context).ifPresent(type -> inputTypes.put(onnxInputName, type)); + } + return inputTypes; + } + + private Optional resolveOnnxInputType(String onnxInputName, OnnxModel model, MapEvaluationTypeContext context) { + String source = model.getInputMap().get(onnxInputName); + if (source != null) { + // Source is either a simple reference (query/attribute/constant/rankingExpression)... + Optional reference = Reference.simple(source); + if (reference.isPresent()) { + if (reference.get().name().equals("rankingExpression") && reference.get().simpleArgument().isPresent()) { + source = reference.get().simpleArgument().get(); // look up function below + } else { + return Optional.of(context.getType(reference.get())); + } + } + // ... or a function + ExpressionFunction func = context.getFunction(source); + if (func != null) { + return Optional.of(func.getBody().type(context)); + } + } + return Optional.empty(); // if this context does not contain this input + } + + private void addAttributeFeatureTypes(ImmutableSDField field, Map featureTypes) { + Attribute attribute = field.getAttribute(); + field.getAttributes().forEach((k, a) -> { + String name = k; + if (attribute == a) // this attribute should take the fields name + name = field.getName(); // switch to that - it is separate for imported fields + featureTypes.put(FeatureNames.asAttributeFeature(name), + a.tensorType().orElse(TensorType.empty)); + }); + } + + @Override + public String toString() { + return "rank profile '" + name() + "'"; + } + + /** + * A rank setting. The identity of a rank setting is its field name and type (not value). + * A rank setting is immutable. + */ + public static class RankSetting implements Serializable { + + private final String fieldName; + + private final Type type; + + /** The rank value */ + private final Object value; + + public enum Type { + + RANKTYPE("rank-type"), + LITERALBOOST("literal-boost"), + WEIGHT("weight"), + PREFERBITVECTOR("preferbitvector",true); + + private final String name; + + /** True if this setting really pertains to an index, not a field within an index */ + private final boolean isIndexLevel; + + Type(String name) { + this(name,false); + } + + Type(String name,boolean isIndexLevel) { + this.name = name; + this.isIndexLevel=isIndexLevel; + } + + /** True if this setting really pertains to an index, not a field within an index */ + public boolean isIndexLevel() { return isIndexLevel; } + + /** Returns the name of this type */ + public String getName() { + return name; + } + + @Override + public String toString() { + return "type " + name; + } + + } + + public RankSetting(String fieldName, RankSetting.Type type, Object value) { + this.fieldName = fieldName; + this.type = type; + this.value = value; + } + + public String getFieldName() { return fieldName; } + + public Type getType() { return type; } + + public Object getValue() { return value; } + + /** Returns the value as an int, or a negative value if it is not an integer */ + public int getIntValue() { + if (value instanceof Integer) { + return ((Integer)value); + } + else { + return -1; + } + } + + @Override + public int hashCode() { + return fieldName.hashCode() + 17 * type.hashCode(); + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof RankSetting)) { + return false; + } + RankSetting other = (RankSetting)object; + return + fieldName.equals(other.fieldName) && + type.equals(other.type); + } + + @Override + public String toString() { + return type + " setting " + fieldName + ": " + value; + } + + } + + /** A rank property. Rank properties are Value Objects */ + public static class RankProperty implements Serializable { + + private final String name; + private final String value; + + public RankProperty(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { return name; } + + public String getValue() { return value; } + + @Override + public int hashCode() { + return name.hashCode() + 17 * value.hashCode(); + } + + @Override + public boolean equals(Object object) { + if (! (object instanceof RankProperty)) return false; + RankProperty other=(RankProperty)object; + return (other.name.equals(this.name) && other.value.equals(this.value)); + } + + @Override + public String toString() { + return name + " = " + value; + } + + } + + /** A function in a rank profile */ + public static class RankingExpressionFunction { + + private ExpressionFunction function; + + /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ + private final boolean inline; + + RankingExpressionFunction(ExpressionFunction function, boolean inline) { + this.function = function; + this.inline = inline; + } + + public void setReturnType(TensorType type) { + this.function = function.withReturnType(type); + } + + public ExpressionFunction function() { return function; } + + public boolean inline() { + return inline && function.arguments().isEmpty(); // only inline no-arg functions; + } + + RankingExpressionFunction withExpression(RankingExpression expression) { + return new RankingExpressionFunction(function.withBody(expression), inline); + } + + @Override + public String toString() { + return function.toString(); + } + + } + + public static final class DiversitySettings { + + private String attribute = null; + private int minGroups = 0; + private double cutoffFactor = 10; + private Diversity.CutoffStrategy cutoffStrategy = Diversity.CutoffStrategy.loose; + + public void setAttribute(String value) { attribute = value; } + public void setMinGroups(int value) { minGroups = value; } + public void setCutoffFactor(double value) { cutoffFactor = value; } + public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; } + public String getAttribute() { return attribute; } + public int getMinGroups() { return minGroups; } + public double getCutoffFactor() { return cutoffFactor; } + public Diversity.CutoffStrategy getCutoffStrategy() { return cutoffStrategy; } + + void checkValid() { + if (attribute == null || attribute.isEmpty()) { + throw new IllegalArgumentException("'diversity' did not set non-empty diversity attribute name."); + } + if (minGroups <= 0) { + throw new IllegalArgumentException("'diversity' did not set min-groups > 0"); + } + if (cutoffFactor < 1.0) { + throw new IllegalArgumentException("diversity.cutoff.factor must be larger or equal to 1.0."); + } + } + } + + public static class MatchPhaseSettings { + + private String attribute = null; + private boolean ascending = false; + private int maxHits = 0; // try to get this many hits before degrading the match phase + private double maxFilterCoverage = 0.2; // Max coverage of original corpus that will trigger the filter. + private DiversitySettings diversity = null; + private double evaluationPoint = 0.20; + private double prePostFilterTippingPoint = 1.0; + + public void setDiversity(DiversitySettings value) { + value.checkValid(); + diversity = value; + } + + public void setAscending(boolean value) { ascending = value; } + public void setAttribute(String value) { attribute = value; } + public void setMaxHits(int value) { maxHits = value; } + public void setMaxFilterCoverage(double value) { maxFilterCoverage = value; } + public void setEvaluationPoint(double evaluationPoint) { this.evaluationPoint = evaluationPoint; } + public void setPrePostFilterTippingPoint(double prePostFilterTippingPoint) { this.prePostFilterTippingPoint = prePostFilterTippingPoint; } + + public boolean getAscending() { return ascending; } + public String getAttribute() { return attribute; } + public int getMaxHits() { return maxHits; } + public double getMaxFilterCoverage() { return maxFilterCoverage; } + public DiversitySettings getDiversity() { return diversity; } + public double getEvaluationPoint() { return evaluationPoint; } + public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } + + public void checkValid() { + if (attribute == null) { + throw new IllegalArgumentException("match-phase did not set any attribute"); + } + if (! (maxHits > 0)) { + throw new IllegalArgumentException("match-phase did not set max-hits > 0"); + } + } + + } + + public static class TypeSettings { + + private final Map types = new HashMap<>(); + + void addType(String name, String type) { + types.put(name, type); + } + + public Map getTypes() { + return Collections.unmodifiableMap(types); + } + + } + + public static final class Input { + + private final Reference name; + private final TensorType type; + private final Optional defaultValue; + + public Input(Reference name, TensorType type, Optional defaultValue) { + this.name = name; + this.type = type; + this.defaultValue = defaultValue; + } + + public Reference name() { return name; } + public TensorType type() { return type; } + public Optional defaultValue() { return defaultValue; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Input)) return false; + Input other = (Input)o; + if ( ! other.name().equals(this.name())) return false; + if ( ! other.type().equals(this.type())) return false; + if ( ! other.defaultValue().equals(this.defaultValue())) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, defaultValue); + } + + @Override + public String toString() { + return "input '" + name + "' " + type + + (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : ""); + } + + } + + public static final class Constant { + + private final Reference name; + private final TensorType type; + + // One of these are non-empty + private final Optional value; + private final Optional valuePath; + + // Always set only if valuePath is set + private final Optional pathType; + + public Constant(Reference name, Tensor value) { + this(name, value.type(), Optional.of(value), Optional.empty(), Optional.empty()); + } + + public Constant(Reference name, TensorType type, String valuePath) { + this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(DistributableResource.PathType.FILE)); + } + + public Constant(Reference name, TensorType type, String valuePath, DistributableResource.PathType pathType) { + this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(pathType)); + } + + private Constant(Reference name, TensorType type, Optional value, + Optional valuePath, Optional pathType) { + this.name = Objects.requireNonNull(name); + this.type = Objects.requireNonNull(type); + this.value = Objects.requireNonNull(value); + this.valuePath = Objects.requireNonNull(valuePath); + this.pathType = Objects.requireNonNull(pathType); + + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type + + ": Dense tensor dimensions must have a size"); + } + + public Reference name() { return name; } + public TensorType type() { return type; } + + /** Returns the value of this, if its path is empty. */ + public Optional value() { return value; } + + /** Returns the path to the value of this, if its value is empty. */ + public Optional valuePath() { return valuePath; } + + /** Returns the path type, if valuePath is set. */ + public Optional pathType() { return pathType; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Constant)) return false; + Constant other = (Constant)o; + if ( ! other.name().equals(this.name())) return false; + if ( ! other.type().equals(this.type())) return false; + if ( ! other.value().equals(this.value())) return false; + if ( ! other.valuePath().equals(this.valuePath())) return false; + if ( ! other.pathType().equals(this.pathType())) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, value, valuePath, pathType); + } + + @Override + public String toString() { + return "constant '" + name + "' " + type + ":" + + (value().isPresent() ? value.get().toAbbreviatedString() : " file:" + valuePath.get()); + } + + } + + private static class CachedFunctions { + + private final Map allRankingExpressionFunctions; + + private final ImmutableMap allExpressionFunctions; + + CachedFunctions(Map functions) { + allRankingExpressionFunctions = functions; + ImmutableMap.Builder mapBuilder = new ImmutableMap.Builder<>(); + for (var entry : functions.entrySet()) { + ExpressionFunction function = entry.getValue().function(); + mapBuilder.put(function.getName(), function); + } + allExpressionFunctions = mapBuilder.build(); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java new file mode 100644 index 00000000000..06ffc934b2d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java @@ -0,0 +1,138 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDDocumentType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Mapping from name to {@link RankProfile} as well as a reverse mapping of {@link RankProfile} to {@link Schema}. + * Having both of these mappings consolidated here make it easier to remove dependencies on these mappings at + * run time, since it is essentially only used when building rank profile config at deployment time. + * + * Global rank profiles are represented by the Search key null. + * + * @author Ulf Lilleengen + */ +public class RankProfileRegistry { + + private final Map> rankProfiles = new LinkedHashMap<>(); + private static final String globalRankProfilesKey = "[global]"; + + /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ + static final Set overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); + + public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Schema schema) { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); + return rankProfileRegistry; + } + + private String extractName(ImmutableSchema search) { + return search != null ? search.getName() : globalRankProfilesKey; + } + + /** Adds a rank profile to this registry */ + public void add(RankProfile rankProfile) { + String schemaName = extractName(rankProfile.schema()); + if ( ! rankProfiles.containsKey(schemaName)) { + rankProfiles.put(schemaName, new LinkedHashMap<>()); + } + checkForDuplicate(rankProfile); + rankProfiles.get(schemaName).put(rankProfile.name(), rankProfile); + } + + private void checkForDuplicate(RankProfile rankProfile) { + String rankProfileName = rankProfile.name(); + RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.schema())).get(rankProfileName); + if (existingRankProfileWithSameName == null) return; + + if ( ! overridableRankProfileNames.contains(rankProfileName)) { + throw new IllegalArgumentException("Duplicate rank profile '" + rankProfileName + "' in " + + rankProfile.schema()); + } + } + + /** + * Returns a named rank profile, null if the search definition doesn't have one with the given name + * + * @param schema the {@link Schema} that owns the rank profile + * @param name the name of the rank profile + * @return the RankProfile to return. + */ + public RankProfile get(String schema, String name) { + Map profiles = rankProfiles.get(schema); + if (profiles == null) return null; + return profiles.get(name); + } + + public RankProfile get(ImmutableSchema schema, String name) { + var profile = get(schema.getName(), name); + if (profile != null) return profile; + if (schema.inherited().isPresent()) return get(schema.inherited().get(), name); + return null; + } + + public RankProfile getGlobal(String name) { + Map profiles = rankProfiles.get(globalRankProfilesKey); + if (profiles == null) return null; + return profiles.get(name); + } + + public RankProfile resolve(SDDocumentType docType, String name) { + RankProfile rankProfile = get(docType.getName(), name); + if (rankProfile != null) return rankProfile; + for (var parent : docType.getInheritedTypes()) { + RankProfile parentProfile = resolve(parent, name); + if (parentProfile != null) return parentProfile; + } + return get(globalRankProfilesKey, name); + } + + /** + * Rank profiles that are collected across clusters. + * + * @return a set of global {@link RankProfile} instances + */ + public Collection all() { + List all = new ArrayList<>(); + for (var entry : rankProfiles.values()) { + all.addAll(entry.values()); + } + return all; + } + + /** + * Retrieve all rank profiles for a schema + * + * @param schema the schema to fetch rank profiles for, or null for the global ones + * @return a collection of {@link RankProfile} instances + */ + public Collection rankProfilesOf(ImmutableSchema schema) { + String key = schema == null ? globalRankProfilesKey : schema.getName(); + + if ( ! rankProfiles.containsKey(key)) return List.of(); + + var profiles = new LinkedHashMap<>(rankProfiles.get(key)); + // Add all profiles in inherited schemas, unless they are already present (overridden) + while (schema != null && schema.inherited().isPresent()) { + schema = schema.inherited().get(); + var inheritedProfiles = rankProfiles.get(schema.getName()); + if (inheritedProfiles != null) { + for (Map.Entry inheritedProfile : inheritedProfiles.entrySet()) { + profiles.putIfAbsent(inheritedProfile.getKey(), inheritedProfile.getValue()); + } + } + } + return profiles.values(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java new file mode 100644 index 00000000000..64bca9367d8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java @@ -0,0 +1,136 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.TemporarySDDocumentType; + +import java.util.*; +import java.util.logging.Level; + +/** + * @author Einar M R Rosenvinge + */ +public class SDDocumentTypeOrderer { + + private final Map createdSDTypes = new LinkedHashMap<>(); + private final Set seenTypes = Collections.newSetFromMap(new IdentityHashMap<>()); + List processingOrder = new LinkedList<>(); + private final DeployLogger deployLogger; + + public SDDocumentTypeOrderer(List sdTypes, DeployLogger deployLogger) { + this.deployLogger = deployLogger; + for (SDDocumentType type : sdTypes) { + createdSDTypes.put(type.getDocumentName(), type); + } + } + + List getOrdered() { return processingOrder; } + + public void process() { + for (SDDocumentType type : createdSDTypes.values()) { + process(type, type); + } + } + + private void process(SDDocumentType docOrStruct, SDDocumentType owningDocument) { + resolveAndProcessInheritedTemporaryTypes(docOrStruct, owningDocument); + if (seenTypes.contains(docOrStruct)) { + return; + } + seenTypes.add(docOrStruct); + for (Field field : docOrStruct.fieldSet()) { + var type = field.getDataType(); + String typeName = type.getName(); + if (!seenTypes.contains(type)) { + seenTypes.add(type); + //we haven't seen this before, do it + visit(type, owningDocument); + } + } + processingOrder.add(docOrStruct); + } + + private void resolveAndProcessInheritedTemporaryTypes(SDDocumentType type, SDDocumentType owningDocument) { + List toReplace = new ArrayList<>(); + for (SDDocumentType sdoc : type.getInheritedTypes()) { + if (sdoc instanceof TemporarySDDocumentType) { + toReplace.add(sdoc.getDocumentName()); + } + } + for (DataTypeName name : toReplace) { + SDDocumentType inherited; + if (type.isStruct()) { + inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); + if (inherited == null) throw new IllegalArgumentException("Struct '" + name + "' not found in " + owningDocument); + process(inherited, owningDocument); + } + else { + inherited = createdSDTypes.get(name); + if (inherited == null) { + throw new IllegalArgumentException("document " + type.getName() + + " inherits from unavailable document " + name); + } + process(inherited, inherited); + } + type.inherit(inherited); + } + } + + private SDDocumentType find(String name) { + SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); + if (sdDocType != null) { + return sdDocType; + } + for(SDDocumentType sdoc : createdSDTypes.values()) { + for (SDDocumentType stype : sdoc.getTypes()) { + if (stype.getName().equals(name)) { + return stype; + } + } + } + return null; + } + + private void visit(DataType type, SDDocumentType owningDocument) { + if (type instanceof StructuredDataType) { + StructuredDataType structType = (StructuredDataType) type; + SDDocumentType sdDocType = owningDocument.getType(structType.getName()); + if (sdDocType == null) { + sdDocType = find(structType.getName()); + } + if (sdDocType == null) { + throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'"); + } + process(sdDocType, owningDocument); + return; + } + + if (type instanceof MapDataType) { + MapDataType mType = (MapDataType) type; + visit(mType.getValueType(), owningDocument); + visit(mType.getKeyType(), owningDocument); + } else if (type instanceof WeightedSetDataType) { + WeightedSetDataType wType = (WeightedSetDataType) type; + visit(wType.getNestedType(), owningDocument); + } else if (type instanceof CollectionDataType) { + CollectionDataType cType = (CollectionDataType) type; + visit(cType.getNestedType(), owningDocument); + } else if (type instanceof AnnotationReferenceDataType) { + //do nothing + } else if (type instanceof PrimitiveDataType) { + //do nothing + } else if (type instanceof TensorDataType) { + //do nothing + } else if (type instanceof NewDocumentReferenceDataType) { + //do nothing + } else { + deployLogger.logApplicationPackage(Level.WARNING, "Unknown type : " + type); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java new file mode 100644 index 00000000000..c733b6012f9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -0,0 +1,754 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.Field; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.document.TemporaryImportedFields; +import com.yahoo.schema.document.annotation.SDAnnotationType; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.stream.Stream; + +/** + * A schema contains a document type, additional fields, rank profiles and document summaries. + * + * @author bratseth + */ +// TODO: Make a class owned by this, for each of these responsibilities: +// Managing indexes, managing attributes, managing summary classes. +// Ensure that after the processing step, all implicit instances of the above types are explicitly represented +public class Schema implements ImmutableSchema { + + private static final String SD_DOC_FIELD_NAME = "sddocname"; + private static final List RESERVED_NAMES = List.of( + "index", "index_url", "summary", "attribute", "select_input", "host", SummaryClass.DOCUMENT_ID_FIELD, + "position", "split_foreach", "tokenize", "if", "else", "switch", "case", SD_DOC_FIELD_NAME, "relevancy"); + + /** The unique name of this schema */ + private String name; + + /** The application package this is constructed from */ + private final ApplicationPackage applicationPackage; + + /** The name of the schema this should inherit all the content of, if any */ + private final Optional inherited; + + /** True if this doesn't define a search, just a document type */ + private final boolean documentsOnly; + + private Boolean rawAsBase64 = null; + + /** The stemming setting of this schema. Default is BEST. */ + private Stemming stemming = null; + + private final FieldSets fieldSets = new FieldSets(Optional.of(this)); + + /** The document contained in this schema */ + private SDDocumentType documentType; + + /** The extra fields of this schema */ + private final Map fields = new LinkedHashMap<>(); + + private final Map indices = new LinkedHashMap<>(); + + /** The explicitly defined summaries of this schema. _Must_ preserve order. */ + private final Map summaries = new LinkedHashMap<>(); + + /** External rank expression files of this */ + private final LargeRankExpressions largeRankExpressions; + + /** Constants that will be available in all rank profiles. */ + // TODO: Remove on Vespa 9: Should always be in a rank profile + private final Map constants = new LinkedHashMap<>(); + + // TODO: Remove on Vespa 9: Should always be in a rank profile + private final Map onnxModels = new LinkedHashMap<>(); + + /** All imported fields of this (and parent schemas) */ + // TODO: Use empty, not optional + // TODO: Merge this and importedFields + private final Optional temporaryImportedFields = Optional.of(new TemporaryImportedFields(this)); + /** The resulting processed field */ + private Optional importedFields = Optional.empty(); + + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + + private Application owner; + + /** Testing only */ + public Schema(String name, ApplicationPackage applicationPackage) { + this(name, applicationPackage, Optional.empty(), null, new BaseDeployLogger(), new TestProperties()); + } + + public Schema(String name, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties) { + this(name, applicationPackage, Optional.empty(), fileRegistry, deployLogger, properties); + } + + /** + * Creates a schema + * + * @param name of the schema + * @param inherited the schema this inherits, if any + */ + public Schema(String name, + ApplicationPackage applicationPackage, + Optional inherited, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties) { + this(inherited, applicationPackage, fileRegistry, deployLogger, properties, false); + this.name = Objects.requireNonNull(name, "A schema must have a name"); + } + + protected Schema(ApplicationPackage applicationPackage, FileRegistry fileRegistry, + DeployLogger deployLogger, ModelContext.Properties properties) { + this(Optional.empty(), applicationPackage, fileRegistry, deployLogger, properties, true); + } + + private Schema(Optional inherited, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + boolean documentsOnly) { + this.inherited = inherited; + this.applicationPackage = applicationPackage; + this.deployLogger = deployLogger; + this.properties = properties; + this.documentsOnly = documentsOnly; + largeRankExpressions = new LargeRankExpressions(fileRegistry); + } + + /** + * Assigns the owner of this + * + * @throws IllegalStateException if an owner is already assigned + */ + public void setOwner(Application owner) { + if (this.owner != null) + throw new IllegalStateException("Cannot reassign the owner of " + this); + this.owner = owner; + } + + protected void setName(String name) { this.name = name; } + + @Override + public String getName() {return name; } + + /** Returns true if this only defines a document type, not a full schema */ + public boolean isDocumentsOnly() { + return documentsOnly; + } + + @Override + public Optional inherited() { + return inherited.map(name -> owner.schemas().get(name)); + } + + /** + * Returns true if 'raw' fields shall be presented as base64 in summary + * Note that this is temporary and will disappear on Vespa 8 as it will become default, and only option. + * + * @return true if raw shall be encoded as base64 in summary + */ + public boolean isRawAsBase64() { + if (rawAsBase64 != null) return rawAsBase64; + // TODO Vespa 8: flip default: + if (inherited.isEmpty()) return false; + return requireInherited().isRawAsBase64(); + } + + public void enableRawAsBase64(boolean value) { rawAsBase64 = value; } + + /** + * Sets the stemming default of fields. Default is ALL + * + * @param stemming set default stemming for this searchdefinition + * @throws NullPointerException if this is attempted set to null + */ + public void setStemming(Stemming stemming) { + this.stemming = Objects.requireNonNull(stemming, "Stemming cannot be null"); + } + + /** Returns whether fields should be stemmed by default or not. Default is BEST. This is never null. */ + public Stemming getStemming() { + if (stemming != null) return stemming; + if (inherited.isEmpty()) return Stemming.BEST; + return requireInherited().getStemming(); + } + + /** + * Adds a document type which is defined in this search definition + * + * @param document the document type to add + */ + public void addDocument(SDDocumentType document) { + if (documentType != null) { + throw new IllegalArgumentException("Schema cannot have more than one document"); + } + documentType = document; + } + + @Override + public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } + + public void add(RankProfile.Constant constant) { + constants.put(constant.name(), constant); + } + + /** Returns an unmodifiable map of the constants declared in this. */ + public Map declaredConstants() { return constants; } + + /** Returns an unmodifiable map of the constants available in this. */ + @Override + public Map constants() { + if (inherited().isEmpty()) return Collections.unmodifiableMap(constants); + if (constants.isEmpty()) return inherited().get().constants(); + + Map allConstants = new LinkedHashMap<>(inherited().get().constants()); + allConstants.putAll(constants); + return allConstants; + } + + public void add(OnnxModel model) { + onnxModels.put(model.getName(), model); + } + + /** Returns an unmodifiable map of the onnx models declared in this. */ + public Map declaredOnnxModels() { return onnxModels; } + + /** Returns an unmodifiable map of the onnx models available in this. */ + @Override + public Map onnxModels() { + if (inherited().isEmpty()) return Collections.unmodifiableMap(onnxModels); + if (onnxModels.isEmpty()) return inherited().get().onnxModels(); + + Map allModels = new LinkedHashMap<>(inherited().get().onnxModels()); + allModels.putAll(onnxModels); + return allModels; + } + + public Optional temporaryImportedFields() { + return temporaryImportedFields; + } + + public Optional importedFields() { + return importedFields; + } + + public void setImportedFields(ImportedFields importedFields) { + this.importedFields = Optional.of(importedFields); + } + + @Override + public Stream allImportedFields() { + return importedFields + .map(fields -> fields.fields().values().stream()) + .orElse(Stream.empty()) + .map(field -> field.asImmutableSDField()); + } + + @Override + public ImmutableSDField getField(String name) { + ImmutableSDField field = getConcreteField(name); + if (field != null) return field; + return allImportedFields() + .filter(f -> f.getName().equals(name)) + .findFirst() + .orElse(null); + } + + @Override + public List allFieldsList() { + List all = new ArrayList<>(); + all.addAll(extraFieldList()); + for (Field field : documentType.fieldSet()) { + all.add((ImmutableSDField) field); + } + if (importedFields.isPresent()) { + for (ImportedField imported : importedFields.get().fields().values()) { + all.add(imported.asImmutableSDField()); + } + } + return all; + } + + /** + * Gets a document from this search definition + * + * @param name the name of the document to return + * @return the contained or used document type, or null if there is no such document + */ + public SDDocumentType getDocument(String name) { + if (documentType != null && name.equals(documentType.getName())) { + return documentType; + } + return null; + } + + /** + * @return true if the document has been added. + */ + public boolean hasDocument() { + return documentType != null; + } + + /** + * @return The document in this search. + */ + @Override + public SDDocumentType getDocument() { + return documentType; + } + + /** + * Returns a list of all the fields of this search definition, that is all fields in all documents, in the documents + * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not + * impact this + */ + @Override + public List allConcreteFields() { + List allFields = new ArrayList<>(); + allFields.addAll(extraFieldList()); + for (Field field : documentType.fieldSet()) { + allFields.add((SDField)field); + } + return allFields; + } + + /** + * Returns the content of a ranking expression file + */ + @Override + public Reader getRankingExpression(String fileName) { + return applicationPackage.getRankingExpression(fileName); + } + + public Application application() { return owner; } + + @Override + public ApplicationPackage applicationPackage() { return applicationPackage; } + + @Override + public DeployLogger getDeployLogger() { return deployLogger; } + + @Override + public ModelContext.Properties getDeployProperties() { return properties; } + + /** + * Returns a field defined in this search definition or one if its documents. Fields in this search definition takes + * precedence over document fields having the same name + * + * @param name of the field + * @return the SDField representing the field + */ + @Override + public SDField getConcreteField(String name) { + SDField field = getExtraField(name); + if (field != null) return field; + + return (SDField) documentType.getField(name); + } + + /** + * Returns a field defined in one of the documents of this search definition. + * This does not include the extra fields defined outside the document + * (those accessible through the getExtraField() method). + * + * @param name the name of the field to return + * @return the named field, or null if not found + */ + public SDField getDocumentField(String name) { + return (SDField) documentType.getField(name); + } + + /** + * Adds an extra field of this search definition not contained in a document + * + * @param field to add to the schemas list of external fields + */ + public void addExtraField(SDField field) { + if (fields.containsKey(field.getName())) { + deployLogger.logApplicationPackage(Level.WARNING, "Duplicate field " + field.getName() + " in search definition " + getName()); + } else { + field.setIsExtraField(true); + fields.put(field.getName(), field); + } + } + + public Collection extraFieldList() { + if (inherited.isEmpty()) return fields.values(); + var fields = new HashSet<>(requireInherited().extraFieldList()); + fields.addAll(this.fields.values()); + return fields; + } + + public Collection allExtraFields() { + Map extraFields = new TreeMap<>(); + if (inherited.isPresent()) + requireInherited().allExtraFields().forEach(field -> extraFields.put(field.getName(), field)); + for (Field field : documentType.fieldSet()) { + SDField sdField = (SDField) field; + if (sdField.isExtraField()) { + extraFields.put(sdField.getName(), sdField); + } + } + for (SDField field : extraFieldList()) { + extraFields.put(field.getName(), field); + } + return extraFields.values(); + } + + /** + * Returns a field by name, or null if it is not present + * + * @param fieldName the name of the external field to get + * @return the SDField of this name + */ + public SDField getExtraField(String fieldName) { + SDField field = fields.get(fieldName); + if (field != null) return field; + if (inherited.isEmpty()) return null; + return requireInherited().getExtraField(fieldName); + } + + /** + * Adds an explicitly defined index to this search definition + * + * @param index the index to add + */ + public void addIndex(Index index) { + indices.put(index.getName(), index); + } + + /** + * Returns an index, or null if no index with this name has had some explicit settings applied. Even if + * this returns null, the index may be implicitly defined by an indexing statement. This will return the + * index whether it is defined on this schema or on one of its fields. + * + * @param name the name of the index to get + * @return the index requested + */ + @Override + public Index getIndex(String name) { + List sameIndices = new ArrayList<>(1); + + getSchemaIndex(name).ifPresent(sameIndices::add); + + for (ImmutableSDField field : allConcreteFields()) { + if (field.getIndex(name) != null) + sameIndices.add(field.getIndex(name)); + } + if (sameIndices.size() == 0) return null; + if (sameIndices.size() == 1) return sameIndices.get(0); + return consolidateIndices(sameIndices); + } + + /** Returns the schema level index of this name, in this or any inherited schema, if any */ + Optional getSchemaIndex(String name) { + if (indices.containsKey(name)) return Optional.of(indices.get(name)); + if (inherited.isPresent()) return requireInherited().getSchemaIndex(name); + return Optional.empty(); + } + + public boolean existsIndex(String name) { + if (indices.get(name) != null) + return true; + if (inherited.isPresent() && requireInherited().existsIndex(name)) + return true; + for (ImmutableSDField field : allConcreteFields()) { + if (field.existsIndex(name)) + return true; + } + return false; + } + + /** + * Consolidates a set of index settings for the same index into one + * + * @param indices the list of indexes to consolidate + * @return the consolidated index + */ + private Index consolidateIndices(List indices) { + Index first = indices.get(0); + Index consolidated = new Index(first.getName()); + consolidated.setRankType(first.getRankType()); + consolidated.setType(first.getType()); + for (Index current : indices) { + if (current.isPrefix()) { + consolidated.setPrefix(true); + } + if (current.useInterleavedFeatures()) { + consolidated.setInterleavedFeatures(true); + } + + if (consolidated.getRankType() == null) { + consolidated.setRankType(current.getRankType()); + } else { + if (current.getRankType() != null && consolidated.getRankType() != current.getRankType()) + deployLogger.logApplicationPackage(Level.WARNING, "Conflicting rank type settings for " + + first.getName() + " in " + this + ", using " + + consolidated.getRankType()); + } + + for (Iterator j = current.aliasIterator(); j.hasNext();) { + consolidated.addAlias(j.next()); + } + } + return consolidated; + } + + /** All explicitly defined indices, both on this schema itself (returned first) and all its fields */ + @Override + public List getExplicitIndices() { + List allIndices = new ArrayList<>(indices.values()); + + if (inherited.isPresent()) { + for (Index inheritedIndex : requireInherited().getExplicitIndices()) { + if ( ! indices.containsKey(inheritedIndex.getName())) // child redefinitions shadows parents + allIndices.add(inheritedIndex); + } + } + + for (ImmutableSDField field : allConcreteFields()) + allIndices.addAll(field.getIndices().values()); + + return Collections.unmodifiableList(allIndices); + } + + /** Adds an explicitly defined summary to this search definition */ + public void addSummary(DocumentSummary summary) { + summaries.put(summary.getName(), summary); + } + + /** + * Returns a summary class defined by this search definition, or null if no summary with this name is defined. + * The default summary, named "default" is always present. + */ + public DocumentSummary getSummary(String name) { + var summary = summaries.get(name); + if (summary != null) return summary; + if (inherited.isEmpty()) return null; + return requireInherited().getSummary(name); + } + + /** + * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly + * or explicitly) in any summary class. + */ + public SummaryField getSummaryField(String name) { + for (DocumentSummary summary : summaries.values()) { + SummaryField summaryField = summary.getSummaryField(name); + if (summaryField != null) { + return summaryField; + } + } + if (inherited.isEmpty()) return null; + return requireInherited().getSummaryField(name); + } + + /** + * Returns the first explicit instance found of a summary field with this name, or null if not present explicitly in + * any summary class + * + * @param name the name of the explicit summary field to get. + * @return the SummaryField found. + */ + public SummaryField getExplicitSummaryField(String name) { + for (DocumentSummary summary : summaries.values()) { + SummaryField summaryField = summary.getSummaryField(name); + if (summaryField != null && !summaryField.isImplicit()) + return summaryField; + } + if (inherited.isEmpty()) return null; + return requireInherited().getExplicitSummaryField(name); + } + + /** + * Summaries defined by fields of this search definition. The default summary, named "default", is always the first + * one in the returned iterator. + */ + public Map getSummaries() { + // Shortcuts + if (inherited.isEmpty()) return summaries; + if (summaries.isEmpty()) return requireInherited().getSummaries(); + + var allSummaries = new LinkedHashMap<>(requireInherited().getSummaries()); + allSummaries.putAll(summaries); + return allSummaries; + } + + /** Returns the summaries defines in this only, not any that are inherited. */ + public Map getSummariesInThis() { return Collections.unmodifiableMap(summaries); } + + /** + * Returns all summary fields, of all document summaries, which has the given field as source. + * The list becomes owned by the receiver. + * + * @param field the source field + * @return the list of summary fields found + */ + @Override + public List getSummaryFields(ImmutableSDField field) { + List summaryFields = inherited.isPresent() + ? requireInherited().getSummaryFields(field) + : new java.util.ArrayList<>(); + for (DocumentSummary documentSummary : summaries.values()) { + for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { + if (summaryField.hasSource(field.getName())) { + boolean wanted = true; + for (var already : summaryFields) { + if (summaryField == already) wanted = false; + } + if (wanted) { + summaryFields.add(summaryField); + } + } + } + } + return summaryFields; + } + + /** + * Returns one summary field for each summary field name. If there are multiple summary fields with the same + * name, the last one will be used. Multiple fields of the same name should all have the same content in a valid + * search definition, except from the destination set. So this method can be used for all summary handling except + * processing the destination set. The map becomes owned by the receiver. + */ + public Map getUniqueNamedSummaryFields() { + Map summaryFields = inherited.isPresent() ? requireInherited().getUniqueNamedSummaryFields() + : new java.util.LinkedHashMap<>(); + for (DocumentSummary documentSummary : summaries.values()) { + for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { + summaryFields.put(summaryField.getName(), summaryField); + } + } + return summaryFields; + } + + /** Returns the first occurrence of an attribute having this name, or null if none */ + public Attribute getAttribute(String name) { + for (ImmutableSDField field : allConcreteFields()) { + Attribute attribute = field.getAttributes().get(name); + if (attribute != null) { + return attribute; + } + } + return null; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Schema)) { + return false; + } + + Schema other = (Schema)o; + return getName().equals(other.getName()); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return "schema '" + getName() + "'"; + } + + public boolean isAccessingDiskSummary(SummaryField field) { + if (!field.getTransform().isInMemory()) return true; + if (field.getSources().size() == 0) return isAccessingDiskSummary(getName()); + for (SummaryField.Source source : field.getSources()) { + if (isAccessingDiskSummary(source.getName())) + return true; + } + return false; + } + + private boolean isAccessingDiskSummary(String source) { + SDField field = getConcreteField(source); + if (field == null) return false; + if (field.doesSummarying() && !field.doesAttributing()) return true; + return false; + } + + public FieldSets fieldSets() { return fieldSets; } + + /** Returns the schema inherited by this, or throws if none */ + private Schema requireInherited() { return owner.schemas().get(inherited.get()); } + + /** + * For adding structs defined in document scope + * + * @param dt the struct to add + * @return self, for chaining + */ + public Schema addType(SDDocumentType dt) { + documentType.addType(dt); // TODO This is a very very dirty thing. It must go + return this; + } + + public Schema addAnnotation(SDAnnotationType dt) { + documentType.addAnnotation(dt); + return this; + } + + public void validate(DeployLogger logger) { + if (inherited.isPresent()) { + if (! owner.schemas().containsKey(inherited.get())) + throw new IllegalArgumentException(this + " inherits '" + inherited.get() + + "', but this schema does not exist"); + + // Require schema and document type inheritance to be consistent to keep things simple + // And require it to be explicit so we have the option to support other possibilities later + var parentDocument = owner.schemas().get(inherited.get()).getDocument(); + if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) + throw new IllegalArgumentException(this + " inherits '" + inherited.get() + + "', but its document type does not inherit the parent's document type"); + } + for (var summary : summaries.values()) + summary.validate(logger); + } + + /** Returns true if the given field name is a reserved name */ + public static boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java new file mode 100644 index 00000000000..b1ce6f5eb4f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.TemporarySDDocumentType; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Level; + +/** + * @author arnej + */ +public class TemporarySDTypeResolver { + + private final DeployLogger deployLogger; + private final Collection toProcess; + private final List docTypes = new LinkedList<>(); + + public TemporarySDTypeResolver(Collection schemas, DeployLogger deployLogger) { + this.deployLogger = deployLogger; + this.toProcess = schemas; + } + + private SDDocumentType findDocType(String name) { + assert(name != null); + for (var doc : docTypes) { + if (doc.getName().equals(name)) { + return doc; + } + } + deployLogger.logApplicationPackage(Level.WARNING, "No document type in application matching name: "+name); + return null; + } + + public void process() { + docTypes.add(SDDocumentType.VESPA_DOCUMENT); + for (Schema schema : toProcess) { + if (schema.hasDocument()) { + docTypes.add(schema.getDocument()); + } + } + // first, fix inheritance + for (SDDocumentType doc : docTypes) { + for (SDDocumentType inherited : doc.getInheritedTypes()) { + if (inherited instanceof TemporarySDDocumentType) { + var actual = findDocType(inherited.getName()); + if (actual != null) { + doc.inherit(actual); + } else { + deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for document "+doc.getName()); + } + } + } + } + // next, check owned types (structs only?) + for (SDDocumentType doc : docTypes) { + for (SDDocumentType owned : doc.getTypes()) { + if (owned instanceof TemporarySDDocumentType) { + deployLogger.logApplicationPackage(Level.WARNING, "Schema '"+doc.getName()+"' owned type '"+owned.getName()+"' is temporary, should not happen"); + continue; + } + for (SDDocumentType inherited : owned.getInheritedTypes()) { + if (inherited instanceof TemporarySDDocumentType) { + var actual = doc.getType(inherited.getName()); + if (actual != null) { + owned.inherit(actual); + } else { + deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for type '"+owned.getName()+"' in document "+doc.getName()); + } + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/UnrankedRankProfile.java new file mode 100644 index 00000000000..6c1f5fc8731 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/UnrankedRankProfile.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; + +/** + * A low-cost ranking profile to use for watcher queries etc. + * + * @author Vegard Havdal + */ +public class UnrankedRankProfile extends RankProfile { + + public UnrankedRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { + super("unranked", schema, rankProfileRegistry); + try { + RankingExpression exp = new RankingExpression("value(0)"); + this.setFirstPhaseRanking(exp); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse the ranking expression 'value(0)' when setting up " + + "the 'unranked' rank profile"); + } + this.setIgnoreDefaultRankFeatures(true); + this.setKeepRankCount(0); + this.setRerankCount(0); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java new file mode 100644 index 00000000000..99f73a75669 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -0,0 +1,324 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Ranking; +import com.yahoo.schema.document.Sorting; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedComplexField; + +/** + * The set of all attribute fields defined by a search definition + * + * @author bratseth + */ +public class AttributeFields extends Derived implements AttributesConfig.Producer { + + public enum FieldSet {ALL, FAST_ACCESS} + + private Map attributes = new java.util.LinkedHashMap<>(); + private Map importedAttributes = new java.util.LinkedHashMap<>(); + + /** Whether this has any position attribute */ + private boolean hasPosition = false; + + public static final AttributeFields empty = new AttributeFields(null); + + public AttributeFields(Schema schema) { + if (schema != null) + derive(schema); + } + + /** Derives everything from a field */ + @Override + protected void derive(ImmutableSDField field, Schema schema) { + if (unsupportedFieldType(field)) { + return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) + } + if (isArrayOfSimpleStruct(field)) { + deriveArrayOfSimpleStruct(field); + } else if (isMapOfSimpleStruct(field)) { + deriveMapOfSimpleStruct(field); + } else if (isMapOfPrimitiveType(field)) { + deriveMapOfPrimitiveType(field); + } else { + deriveAttributes(field); + } + } + + private static boolean unsupportedFieldType(ImmutableSDField field) { + return (field.usesStructOrMap() && + !isSupportedComplexField(field) && + !GeoPos.isAnyPos(field)); + } + + /** Returns an attribute by name, or null if it doesn't exist */ + public Attribute getAttribute(String attributeName) { + return attributes.get(attributeName); + } + + public boolean containsAttribute(String attributeName) { + return getAttribute(attributeName) != null; + } + + /** Derives one attribute. TODO: Support non-default named attributes */ + private void deriveAttributes(ImmutableSDField field) { + if (field.isImportedField()) { + deriveImportedAttributes(field); + return; + } + for (Attribute fieldAttribute : field.getAttributes().values()) { + deriveAttribute(field, fieldAttribute); + } + + if (field.containsExpression(ToPositionExpression.class)) { + // TODO: Move this check to processing and remove this + if (hasPosition) { + throw new IllegalArgumentException("Can not specify more than one set of position attributes per field: " + field.getName()); + } + hasPosition = true; + } + } + + private void applyRanking(ImmutableSDField field, Attribute attribute) { + Ranking ranking = field.getRanking(); + if (ranking != null && ranking.isFilter()) { + attribute.setEnableBitVectors(true); + attribute.setEnableOnlyBitVector(true); + } + } + + private void deriveAttribute(ImmutableSDField field, Attribute fieldAttribute) { + Attribute attribute = getAttribute(fieldAttribute.getName()); + if (attribute == null) { + attributes.put(fieldAttribute.getName(), fieldAttribute); + attribute = getAttribute(fieldAttribute.getName()); + } + applyRanking(field, attribute); + } + + private void deriveImportedAttributes(ImmutableSDField field) { + for (Attribute attribute : field.getAttributes().values()) { + if (!importedAttributes.containsKey(field.getName())) { + importedAttributes.put(field.getName(), attribute); + } + } + } + + private void deriveArrayOfSimpleStruct(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + deriveAttributeAsArrayType(structField); + } + } + + private void deriveAttributeAsArrayType(ImmutableSDField field) { + if (field.isImportedField()) { + deriveImportedAttributes(field); + return; + } + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null) { + applyRanking(field, attribute); + attributes.put(attribute.getName(), attribute.convertToArray()); + } + } + + private void deriveMapOfSimpleStruct(ImmutableSDField field) { + deriveAttributeAsArrayType(field.getStructField("key")); + deriveMapValueField(field.getStructField("value")); + } + + private void deriveMapValueField(ImmutableSDField valueField) { + for (ImmutableSDField structField : valueField.getStructFields()) { + deriveAttributeAsArrayType(structField); + } + } + + private void deriveMapOfPrimitiveType(ImmutableSDField field) { + deriveAttributeAsArrayType(field.getStructField("key")); + deriveAttributeAsArrayType(field.getStructField("value")); + } + + /** Returns a read only attribute iterator */ + public Iterator attributeIterator() { + return attributes().iterator(); + } + + public Collection attributes() { + return Collections.unmodifiableCollection(attributes.values()); + } + + public Collection structFieldAttributes(String baseFieldName) { + String structPrefix = baseFieldName + "."; + return attributes().stream() + .filter(attribute -> attribute.getName().startsWith(structPrefix)) + .collect(Collectors.toList()); + } + + public String toString() { + return "attributes " + getName(); + } + + @Override + protected String getDerivedName() { + return "attributes"; + } + + @SuppressWarnings("removal") // TODO Vespa 8: remove + private Map toMap(List ls) { + Map ret = new LinkedHashMap<>(); + for (AttributesConfig.Attribute.Builder builder : ls) { + ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder); + } + return ret; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + //TODO This is just to get some exporting tests to work, Should be undone and removed + getConfig(builder, FieldSet.ALL, 77777, false); + } + + private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { + return (fs == FieldSet.ALL) || ((fs == FieldSet.FAST_ACCESS) && attribute.isFastAccess()); + } + + private AttributesConfig.Attribute.Builder getConfig(String attrName, Attribute attribute, boolean imported) { + AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder() + .name(attrName) + .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName())) + .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName())); + if (attribute.isRemoveIfZero()) { + aaB.removeifzero(true); + } + if (attribute.isCreateIfNonExistent()) { + aaB.createifnonexistent(true); + } + aaB.enablebitvectors(attribute.isEnabledBitVectors()); + aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector()); + if (attribute.isFastSearch() || attribute.isFastRank()) { + // TODO make a separate fastrank flag in config instead of overloading fastsearch + aaB.fastsearch(true); + } + if (attribute.isFastAccess()) { + aaB.fastaccess(true); + } + if (attribute.isMutable()) { + aaB.ismutable(true); + } + if (attribute.isHuge()) { + aaB.huge(true); + } + if (attribute.isPaged()) { + aaB.paged(true); + } + if (attribute.getSorting().isDescending()) { + aaB.sortascending(false); + } + if (attribute.getSorting().getFunction() != Sorting.Function.UCA) { + aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString())); + } + if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) { + aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString())); + } + if (!attribute.getSorting().getLocale().isEmpty()) { + aaB.sortlocale(attribute.getSorting().getLocale()); + } + aaB.arity(attribute.arity()); + aaB.lowerbound(attribute.lowerBound()); + aaB.upperbound(attribute.upperBound()); + aaB.densepostinglistthreshold(attribute.densePostingListThreshold()); + if (attribute.tensorType().isPresent()) { + aaB.tensortype(attribute.tensorType().get().toString()); + } + aaB.imported(imported); + var dma = attribute.distanceMetric(); + aaB.distancemetric(AttributesConfig.Attribute.Distancemetric.Enum.valueOf(dma.toString())); + if (attribute.hnswIndexParams().isPresent()) { + var ib = new AttributesConfig.Attribute.Index.Builder(); + var params = attribute.hnswIndexParams().get(); + ib.hnsw.enabled(true); + ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); + ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); + ib.hnsw.multithreadedindexing(params.multiThreadedIndexing()); + aaB.index(ib); + } + Dictionary dictionary = attribute.getDictionary(); + if (dictionary != null) { + aaB.dictionary.type(convert(dictionary.getType())); + aaB.dictionary.match(convert(dictionary.getMatch())); + } + aaB.match(convertMatch(attribute.getCase())); + return aaB; + } + + private static AttributesConfig.Attribute.Dictionary.Type.Enum convert(Dictionary.Type type) { + switch (type) { + case BTREE: + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + case HASH: + return AttributesConfig.Attribute.Dictionary.Type.HASH; + case BTREE_AND_HASH: + return AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; + } + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + } + private static AttributesConfig.Attribute.Dictionary.Match.Enum convert(Case type) { + switch (type) { + case CASED: + return AttributesConfig.Attribute.Dictionary.Match.CASED; + case UNCASED: + return AttributesConfig.Attribute.Dictionary.Match.UNCASED; + } + return AttributesConfig.Attribute.Dictionary.Match.UNCASED; + } + private static AttributesConfig.Attribute.Match.Enum convertMatch(Case type) { + switch (type) { + case CASED: + return AttributesConfig.Attribute.Match.CASED; + case UNCASED: + return AttributesConfig.Attribute.Match.UNCASED; + } + return AttributesConfig.Attribute.Match.UNCASED; + } + + public void getConfig(AttributesConfig.Builder builder, FieldSet fs, long maxUnCommittedMemory, boolean enableBitVectors) { + for (Attribute attribute : attributes.values()) { + if (isAttributeInFieldSet(attribute, fs)) { + AttributesConfig.Attribute.Builder attrBuilder = getConfig(attribute.getName(), attribute, false); + attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); + if (enableBitVectors && attribute.isFastSearch()) { + attrBuilder.enablebitvectors(true); + } + builder.attribute(attrBuilder); + } + } + if (fs == FieldSet.ALL) { + for (Map.Entry entry : importedAttributes.entrySet()) { + AttributesConfig.Attribute.Builder attrBuilder = getConfig(entry.getKey(), entry.getValue(), true); + attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); + builder.attribute(attrBuilder); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Derived.java b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java new file mode 100644 index 00000000000..9943a02a2f2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java @@ -0,0 +1,141 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigInstance.Builder; +import com.yahoo.document.Field; +import com.yahoo.io.IOUtils; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.text.StringUtilities; + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Method; +import java.util.List; + +/** + * Superclass of all derived configurations + * + * @author bratseth + */ +public abstract class Derived implements Exportable { + + private String name; + + public Derived() { + this(""); + } + + public Derived(String name) { + this.name = name; + } + + public String getName() { return name; } + + protected final void setName(String name) { this.name = name; } + + /** + * Derives the content of this configuration. This + * default calls derive(Document) for each document + * and derive(SDField) for each search definition level field + * AND sets the name of this to the name of the input search definition + */ + protected void derive(Schema schema) { + setName(schema.getName()); + derive(schema.getDocument(), schema); + for (Index index : schema.getExplicitIndices()) + derive(index, schema); + for (SDField field : schema.allExtraFields()) + derive(field, schema); + schema.allImportedFields().forEach(importedField -> derive(importedField, schema)); + } + + + /** + * Derives the content of this configuration. This + * default calls derive(SDField) for each document field + */ + protected void derive(SDDocumentType document, Schema schema) { + for (Field field : document.fieldSet()) { + SDField sdField = (SDField) field; + if ( ! sdField.isExtraField()) { + derive(sdField, schema); + } + } + } + + /** + * Derives the content of this configuration. This + * default does nothing. + */ + protected void derive(ImmutableSDField field, Schema schema) {} + + /** + * Derives the content of this configuration. This + * default does nothing. + */ + protected void derive(Index index, Schema schema) { + } + + protected abstract String getDerivedName(); + + /** Returns the value of getName if true, the given number as a string otherwise */ + protected String getIndex(int number, boolean labels) { + return labels ? getName() : String.valueOf(number); + } + + /** + * Exports this derived configuration to its .cfg file + * in toDirectory + * + * @param toDirectory the directory to export to, or null + * + */ + public final void export(String toDirectory) throws IOException { + Writer writer = null; + try { + String fileName = getDerivedName() + ".cfg"; + if (toDirectory != null) + writer = IOUtils.createWriter(toDirectory + "/" + fileName,false); + try { + exportBuilderConfig(writer); + } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) { + throw new RuntimeException(e); + } + } + finally { + if (writer != null) IOUtils.closeWriter(writer); + } + } + + /** + * Checks what this is a producer of, instantiate that and export to writer + */ + // TODO move to ReflectionUtil, and move that to unexported pkg + private void exportBuilderConfig(Writer writer) throws ReflectiveOperationException, SecurityException, IllegalArgumentException, IOException { + for (Class intf : getClass().getInterfaces()) { + if (ConfigInstance.Producer.class.isAssignableFrom(intf)) { + Class configClass = intf.getEnclosingClass(); + String builderClassName = configClass.getCanonicalName()+"$Builder"; + Class builderClass = Class.forName(builderClassName); + ConfigInstance.Builder builder = (Builder) builderClass.getDeclaredConstructor().newInstance(); + Method getConfig = getClass().getMethod("getConfig", builderClass); + getConfig.invoke(this, builder); + ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder); + List payloadL = ConfigInstance.serialize(inst); + String payload = StringUtilities.implodeMultiline(payloadL); + writer.write(payload); + } + } + } + + @Override + public String getFileName() { + return getDerivedName() + ".cfg"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java new file mode 100644 index 00000000000..0f5721bbab3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -0,0 +1,212 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.document.config.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.protect.Validator; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.validation.Validation; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.io.IOException; +import java.io.Writer; + +/** + * A set of all derived configuration of a schema. Use this as a facade to individual configurations when + * necessary. + * + * @author bratseth + */ +public class DerivedConfiguration implements AttributesConfig.Producer { + + private final Schema schema; + private Summaries summaries; + private SummaryMap summaryMap; + private Juniperrc juniperrc; + private AttributeFields attributeFields; + private RankProfileList rankProfileList; + private IndexingScript indexingScript; + private IndexInfo indexInfo; + private SchemaInfo schemaInfo; + private VsmFields streamingFields; + private VsmSummary streamingSummary; + private IndexSchema indexSchema; + private ImportedFields importedFields; + private final QueryProfileRegistry queryProfiles; + private final long maxUncommittedMemory; + private final boolean enableBitVectors; + + /** + * Creates a complete derived configuration from a search definition. + * Only used in tests. + * + * @param schema the search to derive a configuration from. Derived objects will be snapshots, but this argument is + * live. Which means that this object will be inconsistent when the given search definition is later + * modified. + * @param rankProfileRegistry a {@link com.yahoo.schema.RankProfileRegistry} + */ + public DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry) { + this(schema, rankProfileRegistry, new QueryProfileRegistry()); + } + + DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); + } + + /** + * Creates a complete derived configuration snapshot from a schema. + * + * @param schema the schema to derive a configuration from. Derived objects will be snapshots, but this + * argument is live. Which means that this object will be inconsistent if the given + * schema is later modified. + */ + public DerivedConfiguration(Schema schema, DeployState deployState) { + Validator.ensureNotNull("Schema", schema); + this.schema = schema; + this.queryProfiles = deployState.getQueryProfiles().getRegistry(); + this.maxUncommittedMemory = deployState.getProperties().featureFlags().maxUnCommittedMemory(); + this.enableBitVectors = deployState.getProperties().featureFlags().enableBitVectors(); + if ( ! schema.isDocumentsOnly()) { + streamingFields = new VsmFields(schema); + streamingSummary = new VsmSummary(schema); + } + if ( ! schema.isDocumentsOnly()) { + attributeFields = new AttributeFields(schema); + summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); + summaryMap = new SummaryMap(schema); + juniperrc = new Juniperrc(schema); + rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); + indexingScript = new IndexingScript(schema); + indexInfo = new IndexInfo(schema); + schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries, summaryMap); + indexSchema = new IndexSchema(schema); + importedFields = new ImportedFields(schema); + } + Validation.validate(this, schema); + } + + /** + * Exports a complete set of configuration-server format config files. + * + * @param toDirectory the directory to export to, current dir if null + * @throws IOException if exporting fails, some files may still be created + */ + public void export(String toDirectory) throws IOException { + if (!schema.isDocumentsOnly()) { + summaries.export(toDirectory); + summaryMap.export(toDirectory); + juniperrc.export(toDirectory); + attributeFields.export(toDirectory); + streamingFields.export(toDirectory); + streamingSummary.export(toDirectory); + indexSchema.export(toDirectory); + rankProfileList.export(toDirectory); + indexingScript.export(toDirectory); + indexInfo.export(toDirectory); + importedFields.export(toDirectory); + schemaInfo.export(toDirectory); + } + } + + public static void exportDocuments(DocumentmanagerConfig.Builder documentManagerCfg, String toDirectory) throws IOException { + exportCfg(new DocumentmanagerConfig(documentManagerCfg), toDirectory + "/" + "documentmanager.cfg"); + } + + public static void exportDocuments(DocumenttypesConfig.Builder documentTypesCfg, String toDirectory) throws IOException { + exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg"); + } + + public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException { + exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg"); + } + + public void exportConstants(String toDirectory) throws IOException { + RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); + rankProfileList.getConfig(b); + exportCfg(b.build(), toDirectory + "/" + "ranking-constants.cfg"); + } + + private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { + Writer writer = null; + try { + writer = IOUtils.createWriter(fileName, false); + writer.write(instance.toString()); + writer.write("\n"); + } finally { + if (writer != null) { + IOUtils.closeWriter(writer); + } + } + } + + public Summaries getSummaries() { + return summaries; + } + + public AttributeFields getAttributeFields() { + return attributeFields; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + getConfig(builder, AttributeFields.FieldSet.ALL); + } + + public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { + attributeFields.getConfig(builder, fs, maxUncommittedMemory, enableBitVectors); + } + + public IndexingScript getIndexingScript() { + return indexingScript; + } + + public IndexInfo getIndexInfo() { + return indexInfo; + } + + public SchemaInfo getSchemaInfo() { return schemaInfo; } + + public void setIndexingScript(IndexingScript script) { + this.indexingScript = script; + } + + public Schema getSchema() { return schema; } + + public RankProfileList getRankProfileList() { + return rankProfileList; + } + + public VsmSummary getVsmSummary() { + return streamingSummary; + } + + public VsmFields getVsmFields() { + return streamingFields; + } + + public IndexSchema getIndexSchema() { + return indexSchema; + } + + public Juniperrc getJuniperrc() { + return juniperrc; + } + + public SummaryMap getSummaryMap() { + return summaryMap; + } + + public ImportedFields getImportedFields() { + return importedFields; + } + + public QueryProfileRegistry getQueryProfiles() { return queryProfiles; } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java new file mode 100644 index 00000000000..44bea43a8e3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; +import com.yahoo.document.config.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * Facade for deriving configs from schemas + * + * @author bratseth + */ +public class Deriver { + + public static ApplicationBuilder getSchemaBuilder(List schemas) { + ApplicationBuilder builder = new ApplicationBuilder(); + try { + for (String schema : schemas) + builder.addSchemaFile(schema); + } catch (ParseException | IOException e) { + throw new IllegalArgumentException(e); + } + builder.build(true); + return builder; + } + + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) { + return getDocumentManagerConfig(Collections.singletonList(sd)); + } + + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List schemas) { + return new DocumentManager().produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); + } + + public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { + return getDocumentTypesConfig(Collections.singletonList(schema)); + } + + public static DocumenttypesConfig.Builder getDocumentTypesConfig(List schemas) { + return new DocumentTypes().produce(getSchemaBuilder(schemas).getModel(), new DocumenttypesConfig.Builder()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java b/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java new file mode 100644 index 00000000000..4fccfb5d9f8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +/** + * Classes exportable to configurations + * + * @author bratseth + */ +public interface Exportable { + + /** + * Exports the configuration of this object + * + * + * @param toDirectory the directory to export to, does not write to disk if null + * @throws java.io.IOException if exporting fails, some files may still be created + */ + public void export(String toDirectory) throws java.io.IOException; + + /** + * The (short) name of the exported file + * @return a String with the (short) name of the exported file + */ + public String getFileName(); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/schema/derived/FieldRankSettings.java new file mode 100644 index 00000000000..ccb25df031c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/FieldRankSettings.java @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.collections.Pair; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The rank settings of a field used for native rank features. + * + * @author geirst + */ +public class FieldRankSettings { + + private final String fieldName; + + private final Map tables = new LinkedHashMap<>(); + + public FieldRankSettings(String fieldName) { + this.fieldName = fieldName; + } + + public void addTable(NativeTable table) { + NativeTable existing = tables.get(table.getType().getName()); + if (existing != null) { + // TODO: Throw? + return; + } + tables.put(table.getType().getName(), table); + } + + public static boolean isIndexFieldTable(NativeTable table) { + return isFieldMatchTable(table) || isProximityTable(table); + } + + public static boolean isAttributeFieldTable(NativeTable table) { + return isAttributeMatchTable(table); + } + + private static boolean isFieldMatchTable(NativeTable table) { + return (table.getType().equals(NativeTable.Type.FIRST_OCCURRENCE) || + table.getType().equals(NativeTable.Type.OCCURRENCE_COUNT)); + } + + private static boolean isAttributeMatchTable(NativeTable table) { + return (table.getType().equals(NativeTable.Type.WEIGHT)); + } + + private static boolean isProximityTable(NativeTable table) { + return (table.getType().equals(NativeTable.Type.PROXIMITY) || + table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY)); + } + + public List> deriveRankProperties() { + List> properties = new ArrayList<>(); + for (NativeTable table : tables.values()) { + if (isFieldMatchTable(table)) + properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isAttributeMatchTable(table)) + properties.add(new Pair<>("nativeAttributeMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isProximityTable(table)) + properties.add(new Pair<>("nativeProximity." + table.getType().getName() + "." + fieldName, table.getName())); + } + return properties; + } + + @Override + public String toString() { + return "rank settings of field " + fieldName; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/schema/derived/FieldResultTransform.java new file mode 100644 index 00000000000..99b2925d714 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/FieldResultTransform.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.vespa.documentmodel.SummaryTransform; + +/** + * The result transformation of a named field + * + * @author bratseth + */ +public class FieldResultTransform { + + private final String fieldName; + + private SummaryTransform transform; + + private final String argument; + + public FieldResultTransform(String fieldName, SummaryTransform transform, String argument) { + this.fieldName = fieldName; + this.transform = transform; + this.argument = argument; + } + + public String getFieldName() { return fieldName; } + + public SummaryTransform getTransform() { return transform; } + + public void setTransform(SummaryTransform transform) { this.transform = transform; } + + /** Returns the argument of this (used as input to the backend docsum rewriter) */ + public String getArgument() { return argument; } + + public int hashCode() { + return fieldName.hashCode() + 11 * transform.hashCode() + 17 * argument.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (! (o instanceof FieldResultTransform)) return false; + FieldResultTransform other = (FieldResultTransform)o; + + return + this.fieldName.equals(other.fieldName) && + this.transform.equals(other.transform) && + this.argument.equals(other.argument); + } + + @Override + public String toString() { + String sourceString = ""; + if ( ! argument.equals(fieldName)) + sourceString = " (argument: " + argument + ")"; + return "field " + fieldName + ": " + transform + sourceString; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java new file mode 100644 index 00000000000..05f6be2f6f1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.schema.DistributableResource; +import com.yahoo.schema.RankProfile; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Constant values for ranking/model execution tied to a rank profile, + * to be distributed as files. + * + * @author bratseth + */ +public class FileDistributedConstants { + + private final Map constants; + + public FileDistributedConstants(FileRegistry fileRegistry, Collection constants) { + Map distributableConstants = new LinkedHashMap<>(); + for (var constant : constants) { + if ( ! constant.valuePath().isPresent()) continue; + + var distributableConstant = new DistributableConstant(constant.name().simpleArgument().get(), + constant.type(), + constant.valuePath().get(), + constant.pathType().get()); + distributableConstant.validate(); + distributableConstant.register(fileRegistry); + distributableConstants.put(distributableConstant.getName(), distributableConstant); + } + this.constants = Collections.unmodifiableMap(distributableConstants); + } + + /** Returns a read-only map of the constants in this indexed by name. */ + public Map asMap() { return constants; } + + public void getConfig(RankingConstantsConfig.Builder builder) { + for (var constant : constants.values()) { + builder.constant(new RankingConstantsConfig.Constant.Builder() + .name(constant.getName()) + .fileref(constant.getFileReference()) + .type(constant.getType())); + } + } + + public static class DistributableConstant extends DistributableResource { + + private final TensorType tensorType; + + public DistributableConstant(String name, TensorType type, String fileName) { + this(name, type, fileName, PathType.FILE); + } + + public DistributableConstant(String name, TensorType type, String fileName, PathType pathType) { + super(name, fileName, pathType); + this.tensorType = type; + validate(); + } + + public TensorType getTensorType() { return tensorType; } + public String getType() { return tensorType.toString(); } + + public void validate() { + super.validate(); + if (tensorType == null) + throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); + if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + + ": Dense tensor dimensions must have a size"); + } + + @Override + public String toString() { + return super.toString() + "' of type '" + tensorType + "'"; + } + + } + +} + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java new file mode 100644 index 00000000000..b5c3909c78c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.schema.OnnxModel; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * ONNX models distributed as files. + * + * @author bratseth + */ +public class FileDistributedOnnxModels { + + private static final Logger log = Logger.getLogger(FileDistributedOnnxModels.class.getName()); + + private final Map models; + + public FileDistributedOnnxModels(FileRegistry fileRegistry, Collection models) { + Map distributableModels = new LinkedHashMap<>(); + for (var model : models) { + model.validate(); + model.register(fileRegistry); + distributableModels.put(model.getName(), model); + } + this.models = Collections.unmodifiableMap(distributableModels); + } + + public Map asMap() { return models; } + + public void getConfig(OnnxModelsConfig.Builder builder) { + for (OnnxModel model : models.values()) { + if ("".equals(model.getFileReference())) + log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way + else { + OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); + modelBuilder.dry_run_on_setup(true); + modelBuilder.name(model.getName()); + modelBuilder.fileref(model.getFileReference()); + model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); + model.getOutputMap().forEach((name, as) -> modelBuilder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); + if (model.getStatelessExecutionMode().isPresent()) + modelBuilder.stateless_execution_mode(model.getStatelessExecutionMode().get()); + if (model.getStatelessInterOpThreads().isPresent()) + modelBuilder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); + if (model.getStatelessIntraOpThreads().isPresent()) + modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); + + builder.model(modelBuilder); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java new file mode 100644 index 00000000000..fa3f49f06d5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java @@ -0,0 +1,105 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.vespa.config.search.ImportedFieldsConfig; + +import java.util.Optional; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; + +/** + * This class derives imported fields from search definition and produces imported-fields.cfg as needed by the search backend. + * + * @author geirst + */ +public class ImportedFields extends Derived implements ImportedFieldsConfig.Producer { + + private Optional importedFields = Optional.empty(); + + public ImportedFields(Schema schema) { + derive(schema); + } + + @Override + protected void derive(Schema schema) { + importedFields = schema.importedFields(); + } + + @Override + protected String getDerivedName() { + return "imported-fields"; + } + + @Override + public void getConfig(ImportedFieldsConfig.Builder builder) { + if (importedFields.isPresent()) { + importedFields.get().fields().forEach( (name, field) -> considerField(builder, field)); + } + } + + private static boolean isNestedFieldName(String fieldName) { + return fieldName.indexOf('.') != -1; + } + + private static void considerField(ImportedFieldsConfig.Builder builder, ImportedField field) { + if (field instanceof ImportedComplexField) { + considerComplexField(builder, (ImportedComplexField) field); + } else { + considerSimpleField(builder, field); + } + } + + private static void considerComplexField(ImportedFieldsConfig.Builder builder, ImportedComplexField field) { + ImmutableSDField targetField = field.targetField(); + if (GeoPos.isAnyPos(targetField)) { + // no action needed + } else if (isArrayOfSimpleStruct(targetField)) { + considerNestedFields(builder, field); + } else if (isMapOfSimpleStruct(targetField)) { + considerSimpleField(builder, field.getNestedField("key")); + considerNestedFields(builder, field.getNestedField("value")); + } else if (isMapOfPrimitiveType(targetField)) { + considerSimpleField(builder, field.getNestedField("key")); + considerSimpleField(builder, field.getNestedField("value")); + } + } + + private static void considerNestedFields(ImportedFieldsConfig.Builder builder, ImportedField field) { + if (field instanceof ImportedComplexField) { + ImportedComplexField complexField = (ImportedComplexField) field; + complexField.getNestedFields().forEach(nestedField -> considerSimpleField(builder, nestedField)); + } + } + + private static void considerSimpleField(ImportedFieldsConfig.Builder builder, ImportedField field) { + ImmutableSDField targetField = field.targetField(); + String targetFieldName = targetField.getName(); + if (!isNestedFieldName(targetFieldName)) { + if (targetField.doesAttributing()) { + builder.attribute.add(createAttributeBuilder(field)); + } + } else { + Attribute attribute = targetField.getAttribute(); + if (attribute != null) { + builder.attribute.add(createAttributeBuilder(field)); + } + } + } + + private static ImportedFieldsConfig.Attribute.Builder createAttributeBuilder(ImportedField field) { + ImportedFieldsConfig.Attribute.Builder result = new ImportedFieldsConfig.Attribute.Builder(); + result.name(field.fieldName()); + result.referencefield(field.reference().referenceField().getName()); + result.targetfield(field.targetField().getName()); + return result; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Index.java b/config-model/src/main/java/com/yahoo/schema/derived/Index.java new file mode 100644 index 00000000000..3b5e617d3dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/Index.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.datatypes.*; + +/** + * A type of an index structure + * + * @author bratseth + */ +public class Index { + + /** The index type enumeration */ + public static class Type { + + public static final Type TEXT=new Type("text"); + public static final Type INT64=new Type("long"); + public static final Type BOOLEANTREE=new Type("booleantree"); + + private String name; + + private Type(String name) { + this.name=name; + } + + public int hashCode() { + return name.hashCode(); + } + + public String getName() { return name; } + + public boolean equals(Object other) { + if ( ! (other instanceof Type)) return false; + return this.name.equals(((Type)other).name); + } + + public String toString() { + return "type: " + name; + } + + } + + /** Sets the right index type from a field type */ + public static Type convertType(DataType fieldType) { + FieldValue fval = fieldType.createFieldValue(); + if (fieldType instanceof NumericDataType) { + return Type.INT64; + } else if (fval instanceof StringFieldValue) { + return Type.TEXT; + } else if (fval instanceof Raw) { + return Type.BOOLEANTREE; + } else if (fval instanceof PredicateFieldValue) { + return Type.BOOLEANTREE; + } else if (fieldType instanceof CollectionDataType) { + return convertType(((CollectionDataType) fieldType).getNestedType()); + } else { + throw new IllegalArgumentException("Don't know which index type to " + + "convert " + fieldType + " to"); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java new file mode 100644 index 00000000000..4887ad52974 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -0,0 +1,595 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.processing.ExactMatch; +import com.yahoo.schema.processing.NGramMatch; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.search.config.IndexInfoConfig; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Per-index commands which should be applied to queries prior to searching + * + * @author bratseth + */ +public class IndexInfo extends Derived implements IndexInfoConfig.Producer { + + private static final String CMD_ATTRIBUTE = "attribute"; + private static final String CMD_DEFAULT_POSITION = "default-position"; + private static final String CMD_DYNTEASER = "dynteaser"; + private static final String CMD_FULLURL = "fullurl"; + private static final String CMD_HIGHLIGHT = "highlight"; + private static final String CMD_INDEX = "index"; + private static final String CMD_LOWERCASE = "lowercase"; + private static final String CMD_NORMALIZE = "normalize"; + private static final String CMD_STEM = "stem"; + private static final String CMD_URLHOST = "urlhost"; + private static final String CMD_WORD = "word"; + private static final String CMD_PLAIN_TOKENS = "plain-tokens"; + private static final String CMD_MULTIVALUE = "multivalue"; + private static final String CMD_FAST_SEARCH = "fast-search"; + private static final String CMD_PREDICATE = "predicate"; + private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds"; + private static final String CMD_NUMERICAL = "numerical"; + private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; + private final Set commands = new java.util.LinkedHashSet<>(); + private final Map aliases = new java.util.LinkedHashMap<>(); + private final Map fieldSets; + private Schema schema; + + public IndexInfo(Schema schema) { + this.fieldSets = schema.fieldSets().userFieldSets(); + addIndexCommand("sddocname", CMD_INDEX); + addIndexCommand("sddocname", CMD_WORD); + derive(schema); + } + + @Override + protected void derive(Schema schema) { + super.derive(schema); // Derive per field + this.schema = schema; + // Populate fieldsets with actual field objects, bit late to do that here but + for (FieldSet fs : fieldSets.values()) { + for (String fieldName : fs.getFieldNames()) { + fs.fields().add(schema.getField(fieldName)); + } + } + // Must follow, because index settings overrides field settings + for (Index index : schema.getExplicitIndices()) { + derive(index, schema); + } + + // Commands for summary fields + // TODO: Move to fieldinfo and implement differently. This is not right + for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { + if (summaryField.getTransform().isTeaser()) { + addIndexCommand(summaryField.getName(), CMD_DYNTEASER); + } + if (summaryField.getTransform().isBolded()) { + addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); + } + } + } + + private static boolean isPositionField(ImmutableSDField field) { + return GeoPos.isAnyPos(field); + } + + @Override + protected void derive(ImmutableSDField field, Schema schema) { + derive(field, schema, false); + } + + protected void derive(ImmutableSDField field, Schema schema, boolean inPosition) { + if (field.getDataType().equals(DataType.PREDICATE)) { + addIndexCommand(field, CMD_PREDICATE); + Index index = field.getIndex(field.getName()); + if (index != null) { + BooleanIndexDefinition options = index.getBooleanIndexDefiniton(); + if (options.hasLowerBound() || options.hasUpperBound()) { + addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" + + (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." + + (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]"); + } + } + } + + // Field level aliases + for (Map.Entry e : field.getAliasToName().entrySet()) { + String alias = e.getKey(); + String name = e.getValue(); + addIndexAlias(alias, name); + } + boolean isPosition = isPositionField(field); + if (field.usesStructOrMap()) { + for (ImmutableSDField structField : field.getStructFields()) { + derive(structField, schema, isPosition); // Recursion + } + } + + if (isPosition) { + addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); + } + + addIndexCommand(field, CMD_INDEX); // List the indices + + if (needLowerCase(field)) { + addIndexCommand(field, CMD_LOWERCASE); + } + + if (field.getDataType().isMultivalue()) { + addIndexCommand(field, CMD_MULTIVALUE); + } + + Attribute attribute = field.getAttribute(); + if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { + addIndexCommand(field.getName(), CMD_ATTRIBUTE); + if (attribute != null && attribute.isFastSearch()) + addIndexCommand(field.getName(), CMD_FAST_SEARCH); + } else if (field.doesIndexing()) { + if (stemSomehow(field, schema)) { + addIndexCommand(field, stemCmd(field, schema), new StemmingOverrider(this, schema)); + } + if (normalizeAccents(field)) { + addIndexCommand(field, CMD_NORMALIZE); + } + if (field.getMatching() == null || field.getMatching().getType().equals(MatchType.TEXT)) { + addIndexCommand(field, CMD_PLAIN_TOKENS); + } + } + + if (isUriField(field)) { + addUriIndexCommands(field); + } + + if (field.getDataType().getPrimitiveType() instanceof NumericDataType) { + addIndexCommand(field, CMD_NUMERICAL); + } + + // Explicit commands + for (String command : field.getQueryCommands()) { + addIndexCommand(field, command); + } + + } + + private static boolean isAnyChildString(DataType dataType) { + PrimitiveDataType primitive = dataType.getPrimitiveType(); + if (primitive == PrimitiveDataType.STRING) return true; + if (primitive != null) return false; + if (dataType instanceof StructuredDataType) { + StructuredDataType structured = (StructuredDataType) dataType; + for (Field field : structured.getFields()) { + if (isAnyChildString(field.getDataType())) return true; + } + } else if (dataType instanceof MapDataType) { + MapDataType mapType = (MapDataType) dataType; + return isAnyChildString(mapType.getKeyType()) || isAnyChildString(mapType.getValueType()); + } + return false; + } + + private static boolean needLowerCase(ImmutableSDField field) { + return field.doesIndexing() + || field.doesLowerCasing() + || ((field.doesAttributing() || (field.getAttribute() != null)) + && isAnyChildString(field.getDataType()) + && field.getMatching().getCase().equals(Case.UNCASED)); + } + + static String stemCmd(ImmutableSDField field, Schema schema) { + return CMD_STEM + ":" + field.getStemming(schema).toStemMode(); + } + + private boolean stemSomehow(ImmutableSDField field, Schema schema) { + if (field.getStemming(schema).equals(Stemming.NONE)) return false; + return isTypeOrNested(field, DataType.STRING); + } + + private boolean normalizeAccents(ImmutableSDField field) { + return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); + } + + private boolean isTypeOrNested(ImmutableSDField field, DataType type) { + return field.getDataType().equals(type) || field.getDataType().equals(DataType.getArray(type)) || + field.getDataType().equals(DataType.getWeightedSet(type)); + } + + private boolean isUriField(ImmutableSDField field) { + DataType fieldType = field.getDataType(); + if (DataType.URI.equals(fieldType)) { + return true; + } + if (fieldType instanceof CollectionDataType && + DataType.URI.equals(((CollectionDataType)fieldType).getNestedType())) + { + return true; + } + return false; + } + + private void addUriIndexCommands(ImmutableSDField field) { + String fieldName = field.getName(); + addIndexCommand(fieldName, CMD_FULLURL); + addIndexCommand(fieldName, CMD_LOWERCASE); + addIndexCommand(fieldName + "." + fieldName, CMD_FULLURL); + addIndexCommand(fieldName + "." + fieldName, CMD_LOWERCASE); + addIndexCommand(fieldName + ".path", CMD_FULLURL); + addIndexCommand(fieldName + ".path", CMD_LOWERCASE); + addIndexCommand(fieldName + ".query", CMD_FULLURL); + addIndexCommand(fieldName + ".query", CMD_LOWERCASE); + addIndexCommand(fieldName + ".hostname", CMD_URLHOST); + addIndexCommand(fieldName + ".hostname", CMD_LOWERCASE); + + // XXX hack + Index index = field.getIndex("hostname"); + if (index != null) { + addIndexCommand(index, CMD_URLHOST); + } + } + + /** + * Sets a command for all indices of a field + */ + private void addIndexCommand(Index index, String command) { + addIndexCommand(index.getName(), command); + } + + /** + * Sets a command for all indices of a field + */ + private void addIndexCommand(ImmutableSDField field, String command) { + addIndexCommand(field, command, null); + } + + /** + * Sets a command for all indices of a field + */ + private void addIndexCommand(ImmutableSDField field, String command, IndexOverrider overrider) { + if (overrider == null || !overrider.override(field.getName(), command, field)) { + addIndexCommand(field.getName(), command); + } + } + + private void addIndexCommand(String indexName, String command) { + commands.add(new IndexCommand(indexName, command)); + } + + private void addIndexAlias(String alias, String indexName) { + aliases.put(alias, indexName); + } + + /** + * Returns whether a particular command is prsent in this index info + */ + public boolean hasCommand(String indexName, String command) { + return commands.contains(new IndexCommand(indexName, command)); + } + + private boolean notInCommands(String index) { + for (IndexCommand command : commands) { + if (command.getIndex().equals(index)) { + return false; + } + } + return true; + } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder(); + iiB.name(getName()); + for (IndexCommand command : commands) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(command.getIndex()) + .command(command.getCommand())); + } + // Make user defined field sets searchable + for (FieldSet fieldSet : fieldSets.values()) { + if (notInCommands(fieldSet.getName())) { + addFieldSetCommands(iiB, fieldSet); + } + } + + for (Map.Entry e : aliases.entrySet()) { + iiB.alias( + new IndexInfoConfig.Indexinfo.Alias.Builder() + .alias(e.getKey()) + .indexname(e.getValue())); + } + builder.indexinfo(iiB); + } + + // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. + private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { + for (String qc : fieldSet.queryCommands()) + iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder().indexname(fieldSet.getName()).command(qc)); + boolean anyIndexing = false; + boolean anyAttributing = false; + boolean anyLowerCasing = false; + boolean anyStemming = false; + boolean anyNormalizing = false; + String phraseSegmentingCommand = null; + String stemmingCommand = null; + Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching + // First a pass over the fields to read some params to decide field settings implicitly: + for (ImmutableSDField field : fieldSet.fields()) { + if (field.doesIndexing()) { + anyIndexing = true; + } + if (field.doesAttributing()) { + anyAttributing = true; + } + if (needLowerCase(field)) { + anyLowerCasing = true; + } + if (stemming(field)) { + anyStemming = true; + stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode(); + } + if (field.getNormalizing().doRemoveAccents()) { + anyNormalizing = true; + } + if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) { + fieldSetMatching = field.getMatching(); + } + Optional explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst(); + if (explicitPhraseSegmentingCommand.isPresent()) { + phraseSegmentingCommand = explicitPhraseSegmentingCommand.get(); + } + } + if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) { + // We have both attributes and indexes and no explicit match setting -> + // use default matching as that at least works if the data in the attribute consists + // of single tokens only. + fieldSetMatching = new Matching(); + } + if (anyLowerCasing) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_LOWERCASE)); + } + if (hasMultiValueField(fieldSet)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_MULTIVALUE)); + } + if (anyIndexing) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_INDEX)); + if ( ! isExactMatch(fieldSetMatching)) { + if (fieldSetMatching == null || fieldSetMatching.getType().equals(MatchType.TEXT)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_PLAIN_TOKENS)); + } + if (anyStemming) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(stemmingCommand)); + } + if (anyNormalizing) + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_NORMALIZE)); + if (phraseSegmentingCommand != null) + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(phraseSegmentingCommand)); + } + } else { + // Assume only attribute fields + iiB + .command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_ATTRIBUTE)) + .command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_INDEX)); + } + if (fieldSetMatching != null) { + // Explicit matching set on fieldset + if (fieldSetMatching.getType().equals(MatchType.EXACT)) { + String term = fieldSetMatching.getExactMatchTerminator(); + if (term==null) term=ExactMatch.DEFAULT_EXACT_TERMINATOR; + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command("exact "+term)); + } else if (fieldSetMatching.getType().equals(MatchType.WORD)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_WORD)); + } else if (fieldSetMatching.getType().equals(MatchType.GRAM)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); + } else if (fieldSetMatching.getType().equals(MatchType.TEXT)) { + + } + } + } + + private boolean hasMultiValueField(FieldSet fieldSet) { + for (ImmutableSDField field : fieldSet.fields()) { + if (field.getDataType().isMultivalue()) + return true; + } + return false; + } + + private Stemming getEffectiveStemming(ImmutableSDField field) { + Stemming active = field.getStemming(schema); + if (field.getIndex(field.getName()) != null) { + if (field.getIndex(field.getName()).getStemming()!=null) { + active = field.getIndex(field.getName()).getStemming(); + } + } + if (active != null) { + return active; + } + return Stemming.BEST; // assume default + } + + private boolean stemming(ImmutableSDField field) { + if (field.getStemming() != null) { + return !field.getStemming().equals(Stemming.NONE); + } + if (schema.getStemming() == Stemming.NONE) return false; + if (field.isImportedField()) return false; + if (field.getIndex(field.getName())==null) return true; + if (field.getIndex(field.getName()).getStemming()==null) return true; + return !(field.getIndex(field.getName()).getStemming().equals(Stemming.NONE)); + } + + private boolean isExactMatch(Matching m) { + if (m == null) return false; + if (m.getType().equals(MatchType.EXACT)) return true; + if (m.getType().equals(MatchType.WORD)) return true; + return false; + } + + @Override + protected String getDerivedName() { + return "index-info"; + } + + /** + * An index command. Null commands are also represented, to detect consistency issues. This is an (immutable) value + * object. + */ + public static class IndexCommand { + + private String index; + + private String command; + + public IndexCommand(String index, String command) { + this.index = index; + this.command = command; + } + + public String getIndex() { + return index; + } + + public String getCommand() { + return command; + } + + /** + * Returns true if this is the null command (do nothing) + */ + public boolean isNull() { + return command.equals(""); + } + + public int hashCode() { + return index.hashCode() + 17 * command.hashCode(); + } + + public boolean equals(Object object) { + if (!(object instanceof IndexCommand)) { + return false; + } + + IndexCommand other = (IndexCommand)object; + return + other.index.equals(this.index) && + other.command.equals(this.command); + } + + public String toString() { + return "index command " + command + " on index " + index; + } + + } + + /** + * A command which may override the command setting of a field for a particular index + */ + private static abstract class IndexOverrider { + + protected IndexInfo owner; + + public IndexOverrider(IndexInfo owner) { + this.owner = owner; + } + + /** + * Override the setting of this index for this field, returns true if overriden, false if this index should be + * set according to the field + */ + public abstract boolean override(String indexName, String command, ImmutableSDField field); + + } + + private static class StemmingOverrider extends IndexOverrider { + + private Schema schema; + + public StemmingOverrider(IndexInfo owner, Schema schema) { + super(owner); + this.schema = schema; + } + + public boolean override(String indexName, String command, ImmutableSDField field) { + if (schema == null) { + return false; + } + + Index index = schema.getIndex(indexName); + if (index == null) { + return false; + } + + Stemming indexStemming = index.getStemming(); + if (indexStemming == null) { + return false; + } + + if (Stemming.NONE.equals(indexStemming)) { + // Add nothing + } else { + owner.addIndexCommand(indexName, CMD_STEM + ":" + indexStemming.toStemMode()); + } + return true; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java new file mode 100644 index 00000000000..7f6c824b979 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -0,0 +1,245 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.config.search.IndexschemaConfig; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Deriver of indexschema config containing information of all text index fields with name and data type. + * + * @author geirst + */ +public class IndexSchema extends Derived implements IndexschemaConfig.Producer { + + private final List fields = new ArrayList<>(); + private final Map collections = new LinkedHashMap<>(); + private final Map fieldSets = new LinkedHashMap<>(); + + public IndexSchema(Schema schema) { + fieldSets.putAll(schema.fieldSets().userFieldSets()); + derive(schema); + } + + public boolean containsField(String fieldName) { + return fields.stream().anyMatch(field -> field.getName().equals(fieldName)); + } + + @Override + protected void derive(Schema schema) { + super.derive(schema); + } + + private boolean isTensorField(ImmutableSDField field) { + return field.getDataType() instanceof TensorDataType; + } + + private void deriveIndexFields(ImmutableSDField field, Schema schema) { + // Note: Indexes for tensor fields are NOT part of the index schema for text fields. + if ((!field.doesIndexing() && !field.isIndexStructureField()) || + isTensorField(field)) + { + return; + } + List lst = flattenField(field.asField()); + if (lst.isEmpty()) { + return; + } + String fieldName = field.getName(); + for (Field flatField : lst) { + deriveIndexFields(flatField, schema); + } + if (lst.size() > 1) { + FieldSet fieldSet = new FieldSet(fieldName); + for (Field flatField : lst) { + fieldSet.addFieldName(flatField.getName()); + } + fieldSets.put(fieldName, fieldSet); + } + } + + private void deriveIndexFields(Field field, Schema schema) { + IndexField toAdd = new IndexField(field.getName(), Index.convertType(field.getDataType()), field.getDataType()); + com.yahoo.schema.Index definedIndex = schema.getIndex(field.getName()); + if (definedIndex != null) { + toAdd.setIndexSettings(definedIndex); + } + fields.add(toAdd); + addFieldToCollection(field.getName(), field.getName()); // implicit + } + + private FieldCollection getCollection(String collectionName) { + FieldCollection retval = collections.get(collectionName); + if (retval == null) { + collections.put(collectionName, new FieldCollection(collectionName)); + return collections.get(collectionName); + } + return retval; + } + + private void addFieldToCollection(String fieldName, String collectionName) { + FieldCollection collection = getCollection(collectionName); + collection.fields.add(fieldName); + } + + @Override + protected void derive(ImmutableSDField field, Schema schema) { + if (field.usesStructOrMap()) { + return; // unsupported + } + deriveIndexFields(field, schema); + } + + @Override + protected String getDerivedName() { + return "indexschema"; + } + + @Override + public void getConfig(IndexschemaConfig.Builder icB) { + for (int i = 0; i < fields.size(); ++i) { + IndexField f = fields.get(i); + IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder() + .name(f.getName()) + .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType())) + .prefix(f.hasPrefix()) + .phrases(f.hasPhrases()) + .positions(f.hasPositions()) + .interleavedfeatures(f.useInterleavedFeatures()); + if (!f.getCollectionType().equals("SINGLE")) { + ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); + } + icB.indexfield(ifB); + } + for (FieldSet fieldSet : fieldSets.values()) { + IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder() + .name(fieldSet.getName()); + for (String f : fieldSet.getFieldNames()) { + fsB.field(new IndexschemaConfig.Fieldset.Field.Builder() + .name(f)); + } + icB.fieldset(fsB); + } + } + + @SuppressWarnings("deprecation") + static List flattenField(Field field) { + DataType fieldType = field.getDataType(); + if (fieldType.getPrimitiveType() != null){ + return Collections.singletonList(field); + } + if (fieldType instanceof ArrayDataType) { + List ret = new LinkedList<>(); + Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType()); + for (Field flatField : flattenField(innerField)) { + ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()))); + } + return ret; + } + if (fieldType instanceof StructuredDataType) { + List ret = new LinkedList<>(); + String fieldName = field.getName(); + for (Field childField : ((StructuredDataType)fieldType).getFields()) { + for (Field flatField : flattenField(childField)) { + ret.add(new Field(fieldName + "." + flatField.getName(), flatField)); + } + } + return ret; + } + throw new UnsupportedOperationException(fieldType.getName()); + } + + public List getFields() { + return fields; + } + + /** + * Representation of an index field with name and data type. + */ + public static class IndexField { + private String name; + private Index.Type type; + private com.yahoo.schema.Index.Type sdType; // The index type in "user intent land" + private DataType sdFieldType; + private boolean prefix = false; + private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility? + private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility? + private BooleanIndexDefinition boolIndex = null; + // Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. + private boolean interleavedFeatures = false; + + public IndexField(String name, Index.Type type, DataType sdFieldType) { + this.name = name; + this.type = type; + this.sdFieldType = sdFieldType; + } + public void setIndexSettings(com.yahoo.schema.Index index) { + if (type.equals(Index.Type.TEXT)) { + prefix = index.isPrefix(); + interleavedFeatures = index.useInterleavedFeatures(); + } + sdType = index.getType(); + boolIndex = index.getBooleanIndexDefiniton(); + } + public String getName() { return name; } + public Index.Type getRawType() { return type; } + public String getType() { + return type.equals(Index.Type.INT64) + ? "INT64" : "STRING"; + } + public String getCollectionType() { + return (sdFieldType == null) + ? "SINGLE" + : (sdFieldType instanceof WeightedSetDataType) + ? "WEIGHTEDSET" + : (sdFieldType instanceof ArrayDataType) + ? "ARRAY" + : "SINGLE"; + } + public boolean hasPrefix() { return prefix; } + public boolean hasPhrases() { return phrases; } + public boolean hasPositions() { return positions; } + public boolean useInterleavedFeatures() { return interleavedFeatures; } + + public BooleanIndexDefinition getBooleanIndexDefinition() { + return boolIndex; + } + + /** + * The user set index type + * @return the type + */ + public com.yahoo.schema.Index.Type getSdType() { + return sdType; + } + } + + /** + * Representation of a collection of fields (aka index, physical view). + */ + @SuppressWarnings({ "UnusedDeclaration" }) + private static class FieldCollection { + + private final String name; + private final List fields = new ArrayList<>(); + + FieldCollection(String name) { + this.name = name; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java new file mode 100644 index 00000000000..6dae89bf692 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -0,0 +1,197 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * An indexing language script derived from a search definition. An indexing script contains a set of indexing + * statements, organized in a composite structure of indexing code snippets. + * + * @author bratseth + */ +public final class IndexingScript extends Derived implements IlscriptsConfig.Producer { + + private final List docFields = new ArrayList<>(); + private final List expressions = new ArrayList<>(); + private List fieldsSettingLanguage; + + public IndexingScript(Schema schema) { + derive(schema); + } + + @Override + protected void derive(Schema schema) { + fieldsSettingLanguage = fieldsSettingLanguage(schema); + if (fieldsSettingLanguage.size() == 1) // Assume this language should be used for all fields + addExpression(fieldsSettingLanguage.get(0).getIndexingScript()); + super.derive(schema); + } + + @Override + protected void derive(ImmutableSDField field, Schema schema) { + if (field.isImportedField()) return; + + if (field.hasFullIndexingDocprocRights()) + docFields.add(field.getName()); + + if (field.usesStructOrMap() && ! GeoPos.isAnyPos(field)) { + return; // unsupported + } + + if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field)) + return; // Already added + + addExpression(field.getIndexingScript()); + } + + private void addExpression(ScriptExpression expression) { + if ( expression.isEmpty()) return; + expressions.add(new StatementExpression(new ClearStateExpression(), new GuardExpression(expression))); + } + + private List fieldsSettingLanguage(Schema schema) { + return schema.allFieldsList().stream() + .filter(field -> ! field.isImportedField()) + .filter(field -> field.containsExpression(SetLanguageExpression.class)) + .collect(Collectors.toList()); + } + + public Iterable expressions() { + return Collections.unmodifiableCollection(expressions); + } + + @Override + public String getDerivedName() { + return "ilscripts"; + } + + @Override + public void getConfig(IlscriptsConfig.Builder configBuilder) { + IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); + ilscriptBuilder.doctype(getName()); + ilscriptBuilder.docfield(docFields); + addContentInOrder(ilscriptBuilder); + configBuilder.ilscript(ilscriptBuilder); + } + + private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { + ArrayList later = new ArrayList<>(); + Set touchedFields = new HashSet<>(); + for (Expression expression : expressions) { + if (modifiesSelf(expression) && ! setsLanguage(expression)) + later.add(expression); + else + ilscriptBuilder.content(expression.toString()); + + FieldScanVisitor fieldFetcher = new FieldScanVisitor(); + fieldFetcher.visit(expression); + touchedFields.addAll(fieldFetcher.touchedFields()); + } + for (Expression exp : later) + ilscriptBuilder.content(exp.toString()); + generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); + } + + private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set touchedFields) { + Set fieldsWithSyntheticStatements = new HashSet<>(docFields); + fieldsWithSyntheticStatements.removeAll(touchedFields); + List orderedFields = new ArrayList<>(fieldsWithSyntheticStatements); + Collections.sort(orderedFields); + for (String fieldName : orderedFields) { + StatementExpression copyField = new StatementExpression(new InputExpression(fieldName), + new PassthroughExpression(fieldName)); + ilscriptBuilder.content(copyField.toString()); + } + } + + private boolean setsLanguage(Expression expression) { + SetsLanguageVisitor visitor = new SetsLanguageVisitor(); + visitor.visit(expression); + return visitor.setsLanguage; + } + + private boolean modifiesSelf(Expression expression) { + ModifiesSelfVisitor visitor = new ModifiesSelfVisitor(); + visitor.visit(expression); + return visitor.modifiesSelf(); + } + + private static class ModifiesSelfVisitor extends ExpressionVisitor { + + private String inputField = null; + private String outputField = null; + + public boolean modifiesSelf() { return outputField != null && outputField.equals(inputField); } + + @Override + protected void doVisit(Expression expression) { + if (modifiesSelf()) return; + + if (expression instanceof InputExpression) { + inputField = ((InputExpression) expression).getFieldName(); + } + if (expression instanceof OutputExpression) { + outputField = ((OutputExpression) expression).getFieldName(); + } + } + } + + private static class SetsLanguageVisitor extends ExpressionVisitor { + + boolean setsLanguage = false; + + @Override + protected void doVisit(Expression expression) { + if (expression instanceof SetLanguageExpression) + setsLanguage = true; + } + + } + + private static class FieldScanVisitor extends ExpressionVisitor { + List touchedFields = new ArrayList(); + List candidates = new ArrayList(); + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof OutputExpression) { + touchedFields.add(((OutputExpression) exp).getFieldName()); + } + if (exp instanceof InputExpression) { + candidates.add(((InputExpression) exp).getFieldName()); + } + if (exp instanceof ZCurveExpression) { + touchedFields.addAll(candidates); + } + } + + Collection touchedFields() { + Collection output = touchedFields; + touchedFields = null; // deny re-use to try and avoid obvious bugs + return output; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java new file mode 100644 index 00000000000..162efbb25b4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; + +import java.util.Set; + +/** + * Generated juniperrc-config for controlling juniper. + * + * @author Simon Thoresen Hult + */ +public class Juniperrc extends Derived implements JuniperrcConfig.Producer { + + // List of all fields that should be bolded. + private Set boldingFields = new java.util.LinkedHashSet<>(); + + /** + * Constructs a new juniper rc instance for a given search object. This will derive the configuration automatically, + * so there is no need to call {@link #derive(Schema)}. + * + * @param schema The search model to use for deriving. + */ + public Juniperrc(Schema schema) { + derive(schema); + } + + // Inherit doc from Derived. + @Override + protected void derive(Schema schema) { + super.derive(schema); + for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { + if (summaryField.getTransform() == SummaryTransform.BOLDED) { + boldingFields.add(summaryField.getName()); + } + } + } + + // Inherit doc from Derived. + @Override + protected String getDerivedName() { + return "juniperrc"; + } + + @Override + public void getConfig(JuniperrcConfig.Builder builder) { + if (boldingFields.size() != 0) { + builder.prefix(true); + for (String name : boldingFields) { + builder.override(new JuniperrcConfig.Override.Builder() + .fieldname(name) + .length(65536) + .max_matches(1) + .min_length(8192) + .surround_max(65536)); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinition.java b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinition.java new file mode 100644 index 00000000000..7d558ea51cc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinition.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.document.RankType; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * The definition of a rank type used for native rank features. + * + * @author geirst + */ +public class NativeRankTypeDefinition { + + /** The type this defines */ + private RankType type; + + /** The rank tables of this rank type */ + private List rankTables = new java.util.ArrayList<>(); + + public NativeRankTypeDefinition(RankType type) { + this.type = type; + } + + public RankType getType() { + return type; + } + + public void addTable(NativeTable table) { + rankTables.add(table); + } + + /** Returns an unmodifiable list of the tables in this type definition */ + public Iterator rankSettingIterator() { + return Collections.unmodifiableList(rankTables).iterator(); + } + + public String toString() { + return "native definition of rank type '" + type + "'"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinitionSet.java b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinitionSet.java new file mode 100644 index 00000000000..65e68181b5b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinitionSet.java @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.document.RankType; + +import java.util.Collections; +import java.util.Map; + +/** + * A set of rank type definitions used for native rank features. + * + * @author geirst + */ +public class NativeRankTypeDefinitionSet { + + /** The name of this rank definition set */ + private String name; + + /** The unmodifiable rank type implementations in this set */ + private final Map typeDefinitions; + + /** Returns the default rank type (about) */ + public static RankType getDefaultRankType() { return RankType.ABOUT; } + + public NativeRankTypeDefinitionSet(String name) { + this.name = name; + + Map typeDefinitions = new java.util.LinkedHashMap<>(); + typeDefinitions.put(RankType.IDENTITY, createIdentityRankType(RankType.IDENTITY)); + typeDefinitions.put(RankType.ABOUT, createAboutRankType(RankType.ABOUT)); + typeDefinitions.put(RankType.TAGS, createTagsRankType(RankType.TAGS)); + typeDefinitions.put(RankType.EMPTY, createEmptyRankType(RankType.EMPTY)); + this.typeDefinitions = Collections.unmodifiableMap(typeDefinitions); + } + + private NativeRankTypeDefinition createEmptyRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)")); + return rank; + } + + private NativeRankTypeDefinition createAboutRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)")); + return rank; + } + + private NativeRankTypeDefinition createIdentityRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)")); + return rank; + } + + private NativeRankTypeDefinition createTagsRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)")); + return rank; + } + + /** + * Returns a rank type definition if given an existing rank type name, + * or null if given a rank type which has no native implementation (meaning somebody forgot to add it), + */ + public NativeRankTypeDefinition getRankTypeDefinition(RankType type) { + if (type == RankType.DEFAULT) + type = getDefaultRankType(); + return typeDefinitions.get(type); + } + + /** Returns an unmodifiable map of the type definitions in this */ + public Map types() { return typeDefinitions; } + + public String toString() { + return "native rank type definitions " + name; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/NativeTable.java b/config-model/src/main/java/com/yahoo/schema/derived/NativeTable.java new file mode 100644 index 00000000000..6eff2487bca --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/NativeTable.java @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +/** + * A named rank table of a certain type. + * + * @author geirst + */ +public class NativeTable { + + private String name; + + private Type type; + + /** A table type enumeration */ + public static class Type { + + public static Type FIRST_OCCURRENCE = new Type("firstOccurrenceTable"); + public static Type OCCURRENCE_COUNT = new Type("occurrenceCountTable"); + public static Type WEIGHT = new Type("weightTable"); + public static Type PROXIMITY = new Type("proximityTable"); + public static Type REVERSE_PROXIMITY = new Type("reverseProximityTable"); + + private String name; + + private Type(String name) { + this.name = name; + } + + public String getName() { return name; } + + public boolean equals(Object object) { + if (!(object instanceof Type)) { + return false; + } + Type other = (Type)object; + return this.name.equals(other.name); + } + + public int hashCode() { + return name.hashCode(); + } + + public String toString() { + return getName(); + } + } + + public NativeTable(Type type, String name) { + this.type = type; + this.name = name; + } + + public Type getType() { return type; } + + public String getName() { return name; } + + public int hashCode() { + return type.hashCode() + 17*name.hashCode(); + } + + public boolean equals(Object object) { + if (! (object instanceof NativeTable)) return false; + NativeTable other = (NativeTable)object; + return other.getName().equals(this.getName()) && other.getType().equals(this.getType()); + } + + public String toString() { + return getType() + ": " + getName(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java new file mode 100644 index 00000000000..98815410876 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java @@ -0,0 +1,210 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +/** + * The derived rank profiles of a schema + * + * @author bratseth + */ +public class RankProfileList extends Derived implements RankProfilesConfig.Producer { + + private final Map rankProfiles; + private final FileDistributedConstants constants; + private final LargeRankExpressions largeRankExpressions; + private final FileDistributedOnnxModels onnxModels; + + public static final RankProfileList empty = new RankProfileList(); + + private RankProfileList() { + constants = new FileDistributedConstants(null, List.of()); + largeRankExpressions = new LargeRankExpressions(null); + onnxModels = new FileDistributedOnnxModels(null, List.of()); + rankProfiles = Map.of(); + } + + /** + * Creates a rank profile list + * + * @param schema the schema this is a rank profile from + * @param attributeFields the attribute fields to create a ranking for + */ + public RankProfileList(Schema schema, + LargeRankExpressions largeRankExpressions, + AttributeFields attributeFields, + DeployState deployState) { + setName(schema == null ? "default" : schema.getName()); + this.largeRankExpressions = largeRankExpressions; + this.rankProfiles = deriveRankProfiles(schema, attributeFields, deployState); + this.constants = deriveFileDistributedConstants(schema, rankProfiles.values(), deployState); + this.onnxModels = deriveFileDistributedOnnxModels(schema, rankProfiles.values(), deployState); + } + + private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry, Set processedProfiles) { + return rank.inheritedNames().isEmpty() || + processedProfiles.containsAll(rank.inheritedNames()) || + (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); + } + + private Map deriveRankProfiles(Schema schema, + AttributeFields attributeFields, + DeployState deployState) { + Map rawRankProfiles = new LinkedHashMap<>(); + if (schema != null) { // profiles belonging to a schema have a default profile + RawRankProfile rawRank = new RawRankProfile(deployState.rankProfileRegistry().get(schema, "default"), + largeRankExpressions, + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + attributeFields, + deployState.getProperties()); + rawRankProfiles.put(rawRank.getName(), rawRank); + } + + Map remaining = new LinkedHashMap<>(); + deployState.rankProfileRegistry().rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); + remaining.remove("default"); + while (!remaining.isEmpty()) { + List ready = new ArrayList<>(); + remaining.forEach((name, profile) -> { + if (areDependenciesReady(profile, deployState.rankProfileRegistry(), rawRankProfiles.keySet())) + ready.add(profile); + }); + rawRankProfiles.putAll(processRankProfiles(ready, + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + attributeFields, + deployState.getProperties(), + deployState.getExecutor())); + ready.forEach(rank -> remaining.remove(rank.name())); + } + return rawRankProfiles; + } + + private Map processRankProfiles(List profiles, + QueryProfileRegistry queryProfiles, + ImportedMlModels importedModels, + AttributeFields attributeFields, + ModelContext.Properties deployProperties, + ExecutorService executor) { + Map> futureRawRankProfiles = new LinkedHashMap<>(); + for (RankProfile profile : profiles) { + futureRawRankProfiles.put(profile.name(), executor.submit(() -> new RawRankProfile(profile, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); + } + try { + Map rawRankProfiles = new LinkedHashMap<>(); + for (Future rawFuture : futureRawRankProfiles.values()) { + RawRankProfile rawRank = rawFuture.get(); + rawRankProfiles.put(rawRank.getName(), rawRank); + } + return rawRankProfiles; + } catch (InterruptedException | ExecutionException e) { + throw new IllegalStateException(e); + } + } + + private static FileDistributedConstants deriveFileDistributedConstants(Schema schema, + Collection rankProfiles, + DeployState deployState) { + Map allFileConstants = new HashMap<>(); + addFileConstants(schema != null ? schema.constants().values() : List.of(), + allFileConstants, + schema != null ? schema.toString() : "[global]"); + for (var profile : rankProfiles) + addFileConstants(profile.compiled().constants().values(), allFileConstants, profile.toString()); + return new FileDistributedConstants(deployState.getFileRegistry(), allFileConstants.values()); + } + + private static void addFileConstants(Collection source, + Map destination, + String sourceName) { + for (var constant : source) { + if (constant.valuePath().isEmpty()) continue; + var existing = destination.get(constant.name()); + if ( existing != null && ! constant.equals(existing)) { + throw new IllegalArgumentException("Duplicate constants: " + sourceName + " have " + constant + + ", but we already have " + existing + + ": Value reference constants must be unique across all rank profiles/models"); + } + destination.put(constant.name(), constant); + } + } + + private static FileDistributedOnnxModels deriveFileDistributedOnnxModels(Schema schema, + Collection rankProfiles, + DeployState deployState) { + Map allModels = new LinkedHashMap<>(); + addOnnxModels(schema != null ? schema.onnxModels().values() : List.of(), + allModels, + schema != null ? schema.toString() : "[global]"); + for (var profile : rankProfiles) + addOnnxModels(profile.compiled().onnxModels().values(), allModels, profile.toString()); + return new FileDistributedOnnxModels(deployState.getFileRegistry(), allModels.values()); + } + + private static void addOnnxModels(Collection source, + Map destination, + String sourceName) { + for (var model : source) { + var existing = destination.get(model.getName()); + if ( existing != null && ! model.equals(existing)) { + throw new IllegalArgumentException("Duplicate onnx model: " + sourceName + " have " + model + + ", but we already have " + existing + + ": Onnx models must be unique across all rank profiles/models"); + } + destination.put(model.getName(), model); + } + } + + public Map getRankProfiles() { return rankProfiles; } + public FileDistributedConstants constants() { return constants; } + public FileDistributedOnnxModels getOnnxModels() { return onnxModels; } + + @Override + public String getDerivedName() { return "rank-profiles"; } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + for (RawRankProfile rank : rankProfiles.values() ) { + rank.getConfig(builder); + } + } + + public void getConfig(RankingExpressionsConfig.Builder builder) { + largeRankExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + } + + public void getConfig(RankingConstantsConfig.Builder builder) { + constants.getConfig(builder); + } + + public void getConfig(OnnxModelsConfig.Builder builder) { + onnxModels.getConfig(builder); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java new file mode 100644 index 00000000000..a8a9b4c8755 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -0,0 +1,524 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.google.common.collect.ImmutableList; +import com.yahoo.collections.Pair; +import com.yahoo.compress.Compressor; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankExpressionBody; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.expressiontransforms.OnnxModelTransformer; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.RankProfilesConfig; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.OptionalDouble; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A rank profile derived from a search definition, containing exactly the features available natively in the server + * + * @author bratseth + */ +public class RawRankProfile implements RankProfilesConfig.Producer { + + /** A reusable compressor with default settings */ + private static final Compressor compressor = new Compressor(); + + private static final String keyEndMarker = "\r="; + private static final String valueEndMarker = "\r\n"; + + private final String name; + private final Compressor.Compression compressedProperties; + + /** The compiled profile this is created from. */ + private final RankProfile compiled; + + /** Creates a raw rank profile from the given rank profile. */ + public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, + QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields, ModelContext.Properties deployProperties) { + this.name = rankProfile.name(); + compiled = rankProfile.compile(queryProfiles, importedModels); + compressedProperties = compress(new Deriver(compiled, attributeFields, deployProperties, queryProfiles) + .derive(largeExpressions)); + } + + public RankProfile compiled() { return compiled; } + + private Compressor.Compression compress(List> properties) { + StringBuilder b = new StringBuilder(); + for (Pair property : properties) + b.append(property.getFirst()).append(keyEndMarker).append(property.getSecond()).append(valueEndMarker); + return compressor.compress(b.toString().getBytes(StandardCharsets.UTF_8)); + } + + private List> decompress(Compressor.Compression compression) { + String propertiesString = new String(compressor.decompress(compression), StandardCharsets.UTF_8); + if (propertiesString.isEmpty()) return ImmutableList.of(); + + ImmutableList.Builder> properties = new ImmutableList.Builder<>(); + for (int pos = 0; pos < propertiesString.length();) { + int keyEndPos = propertiesString.indexOf(keyEndMarker, pos); + String key = propertiesString.substring(pos, keyEndPos); + pos = keyEndPos + keyEndMarker.length(); + int valueEndPos = propertiesString.indexOf(valueEndMarker, pos); + String value = propertiesString.substring(pos, valueEndPos); + pos = valueEndPos + valueEndMarker.length(); + properties.add(new Pair<>(key, value)); + } + return properties.build(); + } + + public String getName() { return name; } + + private void getRankProperties(RankProfilesConfig.Rankprofile.Builder b) { + RankProfilesConfig.Rankprofile.Fef.Builder fefB = new RankProfilesConfig.Rankprofile.Fef.Builder(); + for (Pair p : decompress(compressedProperties)) + fefB.property(new RankProfilesConfig.Rankprofile.Fef.Property.Builder().name(p.getFirst()).value(p.getSecond())); + b.fef(fefB); + } + + /** + * Returns the properties of this as an unmodifiable list. + * Note: This method is expensive. + */ + public List> configProperties() { return decompress(compressedProperties); } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); + getRankProperties(b); + builder.rankprofile(b); + } + + @Override + public String toString() { + return " rank profile " + name; + } + + private static class Deriver { + + private final Map fieldRankSettings = new java.util.LinkedHashMap<>(); + private final Set summaryFeatures; + private final Set matchFeatures; + private final Set rankFeatures; + private final Map featureRenames = new java.util.LinkedHashMap<>(); + private final List rankProperties; + + /** + * Rank properties for weight settings to make these available to feature executors + */ + private final List boostAndWeightRankProperties = new ArrayList<>(); + + private final boolean ignoreDefaultRankFeatures; + private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final int rerankCount; + private final int keepRankCount; + private final int numThreadsPerSearch; + private final int minHitsPerThread; + private final int numSearchPartitions; + private final double termwiseLimit; + private final OptionalDouble postFilterThreshold; + private final OptionalDouble approximateThreshold; + private final double rankScoreDropLimit; + private final boolean mapBackRankingExpressionFeatures; + + /** + * The rank type definitions used to derive settings for the native rank features + */ + private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); + private final Map attributeTypes; + private final Map inputs; + private final Set filterFields = new java.util.LinkedHashSet<>(); + private final String rankprofileName; + + private RankingExpression firstPhaseRanking; + private RankingExpression secondPhaseRanking; + + /** + * Creates a raw rank profile from the given rank profile + */ + Deriver(RankProfile compiled, + AttributeFields attributeFields, + ModelContext.Properties deployProperties, + QueryProfileRegistry queryProfiles) { + rankprofileName = compiled.name(); + attributeTypes = compiled.getAttributeTypes(); + inputs = compiled.inputs(); + firstPhaseRanking = compiled.getFirstPhaseRanking(); + secondPhaseRanking = compiled.getSecondPhaseRanking(); + summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); + matchFeatures = new LinkedHashSet<>(compiled.getMatchFeatures()); + rankFeatures = compiled.getRankFeatures(); + rerankCount = compiled.getRerankCount(); + matchPhaseSettings = compiled.getMatchPhaseSettings(); + numThreadsPerSearch = compiled.getNumThreadsPerSearch(); + minHitsPerThread = compiled.getMinHitsPerThread(); + numSearchPartitions = compiled.getNumSearchPartitions(); + termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); + postFilterThreshold = compiled.getPostFilterThreshold(); + approximateThreshold = compiled.getApproximateThreshold(); + keepRankCount = compiled.getKeepRankCount(); + rankScoreDropLimit = compiled.getRankScoreDropLimit(); + mapBackRankingExpressionFeatures = deployProperties.featureFlags().avoidRenamingSummaryFeatures(); + ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); + rankProperties = new ArrayList<>(compiled.getRankProperties()); + + Map functions = compiled.getFunctions(); + List functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); + Map functionProperties = new LinkedHashMap<>(); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, + Map.of(), + compiled.typeContext(queryProfiles)); + + if (firstPhaseRanking != null) { + functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); + } + if (secondPhaseRanking != null) { + functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext)); + } + + derivePropertiesAndFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); + deriveOnnxModelFunctionsAndFeatures(compiled); + + deriveRankTypeSetting(compiled, attributeFields); + deriveFilterFields(compiled); + deriveWeightProperties(compiled); + } + + private void deriveFilterFields(RankProfile rp) { + filterFields.addAll(rp.allFilterFields()); + } + + private void derivePropertiesAndFeaturesFromFunctions(Map functions, + Map functionProperties, + SerializationContext functionContext) { + if (functions.isEmpty()) return; + + replaceFunctionFeatures(summaryFeatures, functionContext); + replaceFunctionFeatures(matchFeatures, functionContext); + + // First phase, second phase and summary features should add all required functions to the context. + // However, we need to add any functions not referenced in those anyway for model-evaluation. + deriveFunctionProperties(functions, functionProperties, functionContext); + + for (Map.Entry e : functionProperties.entrySet()) { + rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); + } + } + + private void deriveFunctionProperties(Map functions, + Map functionProperties, + SerializationContext context) { + for (Map.Entry e : functions.entrySet()) { + String propertyName = RankingExpression.propertyName(e.getKey()); + if (context.serializedFunctions().containsKey(propertyName)) continue; + + String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); + + context.addFunctionSerialization(propertyName, expressionString); + for (Map.Entry argumentType : e.getValue().function().argumentTypes().entrySet()) + context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); + if (e.getValue().function().returnType().isPresent()) + context.addFunctionTypeSerialization(e.getKey(), e.getValue().function().returnType().get()); + // else if (e.getValue().function().arguments().isEmpty()) TODO: Enable this check when we resolve all types + // throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); + } + functionProperties.putAll(context.serializedFunctions()); + } + + private void replaceFunctionFeatures(Set features, SerializationContext context) { + if (features == null) return; + Map functionFeatures = new LinkedHashMap<>(); + for (Iterator i = features.iterator(); i.hasNext(); ) { + ReferenceNode referenceNode = i.next(); + // Is the feature a function? + ExpressionFunction function = context.getFunction(referenceNode.getName()); + if (function != null) { + String propertyName = RankingExpression.propertyName(referenceNode.getName()); + String expressionString = function.getBody().getRoot().toString(context).toString(); + context.addFunctionSerialization(propertyName, expressionString); + ReferenceNode backendReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", + referenceNode.getArguments().expressions(), + referenceNode.getOutput()); + if (mapBackRankingExpressionFeatures) { + // tell backend to map back to the name the user expects: + featureRenames.put(backendReferenceNode.toString(), referenceNode.toString()); + } + functionFeatures.put(referenceNode.getName(), backendReferenceNode); + i.remove(); // Will add the expanded one in next block + } + } + // Then, replace the features that were functions + for (Map.Entry e : functionFeatures.entrySet()) { + features.add(e.getValue()); + } + } + + private void deriveWeightProperties(RankProfile rankProfile) { + + for (RankProfile.RankSetting setting : rankProfile.rankSettings()) { + if (setting.getType() != RankProfile.RankSetting.Type.WEIGHT) continue; + boostAndWeightRankProperties.add(new RankProfile.RankProperty("vespa.fieldweight." + setting.getFieldName(), + String.valueOf(setting.getIntValue()))); + } + } + + /** + * Adds the type boosts from a rank profile + */ + private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) { + for (Iterator i = rankProfile.rankSettingIterator(); i.hasNext(); ) { + RankProfile.RankSetting setting = i.next(); + if (setting.getType() != RankProfile.RankSetting.Type.RANKTYPE) continue; + + deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields, + hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); + } + } + + private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, + boolean isDefaultSetting) { + if (isDefaultSetting) return; + + NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType); + if (definition == null) throw new IllegalArgumentException("In field '" + fieldName + "': " + + rankType + " is known but has no implementation. " + + "Supported rank types: " + + nativeRankTypeDefinitions.types().keySet()); + + FieldRankSettings settings = deriveFieldRankSettings(fieldName); + for (Iterator i = definition.rankSettingIterator(); i.hasNext(); ) { + NativeTable table = i.next(); + // only add index field tables if we are processing an index field and + // only add attribute field tables if we are processing an attribute field + if ((FieldRankSettings.isIndexFieldTable(table) && attributeFields.getAttribute(fieldName) == null) || + (FieldRankSettings.isAttributeFieldTable(table) && attributeFields.getAttribute(fieldName) != null)) { + settings.addTable(table); + } + } + } + + private boolean hasDefaultRankTypeSetting(RankProfile rankProfile, String fieldName) { + RankProfile.RankSetting setting = + rankProfile.getRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE); + return setting != null && setting.getValue().equals(RankType.DEFAULT); + } + + private FieldRankSettings deriveFieldRankSettings(String fieldName) { + FieldRankSettings settings = fieldRankSettings.get(fieldName); + if (settings == null) { + settings = new FieldRankSettings(fieldName); + fieldRankSettings.put(fieldName, settings); + } + return settings; + } + + /** Derives the properties this produces */ + public List> derive(LargeRankExpressions largeRankExpressions) { + List> properties = new ArrayList<>(); + for (RankProfile.RankProperty property : rankProperties) { + if (RankingExpression.propertyName(RankProfile.FIRST_PHASE).equals(property.getName())) { + // Could have been set by function expansion. Set expressions, then skip this property. + try { + firstPhaseRanking = new RankingExpression(property.getValue()); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse first phase expression", e); + } + } + else if (RankingExpression.propertyName(RankProfile.SECOND_PHASE).equals(property.getName())) { + try { + secondPhaseRanking = new RankingExpression(property.getValue()); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse second phase expression", e); + } + } + else { + properties.add(new Pair<>(property.getName(), property.getValue())); + } + } + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, RankProfile.SECOND_PHASE)); + for (FieldRankSettings settings : fieldRankSettings.values()) { + properties.addAll(settings.deriveRankProperties()); + } + for (RankProfile.RankProperty property : boostAndWeightRankProperties) { + properties.add(new Pair<>(property.getName(), property.getValue())); + } + for (ReferenceNode feature : summaryFeatures) { + properties.add(new Pair<>("vespa.summary.feature", feature.toString())); + } + for (ReferenceNode feature : matchFeatures) { + properties.add(new Pair<>("vespa.match.feature", feature.toString())); + } + for (ReferenceNode feature : rankFeatures) { + properties.add(new Pair<>("vespa.dump.feature", feature.toString())); + } + for (var entry : featureRenames.entrySet()) { + properties.add(new Pair<>("vespa.feature.rename", entry.getKey())); + properties.add(new Pair<>("vespa.feature.rename", entry.getValue())); + } + if (numThreadsPerSearch > 0) { + properties.add(new Pair<>("vespa.matching.numthreadspersearch", numThreadsPerSearch + "")); + } + if (minHitsPerThread > 0) { + properties.add(new Pair<>("vespa.matching.minhitsperthread", minHitsPerThread + "")); + } + if (numSearchPartitions >= 0) { + properties.add(new Pair<>("vespa.matching.numsearchpartitions", numSearchPartitions + "")); + } + if (termwiseLimit < 1.0) { + properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + "")); + } + if (postFilterThreshold.isPresent()) { + properties.add(new Pair<>("vespa.matching.global_filter.upper_limit", String.valueOf(postFilterThreshold.getAsDouble()))); + } + if (approximateThreshold.isPresent()) { + properties.add(new Pair<>("vespa.matching.global_filter.lower_limit", String.valueOf(approximateThreshold.getAsDouble()))); + } + if (matchPhaseSettings != null) { + properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); + RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); + if (diversitySettings != null) { + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); + } + } + if (rerankCount > -1) { + properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); + } + if (keepRankCount > -1) { + properties.add(new Pair<>("vespa.hitcollector.arraysize", keepRankCount + "")); + } + if (rankScoreDropLimit > -Double.MAX_VALUE) { + properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); + } + if (ignoreDefaultRankFeatures) { + properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); + } + for (String fieldName : filterFields) { + properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); + } + for (Map.Entry attributeType : attributeTypes.entrySet()) { + properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); + } + + for (var input : inputs.values()) { + if (FeatureNames.isQueryFeature(input.name())) { + if (input.type().rank() > 0) // Proton does not like representing the double type as a rank 0 tensor + properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), + input.type().toString())); + if (input.defaultValue().isPresent()) { + properties.add(new Pair<>(input.name().toString(), + input.type().rank() == 0 ? + String.valueOf(input.defaultValue().get().asDouble()) : + input.defaultValue().get().toString(true, false))); + } + } + } + if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); + distributeLargeExpressionsAsFiles(properties, largeRankExpressions); + return properties; + } + + private void distributeLargeExpressionsAsFiles(List> properties, LargeRankExpressions largeRankExpressions) { + for (ListIterator> iter = properties.listIterator(); iter.hasNext();) { + Pair property = iter.next(); + String expression = property.getSecond(); + if (expression.length() > largeRankExpressions.limit()) { + String propertyName = property.getFirst(); + String functionName = RankingExpression.extractScriptName(propertyName); + if (functionName != null) { + String mangledName = rankprofileName + "." + functionName; + largeRankExpressions.add(new RankExpressionBody(mangledName, ByteBuffer.wrap(expression.getBytes(StandardCharsets.UTF_8)))); + iter.set(new Pair<>(RankingExpression.propertyExpressionName(functionName), mangledName)); + } + } + } + } + + private List> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + List> properties = new ArrayList<>(); + if (expression == null) return properties; + + String name = expression.getName(); + if ("".equals(name)) + name = phase; + + if (expression.getRoot() instanceof ReferenceNode) { + properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); + } else { + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); + properties.add(new Pair<>(RankingExpression.propertyName(name), expression.getRoot().toString())); + } + return properties; + } + + private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { + if (rankProfile.schema() == null) return; + if (rankProfile.onnxModels().isEmpty()) return; + replaceOnnxFunctionInputs(rankProfile); + replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); + replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); + } + + private void replaceOnnxFunctionInputs(RankProfile rankProfile) { + Set functionNames = rankProfile.getFunctions().keySet(); + if (functionNames.isEmpty()) return; + for (OnnxModel onnxModel: rankProfile.onnxModels().values()) { + for (Map.Entry mapping : onnxModel.getInputMap().entrySet()) { + String source = mapping.getValue(); + if (functionNames.contains(source)) { + onnxModel.addInputNameMapping(mapping.getKey(), "rankingExpression(" + source + ")"); + } + } + } + } + + private void replaceImplicitOnnxConfigFeatures(Set features, RankProfile rankProfile) { + if (features == null || features.isEmpty()) return; + Set replacedFeatures = new HashSet<>(); + for (Iterator i = features.iterator(); i.hasNext(); ) { + ReferenceNode referenceNode = i.next(); + ReferenceNode replacedNode = (ReferenceNode) OnnxModelTransformer.transformFeature(referenceNode, rankProfile); + if (referenceNode != replacedNode) { + replacedFeatures.add(replacedNode); + i.remove(); + } + } + features.addAll(replacedFeatures); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java new file mode 100644 index 00000000000..18c6f335787 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Information about a schema. + * + * @author bratseth + */ +public final class SchemaInfo extends Derived implements SchemaInfoConfig.Producer { + + private final Schema schema; + + // Info about profiles needed in memory after build. + // The rank profile registry itself is not kept around due to its size. + private final Map rankProfiles; + + private final Summaries summaries; + private final SummaryMap summaryMap; + + public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, + Summaries summaries, SummaryMap summaryMap) { + this.schema = schema; + this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); + this.summaries = summaries; + this.summaryMap = summaryMap; + } + + public String name() { return schema.getName(); } + + @Override + public String getDerivedName() { return "schema-info"; } + + public Schema fullSchema() { return schema; } + + public Map rankProfiles() { return rankProfiles; } + + private Map toRankProfiles(Collection rankProfiles) { + Map rankProfileInfos = new LinkedHashMap<>(); + rankProfiles.forEach(profile -> rankProfileInfos.put(profile.name(), new RankProfileInfo(profile))); + return rankProfileInfos; + } + + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); + schemaBuilder.name(schema.getName()); + addSummaryConfig(schemaBuilder); + addRankProfilesConfig(schemaBuilder); + builder.schema(schemaBuilder); + } + + private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var summary : summaries.asList()) { + var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); + summaryBuilder.name(summary.getName()); + for (var field : summary.fields().values()) { + var fieldsBuilder = new SchemaInfoConfig.Schema.Summaryclass.Fields.Builder(); + fieldsBuilder.name(field.getName()) + .type(field.getType().getName()) + .dynamic(isDynamic(field.getName())); + summaryBuilder.fields(fieldsBuilder); + } + schemaBuilder.summaryclass(summaryBuilder); + } + } + + /** Returns whether the given field is a dynamic summary field. */ + private boolean isDynamic(String fieldName) { + if (summaryMap == null) return false; // not know for streaming, but also not used + + var fieldTransform = summaryMap.resultTransforms().get(fieldName); + if (fieldTransform == null) return false; + // TODO: Move this into SummaryTransform and call it something else than "dynamic" + return fieldTransform.getTransform().isDynamic() || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER; + } + + private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (RankProfileInfo rankProfile : rankProfiles().values()) { + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileConfig.name(rankProfile.name()); + rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); + rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + for (var input : rankProfile.inputs().entrySet()) { + var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); + inputConfig.name(input.getKey().toString()); + inputConfig.type(input.getValue().type().toString()); + rankProfileConfig.input(inputConfig); + } + schemaBuilder.rankprofile(rankProfileConfig); + } + } + + /** A store of a *small* (in memory) amount of rank profile info. */ + public static final class RankProfileInfo { + + private final String name; + private final boolean hasSummaryFeatures; + private final boolean hasRankFeatures; + private final Map inputs; + + public RankProfileInfo(RankProfile profile) { + this.name = profile.name(); + this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); + this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); + this.inputs = profile.inputs(); + } + + public String name() { return name; } + public boolean hasSummaryFeatures() { return hasSummaryFeatures; } + public boolean hasRankFeatures() { return hasRankFeatures; } + public Map inputs() { return inputs; } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java new file mode 100644 index 00000000000..3bab808beff --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java @@ -0,0 +1,123 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.DataTypeName; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; + +import java.util.*; + +/** + *

A class which can reorder a list of search definitions such that any supertype + * always preceed any subtype. Subject to this condition the given order + * is preserved (the minimal reordering is done).

+ * + *

This class is not multithread safe. Only one ordering must be done + * at the time in any instance.

+ * + * @author bratseth + * @author bjorncs + */ +public class SearchOrderer { + + /** A map from DataTypeName to the Search defining them */ + private final Map documentNameToSearch = new HashMap<>(); + + /** + * Reorders the given list of search definitions such that any supertype + * always preceed any subtype. Subject to this condition the given order + * is preserved (the minimal reordering is done). + * + * @return a new list containing the same search instances in the right order + */ + public List order(List unordered) { + // Description above state that the original order should be preserved, except for the dependency constraint. + // Yet we botch that guarantee by sorting the list... + unordered.sort(Comparator.comparing(Schema::getName)); + + // No, this is not a fast algorithm... + indexOnDocumentName(unordered); + List ordered = new ArrayList<>(unordered.size()); + List moveOutwards = new ArrayList<>(); + for (Schema schema : unordered) { + if (allDependenciesAlreadyEmitted(ordered, schema)) { + addOrdered(ordered, schema, moveOutwards); + } + else { + moveOutwards.add(schema); + } + } + + // Any leftovers means we have search definitions with undefined inheritants. + // This is warned about elsewhere. + ordered.addAll(moveOutwards); + + documentNameToSearch.clear(); + return ordered; + } + + private void addOrdered(List ordered, Schema schema, List moveOutwards) { + ordered.add(schema); + Schema eligibleMove; + do { + eligibleMove = removeFirstEntryWithFullyEmittedDependencies(moveOutwards, ordered); + if (eligibleMove != null) { + ordered.add(eligibleMove); + } + } while (eligibleMove != null); + } + + /** Removes and returns the first search from the move list which can now be added, or null if none */ + private Schema removeFirstEntryWithFullyEmittedDependencies(List moveOutwards, List ordered) { + for (Schema move : moveOutwards) { + if (allDependenciesAlreadyEmitted(ordered, move)) { + moveOutwards.remove(move); + return move; + } + } + return null; + } + + private boolean allDependenciesAlreadyEmitted(List alreadyOrdered, Schema schema) { + if (schema.getDocument() == null) { + return true; + } + SDDocumentType document = schema.getDocument(); + return allInheritedDependenciesEmitted(alreadyOrdered, document) && allReferenceDependenciesEmitted(alreadyOrdered, document); + } + + private boolean allInheritedDependenciesEmitted(List alreadyOrdered, SDDocumentType document) { + for (SDDocumentType sdoc : document.getInheritedTypes() ) { + DataTypeName inheritedName = sdoc.getDocumentName(); + if ("document".equals(inheritedName.getName())) { + continue; + } + Schema inheritedSchema = documentNameToSearch.get(inheritedName); + if (!alreadyOrdered.contains(inheritedSchema)) { + return false; + } + } + return true; + } + + private static boolean allReferenceDependenciesEmitted(List alreadyOrdered, SDDocumentType document) { + DocumentReferences documentReferences = document.getDocumentReferences() + .orElseThrow(() -> new IllegalStateException("Missing document references. Should have been processed by now.")); + return documentReferences.stream() + .map(Map.Entry::getValue) + .map(DocumentReference::targetSearch) + .allMatch(alreadyOrdered::contains); + } + + private void indexOnDocumentName(List schemas) { + documentNameToSearch.clear(); + for (Schema schema : schemas) { + if (schema.getDocument() != null) { + documentNameToSearch.put(schema.getDocument().getDocumentName(), schema); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java new file mode 100644 index 00000000000..2b41fbb3b1a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.config.search.SummaryConfig; + +import java.util.ArrayList; +import java.util.List; + +/** + * A list of derived summaries + * + * @author bratseth + */ +public class Summaries extends Derived implements SummaryConfig.Producer { + + private final boolean useV8GeoPositions; + private final List summaries; + + public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) { + super(); + this.useV8GeoPositions = featureFlags.useV8GeoPositions(); + + // Make sure the default is first + List summaries = new ArrayList<>(); + summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger)); + for (DocumentSummary summary : schema.getSummaries().values()) { + if (!summary.getName().equals("default")) + summaries.add(new SummaryClass(schema, summary, deployLogger)); + } + this.summaries = List.copyOf(summaries); + } + + public List asList() { return summaries; } + + @Override + protected String getDerivedName() { return "summary"; } + + @Override + public void getConfig(SummaryConfig.Builder builder) { + builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); + builder.usev8geopositions(useV8GeoPositions); + for (SummaryClass summaryClass : summaries) { + builder.classes(summaryClass.getSummaryClassConfig()); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java new file mode 100644 index 00000000000..193c6893203 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import java.util.Collections; +import java.util.Map; +import java.util.logging.Level; + +/** + * A summary derived from a search definition. + * Each summary definition have at least one summary, the default + * which has the same name as the search definition. + * + * @author bratseth + */ +public class SummaryClass extends Derived { + + public static final String DOCUMENT_ID_FIELD = "documentid"; + + private final int id; + + /** True if this summary class needs to access summary information on disk */ + private boolean accessingDiskSummary = false; + private final boolean rawAsBase64; + private final boolean omitSummaryFeatures; + + /** The summary fields of this indexed by name */ + private final Map fields; + + private final DeployLogger deployLogger; + + /** + * Creates a summary class from a search definition summary + * + * @param deployLogger a {@link DeployLogger} + */ + public SummaryClass(Schema schema, DocumentSummary summary, DeployLogger deployLogger) { + super(summary.getName()); + this.deployLogger = deployLogger; + this.rawAsBase64 = schema.isRawAsBase64(); + this.omitSummaryFeatures = summary.omitSummaryFeatures(); + Map fields = new java.util.LinkedHashMap<>(); + deriveFields(schema, summary, fields); + deriveImplicitFields(summary, fields); + this.fields = Collections.unmodifiableMap(fields); + this.id = deriveId(summary.getName(), fields); + } + + public int id() { return id; } + + /** MUST be called after all other fields are added */ + private void deriveImplicitFields(DocumentSummary summary, Map fields) { + if (summary.getName().equals("default")) { + addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields); + } + } + + private void deriveFields(Schema schema, DocumentSummary summary, Map fields) { + for (SummaryField summaryField : summary.getSummaryFields().values()) { + if (!accessingDiskSummary && schema.isAccessingDiskSummary(summaryField)) { + accessingDiskSummary = true; + } + addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), fields); + } + } + + private void addField(String name, DataType type, Map fields) { + addField(name, type, null, fields); + } + + private void addField(String name, DataType type, + SummaryTransform transform, + Map fields) { + if (fields.containsKey(name)) { + SummaryClassField sf = fields.get(name); + if ( SummaryClassField.convertDataType(type, transform, rawAsBase64) != sf.getType()) { + deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + + ". " + "Declared as type " + sf.getType() + " and " + type); + } + } else { + fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); + } + } + + public Map fields() { return fields; } + + private static int deriveId(String name, Map fields) { + int hash = name.hashCode(); + int number = 1; + for (var field : fields.values()) { + hash += number++ * (field.getName().hashCode() + + 17 * field.getType().getName().hashCode()); + } + hash = Math.abs(hash); + if (hash == DocsumDefinitionSet.SLIME_MAGIC_ID) + hash++; + return hash; + } + + public SummaryConfig.Classes.Builder getSummaryClassConfig() { + SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder(); + classBuilder. + id(id). + name(getName()). + omitsummaryfeatures(omitSummaryFeatures); + for (SummaryClassField field : fields.values() ) { + classBuilder.fields(new SummaryConfig.Classes.Fields.Builder(). + name(field.getName()). + type(field.getType().getName())); + } + return classBuilder; + } + + @Override + public int hashCode() { return id; } + + @Override + protected String getDerivedName() { return "summary"; } + + @Override + public String toString() { + return "summary class '" + getName() + "'"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java new file mode 100644 index 00000000000..f042054a0b5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -0,0 +1,132 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Float16FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +/** + * A summary field derived from a search definition + * + * @author bratseth + */ +public class SummaryClassField { + + private final String name; + + private final Type type; + + /** The summary field type enumeration */ + public enum Type { + + BOOL("bool"), + BYTE("byte"), + SHORT("short"), + INTEGER("integer"), + INT64("int64"), + FLOAT16("float16"), + FLOAT("float"), + DOUBLE("double"), + STRING("string"), + DATA("data"), + RAW("raw"), + LONGSTRING("longstring"), + LONGDATA("longdata"), + XMLSTRING("xmlstring"), + FEATUREDATA("featuredata"), + JSONSTRING("jsonstring"), + TENSOR("tensor"); + + private final String name; + + Type(String name) { + this.name = name; + } + + /** Returns the name of this type */ + public String getName() { + return name; + } + + public String toString() { + return "type: " + name; + } + } + + public SummaryClassField(String name, DataType type, SummaryTransform transform, boolean rawAsBase64) { + this.name = name; + this.type = convertDataType(type, transform, rawAsBase64); + } + + public String getName() { return name; } + + public Type getType() { return type; } + + /** Converts to the right summary field type from a field datatype and a transform*/ + public static Type convertDataType(DataType fieldType, SummaryTransform transform, boolean rawAsBase64) { + FieldValue fval = fieldType.createFieldValue(); + if (fval instanceof StringFieldValue) { + if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) { + return Type.FEATUREDATA; + } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { + return Type.FEATUREDATA; + } else { + return Type.LONGSTRING; + } + } else if (fval instanceof IntegerFieldValue) { + return Type.INTEGER; + } else if (fval instanceof LongFieldValue) { + return Type.INT64; + } else if (fval instanceof Float16FieldValue) { + return Type.FLOAT16; + } else if (fval instanceof FloatFieldValue) { + return Type.FLOAT; + } else if (fval instanceof DoubleFieldValue) { + return Type.DOUBLE; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; + } else if (fval instanceof ByteFieldValue) { + return Type.BYTE; + } else if (fval instanceof Raw) { + return rawAsBase64 ? Type.RAW : Type.DATA; + } else if (fval instanceof Struct) { + return Type.JSONSTRING; + } else if (fval instanceof PredicateFieldValue) { + return Type.STRING; + } else if (fval instanceof TensorFieldValue) { + return Type.TENSOR; + } else if (fieldType instanceof CollectionDataType) { + if (transform != null && transform.equals(SummaryTransform.POSITIONS)) { + return Type.XMLSTRING; + } else { + return Type.JSONSTRING; + } + } else if (fieldType instanceof MapDataType) { + return Type.JSONSTRING; + } else if (fieldType instanceof NewDocumentReferenceDataType) { + return Type.LONGSTRING; + } else { + throw new IllegalArgumentException("Don't know which summary type to convert " + fieldType + " to"); + } + } + + public String toString() { + return "summary class field " + name; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java new file mode 100644 index 00000000000..df9174a12ed --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import java.util.Collections; +import java.util.Map; + +/** + * A summary map (describing search-time summary field transformations) + * derived from a Schema. + * + * @author bratseth + */ +public class SummaryMap extends Derived implements SummarymapConfig.Producer { + + private final Map resultTransforms = new java.util.LinkedHashMap<>(); + + /** Creates a summary map from a search definition */ + SummaryMap(Schema schema) { + derive(schema); + } + + protected void derive(Schema schema) { + for (DocumentSummary documentSummary : schema.getSummaries().values()) { + derive(documentSummary); + } + super.derive(schema); + } + + @Override + protected void derive(ImmutableSDField field, Schema schema) { + } + + private void derive(DocumentSummary documentSummary) { + for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { + if (summaryField.getTransform()== SummaryTransform.NONE) continue; + + if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || + summaryField.getTransform()==SummaryTransform.DISTANCE || + summaryField.getTransform()==SummaryTransform.GEOPOS || + summaryField.getTransform()==SummaryTransform.POSITIONS || + summaryField.getTransform()==SummaryTransform.MATCHED_ELEMENTS_FILTER || + summaryField.getTransform()==SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) + { + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getSingleSource())); + } else { + // Note: Currently source mapping is handled in the indexing statement, + // by creating a summary field for each of the values + // This works, but is suboptimal. We could consolidate to a minimal set and + // use the right value from the minimal set as the third parameter here, + // and add "override" commands to multiple static values + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getName())); + } + } + } + + /** Returns a read-only iterator of the FieldResultTransforms of this summary map */ + public Map resultTransforms() { + return Collections.unmodifiableMap(resultTransforms); + } + + protected String getDerivedName() { return "summarymap"; } + + /** Returns the command name of a transform */ + private String getCommand(SummaryTransform transform) { + if (transform == SummaryTransform.DISTANCE) + return "absdist"; + else if (transform.isDynamic()) + return "dynamicteaser"; + else + return transform.getName(); + } + + /** + * Does this summary command name stand for a dynamic transform? + * We need this because some model information is shared through configs instead of model - see usage + * A dynamic transform needs the query to perform its computations. + */ + // TODO/Note: "dynamic" here means something else than in SummaryTransform + public static boolean isDynamicCommand(String commandName) { + return (commandName.equals("dynamicteaser") || + commandName.equals(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()) || + commandName.equals(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName())); + } + + @Override + public void getConfig(SummarymapConfig.Builder builder) { + builder.defaultoutputclass(-1); + for (FieldResultTransform frt : resultTransforms.values()) { + SummarymapConfig.Override.Builder oB = new SummarymapConfig.Override.Builder() + .field(frt.getFieldName()) + .command(getCommand(frt.getTransform())); + if (frt.getTransform().isDynamic() || + frt.getTransform().equals(SummaryTransform.ATTRIBUTE) || + frt.getTransform().equals(SummaryTransform.DISTANCE) || + frt.getTransform().equals(SummaryTransform.GEOPOS) || + frt.getTransform().equals(SummaryTransform.POSITIONS) || + frt.getTransform().equals(SummaryTransform.TEXTEXTRACTOR) || + frt.getTransform().equals(SummaryTransform.MATCHED_ELEMENTS_FILTER) || + frt.getTransform().equals(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER)) + { + oB.arguments(frt.getArgument()); + } else { + oB.arguments(""); + } + builder.override(oB); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java new file mode 100644 index 00000000000..c8679b6166c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -0,0 +1,313 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Vertical streaming matcher field specification + */ +public class VsmFields extends Derived implements VsmfieldsConfig.Producer { + + private final Map fields=new LinkedHashMap<>(); + private final Map doctypes=new LinkedHashMap<>(); + + public VsmFields(Schema schema) { + addSearchdefinition(schema); + } + + private void addSearchdefinition(Schema schema) { + derive(schema); + } + + @Override + protected void derive(SDDocumentType document, Schema schema) { + super.derive(document, schema); + StreamingDocumentType docType=getDocumentType(document.getName()); + if (docType == null) { + docType = new StreamingDocumentType(document.getName(), schema.fieldSets()); + doctypes.put(document.getName(), docType); + } + for (Object o : document.fieldSet()) { + derive(docType, (SDField) o); + } + } + + protected void derive(StreamingDocumentType document, SDField field) { + if (field.usesStructOrMap()) { + if (GeoPos.isAnyPos(field)) { + StreamingField streamingField = new StreamingField(field); + addField(streamingField.getName(), streamingField); + addFieldToIndices(document, field.getName(), streamingField); + } + for (SDField structField : field.getStructFields()) { + derive(document, structField); // Recursion + } + } else { + if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) ) + return; + + StreamingField streamingField = new StreamingField(field); + addField(streamingField.getName(),streamingField); + deriveIndices(document, field, streamingField); + } + } + + private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField) { + if (field.doesIndexing()) { + addFieldToIndices(document, field.getName(), streamingField); + } else if (field.doesAttributing()) { + for (String indexName : field.getAttributes().keySet()) { + addFieldToIndices(document, indexName, streamingField); + } + } + } + + private void addFieldToIndices(StreamingDocumentType document, String indexName, StreamingField streamingField) { + if (indexName.contains(".")) { + addFieldToIndices(document, indexName.substring(0,indexName.lastIndexOf(".")), streamingField); // Recursion + } + document.addIndexField(indexName, streamingField.getName()); + } + + private void addField(String name, StreamingField field) { + fields.put(name, field); + } + + /** Returns a streaming index, or null if there is none with this name */ + public StreamingDocumentType getDocumentType(String name) { + return doctypes.get(name); + } + + public String getDerivedName() { + return "vsmfields"; + } + + @Override + public void getConfig(VsmfieldsConfig.Builder vsB) { + for (StreamingField streamingField : fields.values()) { + vsB.fieldspec(streamingField.getFieldSpecConfig()); + } + for (StreamingDocumentType streamingDocType : doctypes.values()) { + vsB.documenttype(streamingDocType.getDocTypeConfig()); + } + } + + private static class StreamingField { + + private final String name; + + /** Whether this field does prefix matching by default */ + private final Matching matching; + + /** The type of this field */ + private final Type type; + + private final boolean isAttribute; + + /** The streaming field type enumeration */ + public static class Type { + + public static Type INT8 = new Type("int8","INT8"); + public static Type INT16 = new Type("int16","INT16"); + public static Type INT32 = new Type("int32","INT32"); + public static Type INT64 = new Type("int64","INT64"); + public static Type FLOAT16 = new Type("float16", "FLOAT16"); + public static Type FLOAT = new Type("float","FLOAT"); + public static Type DOUBLE = new Type("double","DOUBLE"); + public static Type STRING = new Type("string","AUTOUTF8"); + public static Type BOOL = new Type("bool","BOOL"); + public static Type UNSEARCHABLESTRING = new Type("string","NONE"); + public static Type GEO_POSITION = new Type("position", "GEOPOS"); + + private String name; + + private String searchMethod; + + private Type(String name, String searchMethod) { + this.name = name; + this.searchMethod = searchMethod; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + /** Returns the name of this type */ + public String getName() { return name; } + + public String getSearchMethod() { return searchMethod; } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof Type)) return false; + return this.name.equals(((Type)other).name); + } + + @Override + public String toString() { + return "type: " + name; + } + + } + + public StreamingField(SDField field) { + this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing()); + } + + private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute) { + this.name = name; + this.type = convertType(sourceType); + this.matching = matching; + this.isAttribute = isAttribute; + } + + /** Converts to the right index type from a field datatype */ + private static Type convertType(DataType fieldType) { + FieldValue fval = fieldType.createFieldValue(); + if (fieldType.equals(DataType.FLOAT16)) { + return Type.FLOAT16; + } else if (fieldType.equals(DataType.FLOAT)) { + return Type.FLOAT; + } else if (fieldType.equals(DataType.LONG)) { + return Type.INT64; + } else if (fieldType.equals(DataType.DOUBLE)) { + return Type.DOUBLE; + } else if (fieldType.equals(DataType.BOOL)) { + return Type.BOOL; + } else if (fieldType.equals(DataType.BYTE)) { + return Type.INT8; + } else if (GeoPos.isAnyPos(fieldType)) { + return Type.GEO_POSITION; + } else if (fieldType instanceof NumericDataType) { + return Type.INT32; + } else if (fval instanceof StringFieldValue) { + return Type.STRING; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; + } else if (fval instanceof Raw) { + return Type.STRING; + } else if (fval instanceof PredicateFieldValue) { + return Type.UNSEARCHABLESTRING; + } else if (fval instanceof TensorFieldValue) { + return Type.UNSEARCHABLESTRING; + } else if (fieldType instanceof CollectionDataType) { + return convertType(((CollectionDataType) fieldType).getNestedType()); + } else if (fieldType instanceof NewDocumentReferenceDataType) { + return Type.UNSEARCHABLESTRING; + } else { + throw new IllegalArgumentException("Don't know which streaming field type to convert " + + fieldType + " to"); + } + } + + public String getName() { return name; } + + public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() { + VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder(); + String matchingName = matching.getType().getName(); + if (matching.getType().equals(MatchType.TEXT)) + matchingName = ""; + if (matching.getType() != MatchType.EXACT) { + if (matching.isPrefix()) { + matchingName = "prefix"; + } else if (matching.isSubstring()) { + matchingName = "substring"; + } else if (matching.isSuffix()) { + matchingName = "suffix"; + } + } + if (type != Type.STRING) { + matchingName = ""; + } + fB.name(getName()) + .searchmethod(VsmfieldsConfig.Fieldspec.Searchmethod.Enum.valueOf(type.getSearchMethod())) + .arg1(matchingName) + .fieldtype(isAttribute + ? VsmfieldsConfig.Fieldspec.Fieldtype.ATTRIBUTE + : VsmfieldsConfig.Fieldspec.Fieldtype.INDEX); + if (matching.maxLength() != null) { + fB.maxlength(matching.maxLength()); + } + return fB; + } + + @Override + public boolean equals(Object o) { + if (o.getClass().equals(getClass())) { + StreamingField sf = (StreamingField)o; + return name.equals(sf.name) && + matching.equals(sf.matching) && + type.equals(sf.type); + } + return false; + } + + @Override public int hashCode() { + return java.util.Objects.hash(name, matching, type); + } + + } + + private static class StreamingDocumentType { + + private final String name; + private final Map fieldSets = new LinkedHashMap<>(); + private final Map userFieldSets; + + public StreamingDocumentType(String name, FieldSets fieldSets) { + this.name=name; + userFieldSets = fieldSets.userFieldSets(); + } + + public VsmfieldsConfig.Documenttype.Builder getDocTypeConfig() { + VsmfieldsConfig.Documenttype.Builder dtB = new VsmfieldsConfig.Documenttype.Builder(); + dtB.name(name); + Map all = new LinkedHashMap<>(); + all.putAll(fieldSets); + all.putAll(userFieldSets); + for (Map.Entry e : all.entrySet()) { + VsmfieldsConfig.Documenttype.Index.Builder indB = new VsmfieldsConfig.Documenttype.Index.Builder(); + indB.name(e.getValue().getName()); + for (String field : e.getValue().getFieldNames()) { + indB.field(new VsmfieldsConfig.Documenttype.Index.Field.Builder().name(field)); + } + dtB.index(indB); + } + return dtB; + } + + public String getName() { return name; } + + public void addIndexField(String indexName, String fieldName) { + FieldSet fs = fieldSets.get(indexName); + if (fs == null) { + fs = new FieldSet(indexName); + fieldSets.put(indexName, fs); + } + fs.addFieldName(fieldName); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java new file mode 100644 index 00000000000..30ae9c97268 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java @@ -0,0 +1,109 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; + +import java.util.*; + +/** + * Vertical streaming matcher summary specification + * + * @author bratseth + */ +public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { + + private final Map> summaryMap = new java.util.LinkedHashMap<>(1); + + public VsmSummary(Schema schema) { + derive(schema); + } + + @Override + protected void derive(Schema schema) { + // Use the default class, as it is the superset + derive(schema, schema.getSummary("default")); + } + + private void derive(Schema schema, DocumentSummary documentSummary) { + if (documentSummary == null) return; + for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { + List from = toStringList(summaryField.sourceIterator()); + + if (doMapField(schema, summaryField)) { + SDField sdField = schema.getConcreteField(summaryField.getName()); + if (sdField != null && GeoPos.isAnyPos(sdField)) { + summaryMap.put(summaryField, Collections.singletonList(summaryField.getName())); + } else { + summaryMap.put(summaryField, from); + } + } + } + } + + /** + * Don't include field in map if sources are the same as the struct sub fields for the SDField. + * But do map if not all do summarying. + * Don't map if not struct either. + * @param summaryField a {@link SummaryField} + */ + private boolean doMapField(Schema schema, SummaryField summaryField) { + SDField sdField = schema.getConcreteField(summaryField.getName()); + SDDocumentType document = schema.getDocument(); + if (sdField==null || ((document != null) && (document.getField(summaryField.getName()) == sdField))) { + return true; + } + if (summaryField.getVsmCommand().equals(SummaryField.VsmCommand.FLATTENJUNIPER)) { + return true; + } + if (!sdField.usesStructOrMap()) { + return !(sdField.getName().equals(summaryField.getName())); + } + if (summaryField.getSourceCount()==sdField.getStructFields().size()) { + for (SummaryField.Source source : summaryField.getSources()) { + if (!sdField.getStructFields().contains(new SDField(schema.getDocument(), source.getName(), sdField.getDataType()))) { // equals() uses just name + return true; + } + if (sdField.getStructField(source.getName())!=null && !sdField.getStructField(source.getName()).doesSummarying()) { + return true; + } + } + // The sources in the summary field are the same as the sub-fields in the SD field. + // All sub fields do summarying. + // Don't map. + return false; + } + return true; + } + + private List toStringList(Iterator i) { + List ret = new ArrayList<>(); + while (i.hasNext()) { + ret.add(i.next().getName()); + } + return ret; + } + + @Override + public String getDerivedName() { + return "vsmsummary"; + } + + @Override + public void getConfig(VsmsummaryConfig.Builder vB) { + for (Map.Entry> entry : summaryMap.entrySet()) { + VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName()); + for (String field : entry.getValue()) { + fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field)); + } + fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString())); + vB.fieldmap(fmB); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/package-info.java b/config-model/src/main/java/com/yahoo/schema/derived/package-info.java new file mode 100644 index 00000000000..370617ac6cc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.schema.derived; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/validation/IndexStructureValidator.java b/config-model/src/main/java/com/yahoo/schema/derived/validation/IndexStructureValidator.java new file mode 100644 index 00000000000..512d9f742bf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/validation/IndexStructureValidator.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived.validation; + +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.IndexingScript; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; + +/** + * @author Mathias M Lidal + */ +public class IndexStructureValidator extends Validator { + + public IndexStructureValidator(DerivedConfiguration config, Schema schema) { + super(config, schema); + } + + public void validate() { + IndexingScript script = config.getIndexingScript(); + for (Expression exp : script.expressions()) { + new OutputVisitor(schema.getDocument(), exp).visit(exp); + } + } + + private static class OutputVisitor extends ExpressionVisitor { + + final SDDocumentType docType; + final Expression exp; + + public OutputVisitor(SDDocumentType docType, Expression exp) { + this.docType = docType; + this.exp = exp; + } + + @Override + protected void doVisit(Expression exp) { + if (!(exp instanceof OutputExpression)) return; + + String fieldName = ((OutputExpression)exp).getFieldName(); + if (docType.getField(fieldName) != null) return; + + throw new IllegalArgumentException("Indexing expression '" + this.exp + "' refers to field '" + + fieldName + "' which does not exist in the index structure."); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/validation/Validation.java b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validation.java new file mode 100644 index 00000000000..dba4dce49f0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validation.java @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived.validation; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; + +public class Validation { + + public static void validate(DerivedConfiguration config, Schema schema) { + new IndexStructureValidator(config, schema).validate(); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validator.java new file mode 100644 index 00000000000..bf0f007841c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validator.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived.validation; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; + +/** + * @author mathiasm + */ +public abstract class Validator { + + protected DerivedConfiguration config; + protected Schema schema; + + protected Validator(DerivedConfiguration config, Schema schema) { + this.config = config; + this.schema = schema; + } + + public abstract void validate(); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java new file mode 100644 index 00000000000..f2279a52855 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -0,0 +1,435 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.Float16FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.TensorType; + +import java.io.Serializable; +import java.util.function.Supplier; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * A search-time document attribute (per-document in-memory value). + * This belongs to the field defining the attribute. + * + * @author bratseth + */ +public final class Attribute implements Cloneable, Serializable { + + public enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT, HAMMING } + + // Remember to change hashCode and equals when you add new fields + + private String name; + + private Type type; + private CollectionType collectionType; + + private boolean removeIfZero = false; + private boolean createIfNonExistent = false; + private boolean enableBitVectors = false; + private boolean enableOnlyBitVector = false; + + private boolean fastRank = false; + private boolean fastSearch = false; + private boolean fastAccess = false; + private boolean huge = false; + private boolean mutable = false; + private boolean paged = false; + private int arity = BooleanIndexDefinition.DEFAULT_ARITY; + private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND; + private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND; + private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD; + + /** This is set if the type of this is TENSOR */ + private Optional tensorType = Optional.empty(); + + /** This is set if the type of this is REFERENCE */ + private final Optional referenceDocumentType; + + private Optional distanceMetric = Optional.empty(); + + private Optional hnswIndexParams = Optional.empty(); + + private boolean isPosition = false; + private final Sorting sorting = new Sorting(); + + /** The aliases for this attribute */ + private final Set aliases = new LinkedHashSet<>(); + + private Dictionary dictionary = null; + private Case casing = Case.UNCASED; + + /** + * True if this attribute should be returned during first pass of search. + * Null means make the default decision for this kind of attribute + */ + private Boolean prefetch = null; + + /** The attribute type enumeration */ + public enum Type { + BYTE("byte", "INT8"), + SHORT("short", "INT16"), + INTEGER("integer", "INT32"), + LONG("long", "INT64"), + FLOAT16("float16", "FLOAT16"), + FLOAT("float", "FLOAT"), + DOUBLE("double", "DOUBLE"), + STRING("string", "STRING"), + BOOL("bool", "BOOL"), + PREDICATE("predicate", "PREDICATE"), + TENSOR("tensor", "TENSOR"), + REFERENCE("reference", "REFERENCE"); + + private final String myName; // different from what name() returns. + private final String exportAttributeTypeName; + + Type(String name, String exportAttributeTypeName) { + this.myName=name; + this.exportAttributeTypeName = exportAttributeTypeName; + } + + public String getName() { return myName; } + public String getExportAttributeTypeName() { return exportAttributeTypeName; } + + @Override + public String toString() { + return "type: " + myName; + } + } + + /** The attribute collection type enumeration */ + public enum CollectionType { + + SINGLE("SINGLE"), + ARRAY("ARRAY"), + WEIGHTEDSET ("WEIGHTEDSET"); + + private final String name; + + CollectionType(String name) { + this.name=name; + } + + public String getName() { return name; } + + @Override + public String toString() { + return "collectiontype: " + name; + } + + } + + /** Creates an attribute with default settings */ + public Attribute(String name, DataType fieldType) { + this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); + setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false); + setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false); + } + + public Attribute(String name, Type type, CollectionType collectionType) { + this(name, type, collectionType, Optional.empty(), Optional.empty()); + } + + public Attribute(String name, + Type type, + CollectionType collectionType, + Optional tensorType, + Optional referenceDocumentType) { + this.name=name; + setType(type); + setCollectionType(collectionType); + this.tensorType = tensorType; + this.referenceDocumentType = referenceDocumentType; + } + + public Attribute convertToArray() { + Attribute result = clone(); + result.collectionType = CollectionType.ARRAY; + return result; + } + + /** + *

Returns whether this attribute should be included in the "attributeprefetch" summary + * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. + * + *

Single value attributes are prefetched by default if summary is true. + * Multi value attributes are not.

+ */ + public boolean isPrefetch() { + if (prefetch!=null) return prefetch.booleanValue(); + + if (CollectionType.SINGLE.equals(collectionType)) { + return true; + } + + return false; + } + + /** Returns the prefetch value of this, null if the default is used. */ + public Boolean getPrefetchValue() { return prefetch; } + + public boolean isRemoveIfZero() { return removeIfZero; } + public boolean isCreateIfNonExistent() { return createIfNonExistent; } + public boolean isEnabledBitVectors() { return enableBitVectors; } + public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; } + public boolean isFastSearch() { return fastSearch; } + public boolean isFastRank() { return fastRank; } + public boolean isFastAccess() { return fastAccess; } + public boolean isHuge() { return huge; } + public boolean isPaged() { return paged; } + public boolean isPosition() { return isPosition; } + public boolean isMutable() { return mutable; } + + public int arity() { return arity; } + public long lowerBound() { return lowerBound; } + public long upperBound() { return upperBound; } + public double densePostingListThreshold() { return densePostingListThreshold; } + public Optional tensorType() { return tensorType; } + public Optional referenceDocumentType() { return referenceDocumentType; } + + public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; + public DistanceMetric distanceMetric() { + return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); + } + public Optional hnswIndexParams() { return hnswIndexParams; } + + public Sorting getSorting() { return sorting; } + public Dictionary getDictionary() { return dictionary; } + public Case getCase() { return casing; } + + public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } + public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; } + + /** + * Sets whether this should be included in the "attributeprefetch" document summary. + * True or false to override default, null to use default + */ + public void setPrefetch(Boolean prefetch) { this.prefetch = prefetch; } + public void setEnableBitVectors(boolean enableBitVectors) { this.enableBitVectors = enableBitVectors; } + public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; } + public void setFastRank(boolean value) { + Supplier badGen = () -> new IllegalArgumentException("fast-rank is only valid for tensor attributes, invalid for: "+this); + var tt = tensorType.orElseThrow(badGen); + for (var dim : tt.dimensions()) { + if (dim.isMapped()) { + this.fastRank = value; + return; + } + } + throw badGen.get(); + } + public void setFastSearch(boolean fastSearch) { this.fastSearch = fastSearch; } + public void setHuge(boolean huge) { this.huge = huge; } + public void setPaged(boolean paged) { this.paged = paged; } + public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; } + public void setPosition(boolean position) { this.isPosition = position; } + public void setMutable(boolean mutable) { this.mutable = mutable; } + public void setArity(int arity) { this.arity = arity; } + public void setLowerBound(long lowerBound) { this.lowerBound = lowerBound; } + public void setUpperBound(long upperBound) { this.upperBound = upperBound; } + public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } + public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setDistanceMetric(DistanceMetric metric) { this.distanceMetric = Optional.of(metric); } + public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } + public void setDictionary(Dictionary dictionary) { this.dictionary = dictionary; } + public void setCase(Case casing) { this.casing = casing; } + + public String getName() { return name; } + public Type getType() { return type; } + public CollectionType getCollectionType() { return collectionType; } + + public void setName(String name) { this.name=name; } + private void setType(Type type) { this.type=type; } + public void setCollectionType(CollectionType type) { this.collectionType=type; } + + /** Converts to the right attribute type from a field datatype */ + public static Type convertDataType(DataType fieldType) { + if (fieldType instanceof NewDocumentReferenceDataType) { + return Type.REFERENCE; + } else if (fieldType instanceof CollectionDataType) { + return convertDataType(((CollectionDataType) fieldType).getNestedType()); + } + FieldValue fval = fieldType.createFieldValue(); + if (fval instanceof StringFieldValue) { + return Type.STRING; + } else if (fval instanceof IntegerFieldValue) { + return Type.INTEGER; + } else if (fval instanceof LongFieldValue) { + return Type.LONG; + } else if (fval instanceof FloatFieldValue) { + return Type.FLOAT; + } else if (fval instanceof DoubleFieldValue) { + return Type.DOUBLE; + } else if (fval instanceof BoolFieldValue) { + return Type.BOOL; + } else if (fval instanceof Float16FieldValue) { + return Type.FLOAT16; + } else if (fval instanceof ByteFieldValue) { + return Type.BYTE; + } else if (fval instanceof Raw) { + return Type.BYTE; + } else if (fval instanceof PredicateFieldValue) { + return Type.PREDICATE; + } else if (fval instanceof TensorFieldValue) { + return Type.TENSOR; + } else { + throw new IllegalArgumentException("Don't know which attribute type to convert " + + fieldType + " [" + fieldType.getClass() + "] to"); + } + } + + /** Converts to the right attribute type from a field datatype */ + private static CollectionType convertCollectionType(DataType fieldType) { + if (fieldType instanceof ArrayDataType) { + return CollectionType.ARRAY; + } else if (fieldType instanceof WeightedSetDataType) { + return CollectionType.WEIGHTEDSET; + } else if (fieldType instanceof TensorDataType) { + return CollectionType.SINGLE; + } else if (fieldType instanceof PrimitiveDataType) { + return CollectionType.SINGLE; + } else if (fieldType instanceof NewDocumentReferenceDataType) { + return CollectionType.SINGLE; + } else { + throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType"); + } + } + + private static Optional convertTensorType(DataType fieldType) { + if ( ! ( fieldType instanceof TensorDataType)) return Optional.empty(); + return Optional.of(((TensorDataType)fieldType).getTensorType()); + } + + private static Optional convertTargetType(DataType fieldType) { + return Optional.of(fieldType) + .filter(NewDocumentReferenceDataType.class::isInstance) + .map(NewDocumentReferenceDataType.class::cast) + .map(NewDocumentReferenceDataType::getTargetType); + } + + /** Converts to the right field type from an attribute type */ + private DataType toDataType(Type attributeType) { + switch (attributeType) { + case STRING : return DataType.STRING; + case INTEGER: return DataType.INT; + case LONG: return DataType.LONG; + case FLOAT16: return DataType.FLOAT16; + case FLOAT: return DataType.FLOAT; + case DOUBLE: return DataType.DOUBLE; + case BOOL: return DataType.BOOL; + case BYTE: return DataType.BYTE; + case PREDICATE: return DataType.PREDICATE; + case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new)); + case REFERENCE: return createReferenceDataType(); + default: throw new IllegalArgumentException("Unknown attribute type " + attributeType); + } + } + + @SuppressWarnings("deprecation") + private DataType createReferenceDataType() { + if (referenceDocumentType.isEmpty()) { + throw new IllegalStateException("Referenced document type is not set"); + } + StructuredDataType type = referenceDocumentType.get(); + if (type instanceof DocumentType) { + return new NewDocumentReferenceDataType((DocumentType) type); + } else { + return NewDocumentReferenceDataType.forDocumentName(type.getName()); + } + } + + public DataType getDataType() { + DataType dataType = toDataType(type); + if (collectionType == Attribute.CollectionType.ARRAY) { + return DataType.getArray(dataType); + } else if (collectionType == Attribute.CollectionType.WEIGHTEDSET) { + return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero); + } else { + return dataType; + } + } + + @Override + public int hashCode() { + return Objects.hash( + name, type, collectionType, sorting, dictionary, isPrefetch(), fastAccess, removeIfZero, + createIfNonExistent, isPosition, huge, mutable, paged, enableBitVectors, enableOnlyBitVector, + tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); + } + + @Override + public boolean equals(Object object) { + if (! (object instanceof Attribute)) return false; + + Attribute other = (Attribute)object; + if (!this.name.equals(other.name)) return false; + return isCompatible(other); + } + + /** Returns whether these attributes describes the same entity, even if they have different names */ + public boolean isCompatible(Attribute other) { + if (! this.type.equals(other.type)) return false; + if (! this.collectionType.equals(other.collectionType)) return false; + if (this.isPrefetch() != other.isPrefetch()) return false; + if (this.removeIfZero != other.removeIfZero) return false; + if (this.createIfNonExistent != other.createIfNonExistent) return false; + if (this.enableBitVectors != other.enableBitVectors) return false; + if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; + if (this.fastSearch != other.fastSearch) return false; + if (this.huge != other.huge) return false; + if (this.mutable != other.mutable) return false; + if (this.paged != other.paged) return false; + if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(dictionary, other.dictionary)) return false; + if (! Objects.equals(tensorType, other.tensorType)) return false; + if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; + if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; + if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false; + + return true; + } + + @Override + public Attribute clone() { + try { + return (Attribute)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + @Override + public String toString() { + return "attribute '" + name + "' (" + type + ")"; + } + + public Set getAliases() { + return aliases; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java new file mode 100644 index 00000000000..8563d414c40 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java @@ -0,0 +1,84 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; + +/** + * Encapsulates values required for predicate fields. + * + * @author lesters + */ +public final class BooleanIndexDefinition { + + public static final int DEFAULT_ARITY = 8; + public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE; + public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE; + public static final double DEFAULT_DENSE_POSTING_LIST_THRESHOLD = 0.4; + + private final OptionalInt arity; // mandatory field value + private final OptionalLong lowerBound; + private final OptionalLong upperBound; + private final OptionalDouble densePostingListThreshold; + + public BooleanIndexDefinition(Optional arity, + Optional lowerBound, + Optional upperBound, + Optional densePLT) + { + this.arity = arity.isPresent() ? OptionalInt.of(arity.get()) : OptionalInt.empty(); + this.lowerBound = lowerBound.isPresent() ? OptionalLong.of(lowerBound.get()) : OptionalLong.empty(); + this.upperBound = upperBound.isPresent() ? OptionalLong.of(upperBound.get()) : OptionalLong.empty(); + this.densePostingListThreshold = densePLT.isPresent() ? OptionalDouble.of(densePLT.get()) : OptionalDouble.empty(); + } + + public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound, + OptionalLong upperBound, OptionalDouble densePostingListThreshold) { + this.arity = arity; + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.densePostingListThreshold = densePostingListThreshold; + } + + public int getArity() { + return arity.getAsInt(); + } + + public boolean hasArity() { + return arity.isPresent(); + } + + public long getLowerBound() { + return lowerBound.orElse(DEFAULT_LOWER_BOUND); + } + + public boolean hasLowerBound() { + return lowerBound.isPresent(); + } + + public long getUpperBound() { + return upperBound.orElse(DEFAULT_UPPER_BOUND); + } + + public boolean hasUpperBound() { + return upperBound.isPresent(); + } + + public double getDensePostingListThreshold() { + return densePostingListThreshold.orElse(DEFAULT_DENSE_POSTING_LIST_THRESHOLD); + } + + public boolean hasDensePostingListThreshold() { + return densePostingListThreshold.isPresent(); + } + + @Override + public String toString() { + return "BooleanIndexDefinition [arity=" + arity + ", lowerBound=" + + lowerBound + ", upperBound=" + upperBound + ", densePostingListThreshold=" + + densePostingListThreshold + "]"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Case.java b/config-model/src/main/java/com/yahoo/schema/document/Case.java new file mode 100644 index 00000000000..443f67940a0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Case.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * Describes if items should preserve lower/upper case, or shall be uncased + * which normally mean they are all normalized to lowercase. + * @author baldersheim + */ +public enum Case { + CASED("cased"), + UNCASED("uncased"); + private String name; + Case(String name) { this.name = name; } + public String getName() { return name;} +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java new file mode 100644 index 00000000000..993bf16405a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java @@ -0,0 +1,123 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; + +/** + * Utils used to check whether a complex field supports being represented as struct field attributes. + * + * Currently we support: + * - array of simple struct + * - map of primitive type to simple struct + * - map of primitive type to primitive type + * + * A simple struct can contain fields of any type, but only fields of primitive type can be defined as + * struct field attributes in the complex field using the simple struct. + * + * @author geirst + */ +public class ComplexAttributeFieldUtils { + + public static boolean isSupportedComplexField(ImmutableSDField field) { + return (isArrayOfSimpleStruct(field) || + isMapOfSimpleStruct(field) || + isMapOfPrimitiveType(field)); + } + + public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { + if (field.getDataType() instanceof ArrayDataType) { + ArrayDataType arrayType = (ArrayDataType)field.getDataType(); + return isStructWithPrimitiveStructFieldAttributes(arrayType.getNestedType(), field); + } else { + return false; + } + } + + public static boolean isMapOfSimpleStruct(ImmutableSDField field) { + if (field.getDataType() instanceof MapDataType) { + MapDataType mapType = (MapDataType)field.getDataType(); + return isPrimitiveType(mapType.getKeyType()) && + isStructWithPrimitiveStructFieldAttributes(mapType.getValueType(), + field.getStructField("value")); + } else { + return false; + } + } + + public static boolean isMapOfPrimitiveType(ImmutableSDField field) { + if (field.getDataType() instanceof MapDataType) { + MapDataType mapType = (MapDataType)field.getDataType(); + return isPrimitiveType(mapType.getKeyType()) && + isPrimitiveType(mapType.getValueType()); + } else { + return false; + } + } + + private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field) { + if (type instanceof StructDataType && ! GeoPos.isPos(type)) { + for (ImmutableSDField structField : field.getStructFields()) { + Attribute attribute = structField.getAttributes().get(structField.getName()); + if (attribute != null) { + if (!isPrimitiveType(attribute)) { + return false; + } + } else if (structField.wasConfiguredToDoAttributing()) { + if (!isPrimitiveType(structField.getDataType())) { + return false; + } + } + } + return true; + } else { + return false; + } + } + + public static boolean isPrimitiveType(Attribute attribute) { + return attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) && + isPrimitiveType(attribute.getDataType()); + } + + public static boolean isPrimitiveType(DataType dataType) { + return dataType.equals(DataType.BYTE) || + dataType.equals(DataType.INT) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.STRING); + } + + public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { + if (isArrayOfSimpleStruct(field)) { + return hasOnlyStructFieldAttributes(field); + } else if (isMapOfSimpleStruct(field)) { + return hasSingleAttribute(field.getStructField("key")) && + hasOnlyStructFieldAttributes(field.getStructField("value")); + } else if (isMapOfPrimitiveType(field)) { + return hasSingleAttribute(field.getStructField("key")) && + hasSingleAttribute(field.getStructField("value")); + } + return false; + } + + private static boolean hasOnlyStructFieldAttributes(ImmutableSDField field) { + for (ImmutableSDField structField : field.getStructFields()) { + if (!hasSingleAttribute(structField)) { + return false; + } + } + return true; + } + + private static boolean hasSingleAttribute(ImmutableSDField field) { + if (field.getAttributes().size() != 1) { + return false; + } + return (field.getAttributes().get(field.getName()) != null); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java b/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java new file mode 100644 index 00000000000..4744547c778 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.document; + +/** + * Represents settings for dictionary control + * + * @author baldersheim + */ +public class Dictionary { + public enum Type { BTREE, HASH, BTREE_AND_HASH }; + private Type type = null; + private Case casing= null; + + public void updateType(Type type) { + if (this.type == null) { + this.type = type; + } else if ((this.type == Type.BTREE) && (type == Type.HASH)) { + this.type = Type.BTREE_AND_HASH; + } else if ((this.type == Type.HASH) && (type == Type.BTREE)) { + this.type = Type.BTREE_AND_HASH; + } else { + throw new IllegalArgumentException("Can not combine previous dictionary setting " + this.type + + " with current " + type); + } + } + public void updateMatch(Case casing) { + if (this.casing != null) { + throw new IllegalArgumentException("dictionary match mode has already been set to " + this.casing); + } + this.casing = casing; + } + public Type getType() { return (type != null) ? type : Type.BTREE; } + public Case getMatch() { return (casing != null) ? casing : Case.UNCASED; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java b/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java new file mode 100644 index 00000000000..e62e784d7b3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.TreeSet; + +import static java.util.Comparator.comparing; + +/** + * Searchable collection of fields. + * + * @author baldersheim + */ +public class FieldSet { + + private final String name; + private final Set queryCommands = new LinkedHashSet<>(); + private final Set fieldNames = new TreeSet<>(); + private final Set fields = new TreeSet<>(comparing(ImmutableSDField::asField)); + private Matching matching = null; + + public FieldSet(String name) { this.name = name; } + public String getName() { return name; } + public FieldSet addFieldName(String field) { fieldNames.add(field); return this; } + public Set getFieldNames() { return fieldNames; } + public Set fields() { return fields; } + public Set queryCommands() { return queryCommands; } + + public void setMatching(Matching matching) { + this.matching = matching; + } + + public Matching getMatching() { + return matching; + } + + @Override + public String toString() { return "fieldset '" + name + "'"; } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java b/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java new file mode 100644 index 00000000000..829555d88c6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; + +/** + * Common utilities for recognizing fields with the built-in "position" datatype, + * possibly in array form. + * @author arnej + */ +public class GeoPos { + static public boolean isPos(DataType type) { + return PositionDataType.INSTANCE.equals(type); + } + static public boolean isPosArray(DataType type) { + return DataType.getArray(PositionDataType.INSTANCE).equals(type); + } + static public boolean isAnyPos(DataType type) { + return isPos(type) || isPosArray(type); + } + + static public boolean isPos(ImmutableSDField field) { return isPos(field.getDataType()); } + static public boolean isPosArray(ImmutableSDField field) { return isPosArray(field.getDataType()); } + static public boolean isAnyPos(ImmutableSDField field) { return isAnyPos(field.getDataType()); } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java new file mode 100644 index 00000000000..cc427356c78 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Optional; + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. + * + * @author geirst + */ +public class HnswIndexParams { + + public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; + public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; + + private final Optional maxLinksPerNode; + private final Optional neighborsToExploreAtInsert; + private final Optional multiThreadedIndexing; + + public static class Builder { + private Optional maxLinksPerNode = Optional.empty(); + private Optional neighborsToExploreAtInsert = Optional.empty(); + private Optional multiThreadedIndexing = Optional.empty(); + + public void setMaxLinksPerNode(int value) { + maxLinksPerNode = Optional.of(value); + } + public void setNeighborsToExploreAtInsert(int value) { + neighborsToExploreAtInsert = Optional.of(value); + } + public void setMultiThreadedIndexing(boolean value) { + multiThreadedIndexing = Optional.of(value); + } + public HnswIndexParams build() { + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert, multiThreadedIndexing); + } + } + + public HnswIndexParams() { + this.maxLinksPerNode = Optional.empty(); + this.neighborsToExploreAtInsert = Optional.empty(); + this.multiThreadedIndexing = Optional.empty(); + } + + public HnswIndexParams(Optional maxLinksPerNode, + Optional neighborsToExploreAtInsert, + Optional multiThreadedIndexing) { + this.maxLinksPerNode = maxLinksPerNode; + this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; + this.multiThreadedIndexing = multiThreadedIndexing; + } + + /** + * Creates a new instance where values from the given parameter instance are used where they are present, + * otherwise we use values from this. + */ + public HnswIndexParams overrideFrom(Optional other) { + if (! other.isPresent()) return this; + HnswIndexParams rhs = other.get(); + return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode), + rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert), + rhs.multiThreadedIndexing.or(() -> multiThreadedIndexing)); + } + + public int maxLinksPerNode() { + return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); + } + + public int neighborsToExploreAtInsert() { + return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); + } + + public boolean multiThreadedIndexing() { + return multiThreadedIndexing.orElse(true); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java new file mode 100644 index 00000000000..553b5b4d940 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Collection; + +import static java.util.stream.Collectors.toList; + +/** + * Wraps {@link ImportedComplexField} as {@link ImmutableSDField}. + */ +public class ImmutableImportedComplexSDField extends ImmutableImportedSDField { + private final ImportedComplexField importedComplexField; + + public ImmutableImportedComplexSDField(ImportedComplexField importedField) { + super(importedField); + importedComplexField = importedField; + } + + @Override + public ImmutableSDField getStructField(String name) { + ImportedField field = importedComplexField.getNestedField(name); + return (field != null) ? field.asImmutableSDField() : null; + } + + @Override + public Collection getStructFields() { + return importedComplexField.getNestedFields().stream().map(field -> field.asImmutableSDField()).collect(toList()); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java new file mode 100644 index 00000000000..335942de99d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java @@ -0,0 +1,218 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Wraps {@link ImportedField} as {@link ImmutableSDField}. + * Methods that are not meaningful or relevant for imported fields will throw {@link UnsupportedOperationException}. + * + * @author bjorncs + */ +public class ImmutableImportedSDField implements ImmutableSDField { + + private final ImportedField importedField; + + ImmutableImportedSDField(ImportedField importedField) { + this.importedField = importedField; + } + + public ImportedField getImportedField() { + return importedField; + } + + @Override + public boolean containsExpression(Class searchFor) { + throw createUnsupportedException(searchFor.getSimpleName()); + } + + @Override + public boolean doesAttributing() { + return importedField.targetField().doesAttributing(); + } + + @Override + public boolean doesIndexing() { + return importedField.targetField().doesIndexing(); + } + + @Override + public boolean doesLowerCasing() { + return importedField.targetField().doesLowerCasing(); + } + + @Override + public boolean isExtraField() { + return false; + } + + @Override + public boolean isImportedField() { + return true; + } + + @Override + public boolean isIndexStructureField() { + return importedField.targetField().isIndexStructureField(); + } + + @Override + public boolean hasIndex() { + return importedField.targetField().hasIndex(); + } + + @Override + public boolean usesStructOrMap() { + return importedField.targetField().usesStructOrMap(); + } + + @Override + public boolean wasConfiguredToDoAttributing() { + return importedField.targetField().wasConfiguredToDoAttributing(); + } + + @Override + public DataType getDataType() { + return importedField.targetField().getDataType(); + } + + @Override + public SummaryField getSummaryField(String name) { + return importedField.targetField().getSummaryField(name); + } + + @Override + public Index getIndex(String name) { + if ( ! importedField.fieldName().equals(name)) { + throw new IllegalArgumentException("Getting an index (" + name + ") with different name than the imported field (" + + importedField.fieldName() + ") is not supported"); + } + String targetIndexName = importedField.targetField().getName(); + return importedField.targetField().getIndex(targetIndexName); + } + + @Override + public List getQueryCommands() { + return importedField.targetField().getQueryCommands(); + } + + @Override + public Map getAttributes() { + return importedField.targetField().getAttributes(); + } + + @Override + public Attribute getAttribute() { return importedField.targetField().getAttribute(); } + + @Override + public Map getAliasToName() { + return Collections.emptyMap(); + } + + @Override + public ScriptExpression getIndexingScript() { + throw createUnsupportedException("indexing"); + } + + @Override + public Matching getMatching() { + return importedField.targetField().getMatching(); + } + + @Override + public NormalizeLevel getNormalizing() { + return importedField.targetField().getNormalizing(); + } + + @Override + public ImmutableSDField getStructField(String name) { + throw createUnsupportedException("struct"); + } + + @Override + public Collection getStructFields() { + throw createUnsupportedException("struct"); + } + + @Override + public Stemming getStemming() { + return importedField.targetField().getStemming(); + } + + @Override + public Stemming getStemming(Schema schema) { + throw createUnsupportedException("stemming"); + } + + @Override + public Ranking getRanking() { + throw createUnsupportedException("ranking"); + } + + @Override + public Map getSummaryFields() { + throw createUnsupportedException("summary fields"); + } + + @Override + public String getName() { + return importedField.fieldName(); // Name of the imported field, not the target field + } + + @Override + public int getWeight() { + return importedField.targetField().getWeight(); + } + + @Override + public int getLiteralBoost() { + return importedField.targetField().getLiteralBoost(); + } + + @Override + public RankType getRankType() { + return importedField.targetField().getRankType(); + } + + @Override + public Map getIndices() { + return importedField.targetField().getIndices(); + } + + @Override + public boolean existsIndex(String name) { + return importedField.targetField().existsIndex(name); + } + + /** + * Returns a field representation of the imported field. + * Changes to the returned instance are not propagated back to the underlying imported field! + */ + @Override + public Field asField() { + return new Field( + importedField.fieldName(), + importedField.targetField().getDataType()); + } + + private static UnsupportedOperationException createUnsupportedException(String aspect) { + return new UnsupportedOperationException("'" + aspect + "' is not meaningful or relevant for an imported field."); + } + + @Override + public boolean hasFullIndexingDocprocRights() { + return importedField.targetField().hasFullIndexingDocprocRights(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java new file mode 100644 index 00000000000..44e442811ba --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * An interface containing the non-mutating methods of {@link SDField}. + * For description of the methods see {@link SDField}. + * + * @author bjorncs + */ +public interface ImmutableSDField { + + boolean containsExpression(Class searchFor); + + boolean doesAttributing(); + + boolean doesIndexing(); + + boolean doesLowerCasing(); + + boolean isExtraField(); + + boolean isImportedField(); + + boolean isIndexStructureField(); + + boolean usesStructOrMap(); + + /** + * Whether this field at some time was configured to do attributing. + * + * This function can typically return a different value than doesAttributing(), + * which uses the final state of the underlying indexing script instead. + */ + boolean wasConfiguredToDoAttributing(); + + DataType getDataType(); + + Index getIndex(String name); + + List getQueryCommands(); + + Map getAttributes(); + + Attribute getAttribute(); + + Map getAliasToName(); + + ScriptExpression getIndexingScript(); + + Matching getMatching(); + + NormalizeLevel getNormalizing(); + + ImmutableSDField getStructField(String name); + + Collection getStructFields(); + + Stemming getStemming(); + + Stemming getStemming(Schema schema); + + Ranking getRanking(); + + String getName(); + + Map getSummaryFields(); + + /** Returns a {@link Field} representation (which is sadly not immutable) */ + Field asField(); + + boolean hasFullIndexingDocprocRights(); + int getWeight(); + int getLiteralBoost(); + RankType getRankType(); + Map getIndices(); + boolean existsIndex(String name); + SummaryField getSummaryField(String name); + boolean hasIndex(); +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java new file mode 100644 index 00000000000..10c400bc4d6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.DocumentReference; + +import java.util.Collection; +import java.util.Map; + +/** + * A complex field that is imported from a concrete field in a referenced document type and given an alias name. + */ +public class ImportedComplexField extends ImportedField { + + private Map nestedFields; + + public ImportedComplexField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { + super(fieldName, reference, targetField); + nestedFields = new java.util.LinkedHashMap<>(0); + } + + @Override + public ImmutableSDField asImmutableSDField() { + return new ImmutableImportedComplexSDField(this); + } + + public void addNestedField(ImportedField importedField) { + String prefix = fieldName() + "."; + assert(importedField.fieldName().substring(0, prefix.length()).equals(prefix)); + String suffix = importedField.fieldName().substring(prefix.length()); + nestedFields.put(suffix, importedField); + } + + public Collection getNestedFields() { + return nestedFields.values(); + } + + public ImportedField getNestedField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + ImportedField superField = nestedFields.get(superFieldName); + if (superField != null && superField instanceof ImportedComplexField) { + return ((ImportedComplexField)superField).getNestedField(subFieldName); + } + return null; + } + return nestedFields.get(name); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java new file mode 100644 index 00000000000..50f8591bbce --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.DocumentReference; + +/** + * A field that is imported from a concrete field in a referenced document type and given an alias name. + * + * @author geirst + */ +public abstract class ImportedField { + + private final String fieldName; + private final DocumentReference reference; + private final ImmutableSDField targetField; + + public ImportedField(String fieldName, + DocumentReference reference, + ImmutableSDField targetField) { + this.fieldName = fieldName; + this.reference = reference; + this.targetField = targetField; + } + + public String fieldName() { + return fieldName; + } + + public DocumentReference reference() { + return reference; + } + + public ImmutableSDField targetField() { + return targetField; + } + + public abstract ImmutableSDField asImmutableSDField(); +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java new file mode 100644 index 00000000000..f6654896fae --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.util.Collections; +import java.util.Map; + +/** + * A set of fields that are imported from concrete fields in referenced document types. + * + * @author geirst + */ +public class ImportedFields { + + private final Map fields; + + public ImportedFields(Map fields) { + this.fields = fields; + } + + public Map fields() { + return Collections.unmodifiableMap(fields); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java new file mode 100644 index 00000000000..244135ecc10 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.DocumentReference; + +/** + * A simple field that is imported from a concrete field in a referenced document type and given an alias name. + */ +public class ImportedSimpleField extends ImportedField { + public ImportedSimpleField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { + super(fieldName, reference, targetField); + } + + @Override + public ImmutableSDField asImmutableSDField() { + return new ImmutableImportedSDField(this); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java b/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java new file mode 100644 index 00000000000..8556fe491d0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** Which match algorithm is used by this matching setup */ + +public enum MatchAlgorithm { + NORMAL("normal"), + PREFIX("prefix"), + SUBSTRING("substring"), + SUFFIX("suffix"); + + private String name; + MatchAlgorithm(String name) { this.name = name; } + + public String getName() { return name; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/MatchType.java b/config-model/src/main/java/com/yahoo/schema/document/MatchType.java new file mode 100644 index 00000000000..d2088e71282 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/MatchType.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +public enum MatchType { + TEXT("text"), + WORD("word"), + EXACT("exact"), + GRAM("gram"); + + private String name; + MatchType(String name) { this.name = name; } + + public String getName() { return name; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java new file mode 100644 index 00000000000..f70f31be0bd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -0,0 +1,141 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.io.Serializable; + +/** + * Defines how a field should be matched. + * Matching objects can be compared based on their content, but they are not immutable. + * + * @author bratseth + */ +public class Matching implements Cloneable, Serializable { + + public static final MatchType defaultType = MatchType.TEXT; + + private MatchType type = MatchType.TEXT; + private Case casing = Case.UNCASED; + + /** The basic match algorithm */ + private MatchAlgorithm algorithm = MatchAlgorithm.NORMAL; + + private boolean typeUserSet = false; + + private boolean algorithmUserSet = false; + + /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ + private int gramSize=-1; + + /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ + private Integer maxLength; + + private String exactMatchTerminator=null; + + /** Creates a matching of type "text" */ + public Matching() {} + + public Matching(MatchType type) { + this.type = type; + } + + public MatchType getType() { return type; } + public Case getCase() { return casing; } + + public void setType(MatchType type) { + this.type = type; + typeUserSet = true; + } + + public void setCase(Case casing) { this.casing = casing; } + + public Integer maxLength() { return maxLength; } + public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } + public boolean isTypeUserSet() { return typeUserSet; } + + public MatchAlgorithm getAlgorithm() { return algorithm; } + + public void setAlgorithm(MatchAlgorithm algorithm) { + this.algorithm = algorithm; + algorithmUserSet = true; + } + + public boolean isAlgorithmUserSet() { return algorithmUserSet; } + + public boolean isPrefix() { return algorithm == MatchAlgorithm.PREFIX; } + + public boolean isSubstring() { return algorithm == MatchAlgorithm.SUBSTRING; } + + public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } + + /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ + public int getGramSize() { return gramSize; } + + public void setGramSize(int gramSize) { this.gramSize=gramSize; } + + /** + * Merge data from another matching object + */ + public void merge(Matching m) { + if (m == null) return; + if (m.isAlgorithmUserSet()) { + this.setAlgorithm(m.getAlgorithm()); + } + if (m.isTypeUserSet()) { + this.setType(m.getType()); + if (m.getType() == MatchType.GRAM) + gramSize = m.gramSize; + } + if (m.getExactMatchTerminator() != null) { + this.setExactMatchTerminator(m.getExactMatchTerminator()); + } + } + + /** + * If exact matching is used, this returns the terminator string + * which terminates an exact matched sequence in queries. If exact + * matching is not used, or no terminator is set, this is null + */ + public String getExactMatchTerminator() { return exactMatchTerminator; } + + /** + * Sets the terminator string which terminates an exact matched + * sequence in queries (used if type is EXACT). + */ + public void setExactMatchTerminator(String exactMatchTerminator) { + this.exactMatchTerminator = exactMatchTerminator; + } + + @Override + public String toString() { + return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + + "], [exact-terminator "+exactMatchTerminator+"]"; + } + + public Matching clone() { + try { + return (Matching)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + @Override + public boolean equals(Object o) { + if (! (o instanceof Matching)) return false; + + Matching other=(Matching)o; + if ( ! other.type.equals(this.type)) return false; + if ( ! other.algorithm.equals(this.algorithm)) return false; + if ( this.exactMatchTerminator == null && other.exactMatchTerminator != null) return false; + if ( this.exactMatchTerminator != null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) + return false; + if ( gramSize != other.gramSize) return false; + return true; + } + + @Override public int hashCode() { + return java.util.Objects.hash(type, algorithm, exactMatchTerminator, gramSize); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java new file mode 100644 index 00000000000..12880d0e1c8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * class representing the character normalization + * we want to do on query and indexed text. + * Levels are strict subsets, so doing accent + * removal means doing codepoint normalizing + * and case normalizing also. + */ +// TODO: Missing author +public class NormalizeLevel { + + /** + * The current levels are as follows: + * NONE: no changes to input text + * CODEPOINT: convert text into Unicode + * Normalization Form Compatibility Composition + * LOWERCASE: also convert text into lowercase letters + * ACCENT: do both above and remove accents on characters + */ + public enum Level { + NONE, CODEPOINT, LOWERCASE, ACCENT + } + + private boolean userSpecified = false; + private Level level = Level.ACCENT; + + /** + * Returns whether accents should be removed from text + */ + public boolean doRemoveAccents() { return level == Level.ACCENT; } + + /** + * Construct a default (full) normalizelevel, + */ + public NormalizeLevel() {} + + /** + * Construct for a specific level, possibly user specified + * + * @param level which level to use + * @param fromUser whether this was specified by the user + */ + public NormalizeLevel(Level level, boolean fromUser) { + this.level = level; + this.userSpecified = fromUser; + } + + /** + * Change the current level to CODEPOINT as inferred + * by other features' needs. If the current level + * was user specified it will not change; also this + * will not increase the level. + */ + public void inferCodepoint() { + if (userSpecified) { + // ignore inferred changes if user specified something + return; + } + // do not increase level + if (level != Level.NONE) level = Level.CODEPOINT; + } + + /** + * Change the current level to LOWERCASE as inferred + * by other features' needs. If the current level + * was user specified it will not change; also this + * will not increase the level. + */ + public void inferLowercase() { + if (userSpecified) { + // ignore inferred changes if user specified something + return; + } + // do not increase level + if (level == Level.NONE) return; + if (level == Level.CODEPOINT) return; + + level = Level.LOWERCASE; + } + + public Level getLevel() { + return level; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/RankType.java b/config-model/src/main/java/com/yahoo/schema/document/RankType.java new file mode 100644 index 00000000000..067c1e7f266 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/RankType.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * The rank type of a field. For now this is just a container of a string name. + * This class is immutable. + * + * @author bratseth + */ +public enum RankType { + + /** *implicit* default: No type has been set. */ + DEFAULT, + + // Rank types which can be set explicitly. These are defined for Vespa in NativeRankTypeDefinitionSet + IDENTITY, ABOUT, TAGS, EMPTY; + + @Override + public String toString() { + return "rank type " + name().toLowerCase(); + } + + /** + * Returns the rank type from a string, regardless of its case. + * + * @param rankTypeName a rank type name in any casing + * @return the rank type found + * @throws IllegalArgumentException if not found + */ + public static RankType fromString(String rankTypeName) { + try { + return RankType.valueOf(rankTypeName.toUpperCase()); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown rank type '" + rankTypeName + "'. Supported rank types are " + + "'identity', 'about', 'tags' and 'empty'."); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Ranking.java b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java new file mode 100644 index 00000000000..31fd9747e2d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.io.Serializable; + +/** + * The rank settings given in a rank clause in the search definition. + * + * @author Vegard Havdal + */ +public class Ranking implements Cloneable, Serializable { + + private boolean literal = false; + private boolean filter = false; + private boolean normal = false; + + /** + *

Returns whether literal (non-stemmed, non-normalized) forms of the words should + * be indexed in a separate index which is searched by a automatically added rank term + * during searches.

+ * + *

Default is false.

+ */ + public boolean isLiteral() { return literal; } + + public void setLiteral(boolean literal) { this.literal = literal; } + + /** + *

Returns whether this is a filter. Filters will only tell if they are matched or not, + * no detailed relevance information will be available about the match.

+ * + *

Matching a filter is much cheaper for the search engine than matching a regular field.

+ * + *

Default is false.

+ */ + public boolean isFilter() { return filter && !normal; } + + public void setFilter(boolean filter) { this.filter = filter; } + + /** Whether user has explicitly requested normal (non-filter) behavior */ + public boolean isNormal() { return normal; } + public void setNormal(boolean n) { this.normal = n; } + + /** Returns true if the given rank settings are the same */ + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Ranking)) return false; + + Ranking other=(Ranking)o; + if (this.filter != other.filter) return false; + if (this.literal != other.literal) return false; + if (this.normal != other.normal) return false; + return true; + } + + @Override + public int hashCode() { + return java.util.Objects.hash(filter, literal, normal); + } + + @Override + public String toString() { + return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]"; + } + + @Override + public Ranking clone() { + try { + return (Ranking)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error", e); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java new file mode 100644 index 00000000000..d300bd08bfd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -0,0 +1,347 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.AnnotationTypeRegistry; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * A document definition is a list of fields. Documents may inherit other documents, + * implicitly acquiring their fields as it's own. If a document is not set to inherit + * any document, it will always inherit the document "document.0". + * + * @author Thomas Gundersen + * @author bratseth + */ +public class SDDocumentType implements Cloneable, Serializable { + + public static final SDDocumentType VESPA_DOCUMENT; + private final Map inheritedTypes = new LinkedHashMap<>(); + private final Map ownedTypes = new LinkedHashMap<>(); + private final AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry(); + private DocumentType docType; + private DataType structType; + // The field sets here are set from the processing step in SD, + // to ensure that the full Search and this SDDocumentType is built first. + private FieldSets fieldSets; + // Document references + private Optional documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; + + static { + VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); + VESPA_DOCUMENT.addType(createSDDocumentType(PositionDataType.INSTANCE)); + } + + public SDDocumentType clone() throws CloneNotSupportedException { + SDDocumentType type = (SDDocumentType) super.clone(); + type.docType = docType.clone(); + type.inheritedTypes.putAll(inheritedTypes); + type.structType = structType; + // TODO this isn't complete; should it be..?! + return type; + } + + /** + * For adding structs defined in document scope + * + * @param dt the struct to add + * @return self, for chaining + */ + public SDDocumentType addType(SDDocumentType dt) { + NewDocumentType.Name name = new NewDocumentType.Name(dt.getName()); + if (getType(name) != null) + throw new IllegalArgumentException("Data type '" + name + "' has already been used."); + if (name.getName() == docType.getName()) + throw new IllegalArgumentException("Data type '" + name + "' can not have same name as its defining document."); + ownedTypes.put(name, dt); + return this; + } + public final SDDocumentType getOwnedType(String name) { + return getOwnedType(new NewDocumentType.Name(name)); + } + public SDDocumentType getOwnedType(DataTypeName name) { + return getOwnedType(name.getName()); + } + + public SDDocumentType getOwnedType(NewDocumentType.Name name) { + return ownedTypes.get(name); + } + + public final SDDocumentType getType(String name) { + return getType(new NewDocumentType.Name(name)); + } + + public SDDocumentType getType(NewDocumentType.Name name) { + SDDocumentType type = ownedTypes.get(name); + if (type == null) { + for (SDDocumentType inherited : inheritedTypes.values()) { + type = inherited.getType(name); + if (type != null) { + return type; + } + } + } + return type; + } + + public SDDocumentType addAnnotation(AnnotationType annotation) { + annotationTypes.register(annotation); + return this; + } + + /** Returns all owned datatypes. */ + public Collection getTypes() { return ownedTypes.values(); } + + // TODO: Include inherited + public Map getAnnotations() { return annotationTypes.getTypes(); } + public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); } + + public Collection getAllTypes() { + Collection list = new ArrayList<>(); + list.addAll(getTypes()); + for (SDDocumentType inherited : inheritedTypes.values()) { + list.addAll(inherited.getAllTypes()); + } + return list; + } + + public Map allTypes() { + Map map = new LinkedHashMap<>(); + for (SDDocumentType inherited : inheritedTypes.values()) + map.putAll(inherited.allTypes()); + map.putAll(ownedTypes); + return map; + } + + /** + * Creates a new document type. + * The document type id will be generated as a hash from the document type name. + * + * @param name The name of the new document type + */ + public SDDocumentType(String name) { + this(name,null); + } + + public SDDocumentType(DataTypeName name) { + this(name.getName()); + } + + /** + * Creates a new document type. + * The document type id will be generated as a hash from the document type name. + * + * @param name the name of the new document type + * @param schema check for type ID collisions in this search definition + */ + public SDDocumentType(String name, Schema schema) { + docType = new DocumentType(name); + validateId(schema); + inherit(VESPA_DOCUMENT); + } + + public boolean isStruct() { return getStruct() != null; } + public DataType getStruct() { return structType; } + public SDDocumentType setStruct(DataType structType) { + if (structType != null) { + this.structType = structType; + inheritedTypes.remove(VESPA_DOCUMENT.getDocumentName()); + } else { + if (docType.contentStruct() != null) { + this.structType = docType.contentStruct(); + inheritedTypes.clear(); + } else { + throw new IllegalArgumentException("You can not set a null struct"); + } + } + return this; + } + + public String getName() { return docType.getName(); } + public DataTypeName getDocumentName() { return docType.getDataTypeName(); } + public DocumentType getDocumentType() { return docType; } + + public void inherit(DataTypeName name) { + inherit(new TemporarySDDocumentType(name)); + } + + public void inherit(SDDocumentType type) { + if (type == null) return; + if (type.getName().equals(this.getName())) + throw new IllegalArgumentException("Document type '" + getName() + "' cannot inherit itself"); + if ( ! inheritedTypes.containsKey(type.getDocumentName()) || + (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) { + inheritedTypes.put(type.getDocumentName(), type); + } + } + + public Collection getInheritedTypes() { return inheritedTypes.values(); } + + public Map inheritedTypes() { return inheritedTypes; } + + protected void validateId(Schema schema) { + if (schema == null) return; + if (schema.getDocument(getName()) == null) return; + SDDocumentType doc = schema.getDocument(); + throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " + + "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); + } + + public void setFieldId(SDField field, int id) { + field.setId(id, docType); + } + + /** Override getField, as it may need to ask inherited types that isn't registered in document type. */ + public Field getField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + Field f = docType.getField(superFieldName); + if (f != null) { + if (f instanceof SDField) { + SDField superField = (SDField)f; + return superField.getStructField(subFieldName); + } else { + throw new IllegalArgumentException("Field " + f.getName() + " is not an SDField"); + } + } + } + Field f = docType.getField(name); + if (f == null) { + for(SDDocumentType parent : inheritedTypes.values()) { + f = parent.getField(name); + if (f != null) return f; + } + } + return f; + } + + public void addField(Field field) { + verifyInheritance(field); + for (Iterator i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) { + if (field.getName().equalsIgnoreCase((i.next()).getName())) { + throw new IllegalArgumentException("Duplicate (case insensitively) " + field + " in " + this); + } + } + docType.addField(field); + } + + /** Parse-time inheritance check. */ + private void verifyInheritance(Field field) { + for (SDDocumentType parent : inheritedTypes.values()) { + for (Field pField : parent.fieldSet()) { + if (pField.getName().equals(field.getName())) { + if (!pField.getDataType().equals(field.getDataType())) { + throw new IllegalArgumentException("For " + this + ", field '" + field.getName() + + "': Datatype can not be different from that of same field " + + "in the supertype '" + parent.getName() + "'"); + } + } + } + } + } + + public SDField addField(String string, DataType dataType) { + SDField field = new SDField(this, string, dataType); + addField(field); + return field; + } + + public Field addField(String fName, DataType dataType, boolean header, int code) { + SDField field = new SDField(this, fName, code, dataType); + addField(field); + return field; + } + + private Map fieldsInherited() { + Map map = new LinkedHashMap<>(); + for (SDDocumentType parent : inheritedTypes.values()) { + for (Field field : parent.fieldSet()) { + map.put(field.getName(), field); + } + } + return map; + } + + public Set fieldSet() { + Map map = fieldsInherited(); + Iterator it = docType.fieldIteratorThisTypeOnly(); + while (it.hasNext()) { + Field field = it.next(); + map.put(field.getName(), field); + } + return new LinkedHashSet<>(map.values()); + } + + public Iterator fieldIterator() { + return fieldSet().iterator(); + } + + /** Returns the number of fields in this only, not including inherited fields */ + // TODO: Remove + public int getFieldCount() { + return docType.getFieldCount(); + } + + @Override + public String toString() { + return "document type '" + docType.getName() + "'"; + } + + private static SDDocumentType createSDDocumentType(StructDataType structType) { + SDDocumentType docType = new SDDocumentType(structType.getName()); + for (Field field : structType.getFields()) { + docType.addField(new SDField(docType, field.getName(), field.getDataType())); + } + docType.setStruct(structType); + return docType; + } + + /** The field sets defined for this type and its {@link Schema} */ + public FieldSets getFieldSets() { + return fieldSets; + } + + /** Sets the field sets for this */ + public void setFieldSets(FieldSets fieldSets) { + this.fieldSets = fieldSets; + } + + public Optional getDocumentReferences() { + return documentReferences; + } + + public void setDocumentReferences(DocumentReferences documentReferences) { + this.documentReferences = Optional.of(documentReferences); + } + + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java new file mode 100644 index 00000000000..668b6388620 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -0,0 +1,802 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.documentmodel.OwnedTemporaryType; +import com.yahoo.documentmodel.TemporaryUnknownType; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.fieldoperation.FieldOperation; +import com.yahoo.schema.fieldoperation.FieldOperationContainer; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.TreeMap; + + +/** + * The field class represents a document field. It is used in + * the Document class to get and set fields. Each SDField has a name, a numeric ID, + * a data type. The numeric ID is used when the fields are stored + * in serialized form. + * + * @author bratseth + */ +public class SDField extends Field implements TypedKey, FieldOperationContainer, ImmutableSDField { + + /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ + private boolean indexStructureField = false; + + /** The indexing statements to be applied to this value during indexing */ + private ScriptExpression indexingScript = new ScriptExpression(); + + /** The default rank type for indices of this field */ + private RankType rankType = RankType.DEFAULT; + + /** Rank settings in a "rank" block for the field. */ + private final Ranking ranking = new Ranking(); + + /** + * The literal boost of this field. This boost is added to a rank score + * when a query term matched as query term exactly (unnormalized and unstemmed). + * Non-positive boosts causes no boosting, 0 allows boosts + * to be specified in other rank profiles, while negative values + * turns the capability off. + */ + private int literalBoost = -1; + + /** + * The weight of this field. This is a percentage, + * so 100 is default to provide the identity transform. + */ + private int weight = 100; + + /** + * Indicates what kind of matching should be done on this field + */ + private Matching matching = new Matching(); + + private Dictionary dictionary = null; + + /** Attribute settings, or null if there are none */ + private final Map attributes = new TreeMap<>(); + + /** + * The stemming setting of this field, or null to use the default. + * Default is determined by the owning search definition. + */ + private Stemming stemming = null; + + /** How content of this field should be accent normalized etc. */ + private NormalizeLevel normalizing = new NormalizeLevel(); + + /** Extra query commands of this field */ + private final List queryCommands = new java.util.ArrayList<>(0); + + /** Summary fields defined in this field */ + private final Map summaryFields = new java.util.LinkedHashMap<>(0); + + /** The explicitly index settings on this field */ + private final Map indices = new java.util.LinkedHashMap<>(); + + private boolean idOverride = false; + + /** Struct fields defined in this field */ + private final Map structFields = new java.util.LinkedHashMap<>(0); + + /** The document that this field was declared in, or null */ + private SDDocumentType repoDocType = null; + + /** The aliases declared for this field. May pertain to indexes or attributes */ + private final Map aliasToName = new HashMap<>(); + + /** Pending operations that must be applied after parsing, due to use of not-yet-defined structs. */ + private final List pendingOperations = new LinkedList<>(); + + private boolean isExtraField = false; + + private boolean wasConfiguredToDoAttributing = false; + + /** + * Creates a new field. This method is only used to create reserved fields. + * + * @param name the name of the field + * @param dataType the datatype of the field + */ + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { + super(name, id, dataType); + this.repoDocType = repo; + populate(name, dataType); + } + + public SDField(String name, DataType dataType) { + this(null, name, dataType); + } + + /** Creates a new field */ + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name, dataType, null); + } + + /** Creates a new field */ + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + this(repo, name, dataType, owner, null, 0); + } + + /** + * Creates a new field + * + * @param name the name of the field + * @param dataType the datatype of the field + * @param owner the owning document (used to check for id collisions) + * @param fieldMatching the matching object to set for the field + */ + protected SDField(SDDocumentType repo, + String name, + DataType dataType, + SDDocumentType owner, + Matching fieldMatching, + int recursion) + { + super(name, dataType, owner == null ? null : owner.getDocumentType()); + this.repoDocType = repo; + this.structFieldDepth = recursion; + if (fieldMatching != null) + this.setMatching(fieldMatching); + populate(name, dataType); + } + + private int structFieldDepth = 0; + + private void populate(String name, DataType dataType) { + if (dataType instanceof TensorDataType) { + TensorType type = ((TensorDataType)dataType).getTensorType(); + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + name + " type " + type + + ": Dense tensor dimensions must have a size"); + addQueryCommand("type " + type); + } + else if (dataType instanceof WeightedSetDataType) { + var nested = ((WeightedSetDataType) dataType).getNestedType().getName(); + addQueryCommand("type WeightedSet<" + nested + ">"); + } + else { + addQueryCommand("type " + dataType.getName()); + } + } + + public void setIsExtraField(boolean isExtra) { + isExtraField = isExtra; + } + + @Override + public boolean isExtraField() { + return isExtraField; + } + + @Override + public boolean isImportedField() { + return false; + } + + @Override + public boolean doesAttributing() { + return containsExpression(AttributeExpression.class); + } + + @Override + public boolean doesIndexing() { + return containsExpression(IndexExpression.class); + } + + public boolean doesSummarying() { + if (usesStruct()) { + for (SDField structField : getStructFields()) { + if (structField.doesSummarying()) { + return true; + } + } + } + return containsExpression(SummaryExpression.class); + } + + @Override + public boolean doesLowerCasing() { + return containsExpression(LowerCaseExpression.class); + } + + @Override + public boolean containsExpression(Class searchFor) { + return findExpression(searchFor) != null; + } + + private T findExpression(Class searchFor) { + return new ExpressionSearcher<>(searchFor).searchIn(indexingScript); + } + + public void addSummaryFieldSources(SummaryField summaryField) { + if (usesStruct()) { + /* + * How this works for structs: When at least one sub-field in a struct is to + * be used for summary, that whole struct field is included in summary.cfg. Then, + * vsmsummary.cfg specifies the sub-fields used for each struct field. + * So we recurse into each struct, adding the destination classes set for each sub-field + * to the main summary-field for the struct field. + */ + for (SDField structField : getStructFields()) { + for (SummaryField sumF : structField.getSummaryFields().values()) { + for (String dest : sumF.getDestinations()) { + summaryField.addDestination(dest); + } + } + structField.addSummaryFieldSources(summaryField); + } + } else { + if (doesSummarying()) { + summaryField.addSource(getName()); + } + } + } + + private boolean doneStructFields = false; + + @SuppressWarnings("deprecation") + private void actuallyMakeStructFields() { + if (doneStructFields) return; + if (getFirstStructOrMapRecursive() == null) { + doneStructFields = true; + return; + } + var sdoc = repoDocType; + var dataType = getDataType(); + + java.util.function.BiConsumer supplyStructField = (fieldName, fieldType) -> { + if (structFields.containsKey(fieldName)) return; + Matching subFieldMatching = new Matching(); + subFieldMatching.merge(this.matching); + String subName = getName().concat(".").concat(fieldName); + var subField = new SDField(sdoc, subName, fieldType, null, + subFieldMatching, structFieldDepth + 1); + structFields.put(fieldName, subField); + }; + + if (dataType instanceof MapDataType) { + MapDataType mdt = (MapDataType) dataType; + supplyStructField.accept("key", mdt.getKeyType()); + supplyStructField.accept("value", mdt.getValueType()); + } else { + if (structFieldDepth >= 10) { + // too risky, infinite recursion + doneStructFields = true; + return; + } + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } + if ((dataType instanceof MapDataType) || (dataType instanceof CollectionDataType)) { + // "array of map" or "array of array" will not have any struct fields + // TODO: consider what this would mean + doneStructFields = true; + return; + } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (dataType instanceof TemporaryUnknownType && subType != null) { + for (Field field : subType.fieldSet()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } else if (dataType instanceof OwnedTemporaryType && subType != null) { + for (Field field : subType.fieldSet()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } else if (dataType instanceof StructDataType) { + var sdt = (StructDataType) dataType; + for (Field field : sdt.getFields()) { + supplyStructField.accept(field.getName(), field.getDataType()); + } + } + if ((subType == null) && (structFields.size() > 0)) { + throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + + this + " [" + getDataType() + getDataType().getClass() + + "] with " + structFields.size() + " struct fields"); + } + // populate struct fields with matching + if (subType != null) { + for (Field f : subType.fieldSet()) { + if (f instanceof SDField) { + SDField field = (SDField) f; + SDField subField = structFields.get(field.getName()); + if (subField != null) { + // we just made this with a copy of our matching (see above) + Matching subFieldMatching = subField.getMatching(); + subFieldMatching.merge(field.getMatching()); + subField.setMatching(subFieldMatching); + } + } else { + throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); + } + } + } + // else ("missing subtype for struct fields in: " + this + " type " + getDataType() + " [" + getDataType().getClass().getSimpleName() + "]"); + } + doneStructFields = true; + } + + private Matching matchingForStructFields = null; + + public void addOperation(FieldOperation op) { + pendingOperations.add(op); + } + + @Override + public void applyOperations(SDField field) { + if (pendingOperations.isEmpty()) return; + + Collections.sort(pendingOperations); + ListIterator ops = pendingOperations.listIterator(); + while (ops.hasNext()) { + FieldOperation op = ops.next(); + ops.remove(); + op.apply(field); + } + } + + public void applyOperations() { + applyOperations(this); + } + + public void setId(int fieldId, DocumentType owner) { + super.setId(fieldId, owner); + idOverride = true; + } + + public StructDataType getFirstStructRecursive() { + DataType dataType = getDataType(); + while (true) { // Currently no nesting of collections + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } else if (dataType instanceof MapDataType) { + dataType = ((MapDataType)dataType).getValueType(); + } else { + break; + } + } + return (dataType instanceof StructDataType) ? (StructDataType)dataType : null; + } + + private DataType getFirstStructOrMapRecursive() { + DataType dataType = getDataType(); + while (dataType instanceof CollectionDataType) { // Currently no nesting of collections + dataType = ((CollectionDataType)dataType).getNestedType(); + } + return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null; + } + + private boolean usesStruct() { + DataType dt = getFirstStructRecursive(); + return (dt != null); + } + + @Override + public boolean usesStructOrMap() { + DataType dt = getFirstStructOrMapRecursive(); + return (dt != null); + } + + @Override + public boolean wasConfiguredToDoAttributing() { + return wasConfiguredToDoAttributing; + } + + /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ + public void parseIndexingScript(String script) { + parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + } + + public void parseIndexingScript(String script, Linguistics linguistics, Map embedders) { + try { + ScriptParserContext config = new ScriptParserContext(linguistics, embedders); + config.setInputStream(new IndexingInput(script)); + setIndexingScript(ScriptExpression.newInstance(config)); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); + } + } + + /** Sets the indexing script of this, or null to not use a script */ + public void setIndexingScript(ScriptExpression exp) { + if (exp == null) { + exp = new ScriptExpression(); + } + indexingScript = exp; + if (indexingScript.isEmpty()) { + return; // TODO: This causes empty expressions not to be propagate to struct fields!! BAD BAD BAD!! + } + if (!wasConfiguredToDoAttributing()) { + wasConfiguredToDoAttributing = doesAttributing(); + } + if (!usesStructOrMap()) { + new ExpressionVisitor() { + + @Override + protected void doVisit(Expression exp) { + if (!(exp instanceof AttributeExpression)) { + return; + } + String fieldName = ((AttributeExpression)exp).getFieldName(); + if (fieldName == null) { + fieldName = getName(); + } + Attribute attribute = attributes.get(fieldName); + if (attribute == null) { + addAttribute(new Attribute(fieldName, getDataType())); + } + } + }.visit(indexingScript); + } + for (SDField structField : getStructFields()) { + structField.setIndexingScript(exp); + } + } + + @Override + public ScriptExpression getIndexingScript() { return indexingScript; } + + @SuppressWarnings("deprecation") + @Override + public void setDataType(DataType type) { + if (type.equals(DataType.URI)) { // Different defaults, naturally + normalizing.inferLowercase(); + stemming = Stemming.NONE; + } + this.dataType = type; + if ( ! idOverride) { + this.fieldId = calculateIdV7(null); + } + } + + @Override + public boolean isIndexStructureField() { + return indexStructureField; + } + + public void setIndexStructureField(boolean indexStructureField) { + this.indexStructureField = indexStructureField; + } + + @Override + public boolean hasIndex() { + return (getIndexingScript() != null) && doesIndexing(); + } + + /** Sets the literal boost of this field */ + public void setLiteralBoost(int literalBoost) { this.literalBoost=literalBoost; } + + /** + * Returns the literal boost of this field. This boost is added to a literal score + * when a query term matched as query term exactly (unnormalized and unstemmed). + * Default is non-positive. + */ + @Override + public int getLiteralBoost() { return literalBoost; } + + /** Sets the weight of this field */ + public void setWeight(int weight) { this.weight=weight; } + + /** Returns the weight of this field, or 0 if nothing is set */ + @Override + public int getWeight() { return weight; } + + /** + * Returns what kind of matching type should be applied. + */ + @Override + public Matching getMatching() { return matching; } + + /** + * Sets what kind of matching type should be applied. + * (Token matching is default, PREFIX, SUBSTRING, SUFFIX are alternatives) + */ + public void setMatching(Matching matching) { this.matching=matching; } + + /** + * Returns Dictionary settings. + */ + public Dictionary getDictionary() { return dictionary; } + public Dictionary getOrSetDictionary() { + if (dictionary == null) { + dictionary = new Dictionary(); + } + return dictionary; + } + + /** + * Set the matching type for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields at lookup time instead. + public void setMatchingType(MatchType type) { + this.getMatching().setType(type); + for (SDField structField : getStructFields()) { + structField.setMatchingType(type); + } + } + + /** + * Set the matching type for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields at lookup time instead. + public void setMatchingCase(Case casing) { + this.getMatching().setCase(casing); + for (SDField structField : getStructFields()) { + structField.setMatchingCase(casing); + } + } + /** + * Set matching algorithm for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields at lookup time instead. + public void setMatchingAlgorithm(MatchAlgorithm algorithm) { + this.getMatching().setAlgorithm(algorithm); + for (SDField structField : getStructFields()) { + structField.getMatching().setAlgorithm(algorithm); + } + } + + /** Adds an explicit index defined in this field */ + public void addIndex(Index index) { + indices.put(index.getName(),index); + } + + /** + * Returns an index, or null if no index with this name has had + * some explicit settings applied in this field (even if this returns null, + * the index may be implicitly defined by an indexing statement) + */ + @Override + public Index getIndex(String name) { + return indices.get(name); + } + + /** + * Returns an index if this field has one (implicitly or + * explicitly) targeting the given name. + */ + @Override + public boolean existsIndex(String name) { + if (indices.get(name) != null) return true; + return name.equals(getName()) && doesIndexing(); + } + + /** + * Defined indices on this field + * @return defined indices on this + */ + @Override + public Map getIndices() { + return indices; + } + + /** + * Sets the default rank type of this fields indices, and sets this rank type + * to all indices explicitly defined here which has no index set. + * (This complex behavior is dues to the fact than we would prefer to have rank types + * per field, not per index) + */ + public void setRankType(RankType rankType) { + this.rankType=rankType; + for (Index index : getIndices().values()) { + if (index.getRankType()==null) + index.setRankType(rankType); + } + + } + + /** Returns the rank settings set in a "rank" block for this field. This is never null. */ + @Override + public Ranking getRanking() { return ranking; } + + /** Returns the default rank type of indices of this field, or null if nothing is set */ + @Override + public RankType getRankType() { return this.rankType; } + + /** + * Returns the search-time attribute settings of this field or null if none is set. + * + *

TODO: Make unmodifiable.

+ */ + @Override + public Map getAttributes() { return attributes; } + + public Attribute getAttribute() { + return attributes.get(getName()); + } + + public void addAttribute(Attribute attribute) { + String name = attribute.getName(); + if (name == null || "".equals(name)) { + name = getName(); + attribute.setName(name); + } + attributes.put(attribute.getName(),attribute); + } + + /** + * Returns the stemming setting of this field. + * Default is determined by the owning search definition. + * + * @return the stemming setting of this, or null, to use the default + */ + @Override + public Stemming getStemming() { return stemming; } + + /** + * Whether this field should be stemmed in this search definition + */ + @Override + public Stemming getStemming(Schema schema) { + if (stemming != null) + return stemming; + else + return schema.getStemming(); + } + + @Override + public Field asField() { + return this; + } + + /** + * Sets how this field should be stemmed, or set to null to use the default. + */ + public void setStemming(Stemming stemming) { + this.stemming = stemming; + } + + /** Returns an unmodifiable map of the summary fields defined in this */ + @Override + public Map getSummaryFields() { + return Collections.unmodifiableMap(summaryFields); + } + + public void removeSummaryFields() { + summaryFields.clear(); + } + + /** Adds a summary field */ + public void addSummaryField(SummaryField summaryField) { + summaryFields.put(summaryField.getName(),summaryField); + } + + /** + * Returns a summary field defined (implicitly or explicitly) by this field. + * Returns null if there is no such summary field defined. + */ + @Override + public SummaryField getSummaryField(String name) { + return summaryFields.get(name); + } + + /** + * Returns a summary field defined (implicitly or explicitly) by this field. + * + * @param create true to create the summary field and add it to this field before returning if it is missing + * @return the summary field, or null if not present and create is false + */ + public SummaryField getSummaryField(String name, boolean create) { + SummaryField summaryField=summaryFields.get(name); + if (summaryField==null && create) { + summaryField=new SummaryField(name, getDataType()); + addSummaryField(summaryField); + } + return summaryFields.get(name); + } + + /** Returns list of static struct fields */ + @Override + public Collection getStructFields() { + actuallyMakeStructFields(); + return structFields.values(); + } + + /** + * Returns a struct field defined in this field, + * potentially traversing into nested structs. + * Returns null if there is no such struct field defined. + */ + @Override + public SDField getStructField(String name) { + actuallyMakeStructFields(); + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + SDField superField = structFields.get(superFieldName); + if (superField != null) { + return superField.getStructField(subFieldName); + } + return null; + } + return structFields.get(name); + } + + /** + * Returns how the content of this field should be accent normalized etc + */ + @Override + public NormalizeLevel getNormalizing() { return normalizing; } + + /** + * Change how the content of this field should be accent normalized etc + */ + public void setNormalizing(NormalizeLevel level) { normalizing = level; } + + public void addQueryCommand(String name) { + queryCommands.add(name); + } + + public boolean hasQueryCommand(String name) { + return queryCommands.contains(name); + } + + /** Returns a list of query commands */ + @Override + public List getQueryCommands() { return queryCommands; } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof SDField)) return false; + return super.equals(other); + } + + @Override + public int hashCode() { + return getName().hashCode(); + } + + @Override + public String toString() { + return "field '" + getName() + "'"; + } + + /** The aliases declared for this field */ + @Override + public Map getAliasToName() { + return aliasToName; + } + + @Override + public boolean hasFullIndexingDocprocRights() { + Attribute self = getAttributes().get(getName()); + return (!isExtraField() || ((self != null) && self.isMutable())); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Sorting.java b/config-model/src/main/java/com/yahoo/schema/document/Sorting.java new file mode 100644 index 00000000000..2d0c9a5d27b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Sorting.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import java.io.Serializable; + +/** + * A search-time document attribute sort specification(per-document in-memory value). + * This belongs to the attribute or field(implicitt attribute). + * + * @author baldersheim + */ +public final class Sorting implements Cloneable, Serializable { + + // Remember to change hashCode and equals when you add new fields + public enum Function {UCA, RAW, LOWERCASE} + public enum Strength {PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL} + private boolean ascending = true; + private Function function = Function.UCA; + private String locale = ""; + private Strength strength = Strength.PRIMARY; + + public boolean isAscending() { return ascending; } + public boolean isDescending() { return ! ascending; } + public String getLocale() { return locale; } + public Function getFunction() { return function; } + public Strength getStrength() { return strength; } + + public void setAscending() { ascending = true; } + public void setDescending() { ascending = false; } + public void setFunction(Function function) { this.function = function; } + public void setLocale(String locale) { this.locale = locale; } + public void setStrength(Strength strength) { this.strength = strength; } + + public int hashCode() { + return locale.hashCode() + + strength.hashCode() + + function.hashCode() + + (isDescending() ? 13 : 0); + } + + public boolean equals(Object object) { + if (! (object instanceof Sorting)) return false; + + Sorting other=(Sorting)object; + return this.locale.equals(other.locale) && + (ascending == other.ascending) && + (function == other.function) && + (strength == other.strength); + } + + @Override + public Sorting clone() { + try { + return (Sorting)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + public String toString() { + return "sorting '" + (isAscending() ? '+' : '-') + function.toString() + "(" + strength.toString() + ", " + locale + ")"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/Stemming.java b/config-model/src/main/java/com/yahoo/schema/document/Stemming.java new file mode 100644 index 00000000000..5ec844e2540 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/Stemming.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.language.process.StemMode; + +/** + * The stemming setting of a field. This describes how the search engine + * should transform content of this field into base forms (stems) to increase + * recall (find "car" when you search for "cars" etc.). + * + * @author bratseth + */ +public enum Stemming { + + /** No stemming */ + NONE("none"), + + /** select shortest possible stem */ + SHORTEST("shortest"), + + /** select the "best" stem alternative */ + BEST("best"), + + /** index multiple stems */ + MULTIPLE("multiple"); + + private final String name; + + /** + * Returns the stemming object for the given string. + * The legal stemming names are the stemming constants in any capitalization. + * + * @throws IllegalArgumentException if there is no stemming type with the given name + */ + public static Stemming get(String stemmingName) { + try { + return Stemming.valueOf(stemmingName.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting"); + } + } + + Stemming(String name) { + this.name = name; + } + + public String getName() { return name; } + + @Override + public String toString() { + return "stemming " + name; + } + + public StemMode toStemMode() { + switch(this) { + case SHORTEST: return StemMode.SHORTEST; + case MULTIPLE: return StemMode.ALL; + case BEST : return StemMode.BEST; + case NONE: return StemMode.NONE; + default: throw new IllegalStateException("Inconvertible stem mode " + this); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java new file mode 100644 index 00000000000..efc0674586d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +/** + * A field that is imported from a field in a referenced document type and given an alias name. + * + * This is temporary AST structure that only refers to the imported field by name. + * + * @author geirst + */ +public class TemporaryImportedField { + + private final String fieldName; + private final String referenceFieldName; + private final String targetFieldName; + + public TemporaryImportedField(String fieldName, + String referenceFieldName, + String targetFieldName) { + this.fieldName = fieldName; + this.referenceFieldName = referenceFieldName; + this.targetFieldName = targetFieldName; + } + + public String fieldName() { + return fieldName; + } + + public String referenceFieldName() { + return referenceFieldName; + } + + public String targetFieldName() { + return targetFieldName; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java new file mode 100644 index 00000000000..7ad4feb6d32 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.Schema; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * A set of fields that are imported from referenced document types. + * + * This is temporary AST structure that only refers to the imported fields by name. + * + * @author geirst + */ +public class TemporaryImportedFields { + + private final Schema owner; + private final Map fields = new LinkedHashMap<>(); + + public TemporaryImportedFields(Schema owner) { + this.owner = owner; + } + + public void add(TemporaryImportedField importedField) { + fields.put(importedField.fieldName(), importedField); + } + + public boolean hasField(String fieldName) { + return fields.get(fieldName) != null; + } + + public Map fields() { + if (owner.inherited().isEmpty()) return Collections.unmodifiableMap(fields); + if (owner.inherited().get().temporaryImportedFields().isEmpty()) return Collections.unmodifiableMap(fields); + + var allFields = new HashMap<>(owner.inherited().get().temporaryImportedFields().get().fields()); + allFields.putAll(fields); + return allFields; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java new file mode 100644 index 00000000000..3194a14a143 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataTypeName; + +/** + * @author baldersheim + */ +public class TemporarySDDocumentType extends SDDocumentType { + public TemporarySDDocumentType(DataTypeName name) { + super(name); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java new file mode 100644 index 00000000000..e455fa78455 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; + +/** + * @author Einar M R Rosenvinge + */ +public class TemporarySDField extends SDField { + + public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { + super(repo, name, dataType, owner); + } + + public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { + super(repo, name, dataType); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java new file mode 100644 index 00000000000..8de8c7b64fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.document.DataType; + +/** + * Common interface for various typed key (or field definitions). + * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping + * + * @author bratseth + */ +public interface TypedKey { + + String getName(); + + void setDataType(DataType type); + + DataType getDataType(); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java new file mode 100644 index 00000000000..3dc46a91c1b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document.annotation; + +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author Einar M R Rosenvinge + */ +public class SDAnnotationType extends AnnotationType { + + private SDDocumentType sdDocType; + private String inherits; + + public SDAnnotationType(String name) { + super(name); + } + + public SDAnnotationType(String name, SDDocumentType dataType, String inherits) { + super(name); + this.sdDocType = dataType; + this.inherits = inherits; + } + + public SDDocumentType getSdDocType() { + return sdDocType; + } + + public void setSdDocType(SDDocumentType value) { + assert(sdDocType == null); + sdDocType = value; + } + + public String getInherits() { + return inherits; + } + + public void inherit(String inherits) { + this.inherits = inherits; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java new file mode 100644 index 00000000000..de9bd977823 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document.annotation; + +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author Einar M R Rosenvinge + */ +public class TemporaryAnnotationReferenceDataType extends AnnotationReferenceDataType { + + private final String target; + + public TemporaryAnnotationReferenceDataType(String target) { + this.target = target; + } + + public String getTarget() { + return target; + } + + @Override + public void setAnnotationType(AnnotationType type) { + super.setName("annotationreference<" + type.getName() + ">"); + super.setAnnotationType(type); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ConstantTensorTransformer.java new file mode 100644 index 00000000000..4e320594918 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ConstantTensorTransformer.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.schema.FeatureNames; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +import java.util.ArrayList; +import java.util.List; + +/** + * Transforms named references to constant tensors with the rank feature 'constant'. + * + * @author geirst + */ +public class ConstantTensorTransformer extends ExpressionTransformer { + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) { + return transformFeature((ReferenceNode) node, context); + } else if (node instanceof CompositeNode) { + return transformChildren((CompositeNode) node, context); + } else { + return node; + } + } + + private ExpressionNode transformFeature(ReferenceNode node, RankProfileTransformContext context) { + if ( ! node.getArguments().isEmpty() && ! FeatureNames.isSimpleFeature(node.reference())) { + return transformArguments(node, context); + } else { + return transformConstantReference(node, context); + } + } + + private ExpressionNode transformArguments(ReferenceNode node, RankProfileTransformContext context) { + List arguments = node.getArguments().expressions(); + List transformedArguments = new ArrayList<>(arguments.size()); + for (ExpressionNode argument : arguments) { + transformedArguments.add(transform(argument, context)); + } + return node.setArguments(transformedArguments); + } + + private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) { + String constantName = node.getName(); + Reference constantReference = node.reference(); + if (FeatureNames.isConstantFeature(constantReference)) { + constantName = constantReference.simpleArgument().orElse(null); + } else if (constantReference.isIdentifier()) { + constantReference = FeatureNames.asConstantFeature(constantName); + } else { + return node; + } + Value value = context.constants().get(constantName); + if (value == null || value.type().rank() == 0) return node; + + TensorValue tensorValue = (TensorValue)value; + String tensorType = tensorValue.asTensor().type().toString(); + context.rankProperties().put(constantReference + ".value", tensorValue.toString()); + context.rankProperties().put(constantReference + ".type", tensorType); + return new ReferenceNode(constantReference); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java new file mode 100644 index 00000000000..86aedd4332a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.google.common.collect.ImmutableList; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.searchlib.rankingexpression.transform.Simplifier; +import com.yahoo.searchlib.rankingexpression.transform.TensorMaxMinTransformer; + +import java.util.List; + +/** + * The transformations done on ranking expressions done at config time before passing them on to the Vespa + * engine for execution. + * + * An instance of this class has scope of a compilation of a single rank profile. + * + * @author bratseth + */ +public class ExpressionTransforms { + + private final List transforms; + + public ExpressionTransforms() { + transforms = + ImmutableList.of(new TensorFlowFeatureConverter(), + new OnnxFeatureConverter(), + new OnnxModelTransformer(), + new XgboostFeatureConverter(), + new LightGBMFeatureConverter(), + new TokenTransformer(), + new ConstantDereferencer(), + new ConstantTensorTransformer(), + new FunctionInliner(), + new FunctionShadower(), + new TensorMaxMinTransformer(), + new Simplifier()); + } + + public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { + for (ExpressionTransformer transformer : transforms) + expression = transformer.transform(expression, context); + return expression; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionInliner.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionInliner.java new file mode 100644 index 00000000000..382d51747bb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionInliner.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.schema.RankProfile; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +/** + * Inlines functions in ranking expressions + * + * @author bratseth + */ +public class FunctionInliner extends ExpressionTransformer { + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeatureNode((ReferenceNode)node, context); + if (node instanceof CompositeNode) + return transformChildren((CompositeNode)node, context); + return node; + } + + private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { + if (feature.getArguments().size() > 0) return feature; // From RankProfile: only inline no-arg functions + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName()); + if (rankingExpressionFunction == null) return feature; + return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionShadower.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionShadower.java new file mode 100644 index 00000000000..702e4ea220e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionShadower.java @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.schema.RankProfile; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +/** + * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function. + * This has the effect of allowing rank profile functions to redefine built-in functions. + * Another effect is that we can add built-in functions over time + * without fear of breaking existing users' functions with the same name. + * + * However, there is a (largish) caveat. If a user has a function with a certain number + * of arguments, and we add in a built-in function with a different arity, + * this will cause parse errors as the Java parser gives precedence to + * built-in functions. + * + * @author lesters + */ +public class FunctionShadower extends ExpressionTransformer { + + @Override + public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { + ExpressionNode node = expression.getRoot(); + ExpressionNode result = transform(node, context); + return (result == node) + ? expression + : new RankingExpression(expression.getName(), result); + } + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof FunctionNode) + return transformFunctionNode((FunctionNode) node, context); + if (node instanceof CompositeNode) + return transformChildren((CompositeNode)node, context); + return node; + } + + private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { + String name = function.getFunction().toString(); + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().findFunction(name); + if (rankingExpressionFunction == null) + return transformChildren(function, context); + + int functionArity = function.getFunction().arity(); + if (functionArity != rankingExpressionFunction.function().arguments().size()) + return transformChildren(function, context); + + ReferenceNode node = new ReferenceNode(name, function.children(), null); + return transformChildren(node, context); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/LightGBMFeatureConverter.java new file mode 100644 index 00000000000..af5fa5ebeab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/LightGBMFeatureConverter.java @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the lightgbm(model-path) pseudofeature with the + * native Vespa ranking expression implementing the same computation. + * + * @author lesters + */ +public class LightGBMFeatureConverter extends ExpressionTransformer { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map convertedLightGBMModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("lightgbm")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedLightGBMModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.size() != 1) + throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " + + "the LightGBM model file under [application]/models"); + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxFeatureConverter.java new file mode 100644 index 00000000000..2277491cd47 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxFeatureConverter.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the onnx(model-path, output) + * pseudofeature with the native Vespa ranking expression implementing + * the same computation. + * + * @author bratseth + * @author lesters + */ +public class OnnxFeatureConverter extends ExpressionTransformer { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map convertedOnnxModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("onnx_vespa")) return feature; + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedOnnxModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } + catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use Onnx model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.isEmpty()) + throw new IllegalArgumentException("An ONNX node must take an argument pointing to " + + "the ONNX model file under [application]/models"); + if (arguments.expressions().size() > 3) + throw new IllegalArgumentException("An onnx feature can have at most 3 arguments"); + + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java new file mode 100644 index 00000000000..4c38c257602 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java @@ -0,0 +1,139 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; +import com.yahoo.vespa.model.ml.ModelName; + +import java.util.List; + +/** + * Transforms ONNX model features of the forms: + * + * onnxModel(config_name) + * onnxModel(config_name).output + * onnxModel("path/to/model") + * onnxModel("path/to/model").output + * onnxModel("path/to/model", "path/to/output") + * onnxModel("path/to/model", "unused", "path/to/output") // signature is unused + * onnx(...) // same as with onnxModel, onnx is an alias of onnxModel + * + * To the format expected by the backend: + * + * onnxModel(config_name).output + * + * @author lesters + */ +public class OnnxModelTransformer extends ExpressionTransformer { + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if (context.rankProfile() == null) return feature; + if (context.rankProfile().schema() == null) return feature; + return transformFeature(feature, context.rankProfile()); + } + + public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile profile) { + String featureName = feature.getName(); + if ( ! featureName.equals("onnxModel") && ! featureName.equals("onnx")) return feature; + + Arguments arguments = feature.getArguments(); + if (arguments.isEmpty()) + throw new IllegalArgumentException("An " + featureName + " feature must take an argument referring to a " + + "onnx-model config or an ONNX file."); + if (arguments.expressions().size() > 3) + throw new IllegalArgumentException("An " + featureName + " feature can have at most 3 arguments."); + + // Check that the model configuration "onnx-model" exists. If not defined, it should have been added + // by the "OnnxModelConfigGenerator" processor. If it still doesn't exist, it is because we can't find + // the actual ONNX file, which can happen if we are restarting or upgrading an application using an + // ONNX file that was transformed to Vespa ranking expressions. We then assume it is in the model store. + + String modelConfigName = getModelConfigName(feature.reference()); + OnnxModel onnxModel = profile.onnxModels().get(modelConfigName); + if (onnxModel == null) { + String path = asString(arguments.expressions().get(0)); + ModelName modelName = new ModelName(null, Path.fromString(path), true); + ConvertedModel convertedModel = ConvertedModel.fromStore(profile.schema().applicationPackage(), modelName, path, profile); + FeatureArguments featureArguments = new FeatureArguments(arguments); + return convertedModel.expression(featureArguments, null); + } + + String defaultOutput = onnxModel.getOutputMap().get(onnxModel.getDefaultOutput()); + String output = getModelOutput(feature.reference(), defaultOutput); + if (! onnxModel.getOutputMap().containsValue(output)) { + throw new IllegalArgumentException(featureName + " argument '" + output + + "' output not found in model '" + onnxModel.getFileName() + "'"); + } + return new ReferenceNode("onnxModel", List.of(new ReferenceNode(modelConfigName)), output); + } + + public static String getModelConfigName(Reference reference) { + if (reference.arguments().size() > 0) { + ExpressionNode expr = reference.arguments().expressions().get(0); + if (expr instanceof ReferenceNode) { // refers to onnx-model config + return expr.toString(); + } + if (expr instanceof ConstantNode) { // refers to a file path + return asValidIdentifier(expr); + } + } + return null; + } + + public static String getModelOutput(Reference reference, String defaultOutput) { + if (reference.output() != null) { + return reference.output(); + } else if (reference.arguments().expressions().size() == 2) { + return asValidIdentifier(reference.arguments().expressions().get(1)); + } else if (reference.arguments().expressions().size() > 2) { + return asValidIdentifier(reference.arguments().expressions().get(2)); + } + return defaultOutput; + } + + public static String stripQuotes(String s) { + if (isNotQuoteSign(s.codePointAt(0))) return s; + if (isNotQuoteSign(s.codePointAt(s.length() - 1))) + throw new IllegalArgumentException("argument [" + s + "] is missing end quote"); + return s.substring(1, s.length()-1); + } + + public static String asValidIdentifier(String str) { + return str.replaceAll("[^\\w\\d\\$@_]", "_"); + } + + private static String asValidIdentifier(ExpressionNode node) { + return asValidIdentifier(asString(node)); + } + + private static boolean isNotQuoteSign(int c) { + return c != '\'' && c != '"'; + } + + public static String asString(ExpressionNode node) { + if ( ! (node instanceof ConstantNode)) + throw new IllegalArgumentException("Expected a constant string as argument, but got '" + node); + return stripQuotes(node.toString()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java new file mode 100644 index 00000000000..cfc859345ad --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.transform.TransformContext; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Extends the transform context with rank profile information + * + * @author bratseth + */ +public class RankProfileTransformContext extends TransformContext { + + private final RankProfile rankProfile; + private final QueryProfileRegistry queryProfiles; + private final ImportedMlModels importedModels; + private final Map inlineFunctions; + private final Map rankProperties = new HashMap<>(); + + public RankProfileTransformContext(RankProfile rankProfile, + QueryProfileRegistry queryProfiles, + Map featureTypes, + ImportedMlModels importedModels, + Map constants, + Map inlineFunctions) { + super(valuesOf(constants), rankProfile.typeContext(queryProfiles, featureTypes)); + this.rankProfile = rankProfile; + this.queryProfiles = queryProfiles; + this.importedModels = importedModels; + this.inlineFunctions = inlineFunctions; + } + + public RankProfile rankProfile() { return rankProfile; } + public QueryProfileRegistry queryProfiles() { return queryProfiles; } + public ImportedMlModels importedModels() { return importedModels; } + public Map inlineFunctions() { return inlineFunctions; } + public Map rankProperties() { return rankProperties; } + + private static Map valuesOf(Map constants) { + return constants.values().stream() + .filter(constant -> constant.value().isPresent()) + .collect(Collectors.toMap(constant -> constant.name().simpleArgument().get(), + constant -> asValue(constant.value().get()))); + } + + private static Value asValue(Tensor tensor) { + if (tensor.type().rank() == 0) + return DoubleValue.of(tensor.asDouble()); + else + return TensorValue.of(tensor); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java new file mode 100644 index 00000000000..fd19c97036e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java @@ -0,0 +1,66 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the tensorflow(model-path, signature, output) + * pseudofeature with the native Vespa ranking expression implementing + * the same computation. + * + * @author bratseth + */ +public class TensorFlowFeatureConverter extends ExpressionTransformer { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map convertedTensorFlowModels = new HashMap<>(); + + public TensorFlowFeatureConverter() {} + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("tensorflow")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedTensorFlowModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, false, context)); + return convertedModel.expression(arguments, context); + } + catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.isEmpty()) + throw new IllegalArgumentException("A tensorflow node must take an argument pointing to " + + "the tensorflow model directory under [application]/models"); + if (arguments.expressions().size() > 3) + throw new IllegalArgumentException("A tensorflow feature can have at most 3 arguments"); + + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TokenTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TokenTransformer.java new file mode 100644 index 00000000000..30d9a3766b3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TokenTransformer.java @@ -0,0 +1,313 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; +import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; +import com.yahoo.searchlib.rankingexpression.rule.ComparisonNode; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.IfNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.TruthOperator; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.Slice; +import com.yahoo.tensor.functions.TensorFunction; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode.wrapScalar; + +/** + * Convenience feature transforms for inputs to Transformer type models. + * + * Replaces features of the form + * + * tokenInputIds + * tokenTypeIds + * tokenAttentionMask + * + * to tensor generation expressions that generate the required input. + * In general, these models expect input of the form: + * + * CLS + arg1 + SEP + arg2 + SEP + 0's + * + * @author lesters + */ +public class TokenTransformer extends ExpressionTransformer { + + static private final ConstantNode ZERO = new ConstantNode(new DoubleValue(0.0)); + static private final ConstantNode ONE = new ConstantNode(new DoubleValue(1.0)); + static private final ConstantNode TWO = new ConstantNode(new DoubleValue(2.0)); + static private final ConstantNode CLS = new ConstantNode(new DoubleValue(101)); + static private final ConstantNode SEP = new ConstantNode(new DoubleValue(102)); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if (feature.getName().equals("tokenInputIds") && shouldTransform(feature, context)) + return transformTokenInputIds(feature, context); + if (feature.getName().equals("tokenTypeIds") && shouldTransform(feature, context)) + return transformTokenTypeIds(feature, context); + if (feature.getName().equals("tokenAttentionMask") && shouldTransform(feature, context)) + return transformTokenAttentionMask(feature, context); + return feature; + } + + /** + * Transforms a feature of the form + * + * tokenInputIds(128, a, b, ...) + * + * to an expression that concatenates the arguments a, b, ... using the + * special Transformers sequences of CLS and SEP, up to length 128, so + * that the sequence becomes + * + * CLS + a + SEP + b + SEP + 0's + * + * Concretely, transforms to a tensor generation expression: + * + * tensor(d0[1],d1[128])( + * if (d1 < 1, + * 101, + * if (d1 < 1 + length_a, + * a{d0:(d1 - (1)}, + * if (d1 < 1 + length_a + 1, + * 102, + * if (d1 < 1 + length_a + 1 + length_b, + * b{d0:(d1 - (1 + length_a + 1))}, + * if (d1 < 1 + length_a + 1 + length_b + 1, + * 102, + * 0.0 + * )))))) + * + * Functions calculating lengths of arguments are added to the rank profile. + */ + private ExpressionNode transformTokenInputIds(ReferenceNode feature, RankProfileTransformContext context) { + checkArguments(feature); + + TensorType type = createTensorType(feature.getName(), feature.getArguments().expressions().get(0)); + + // we need to add functions calculating the token lengths of the arguments + createTokenLengthFunctions(feature, context); + + // create token sequence: CLS + arg1 + SEP + arg2 + SEP + .... + ExpressionNode tokenSequenceExpr = createTokenSequenceExpr(0, createTokenSequence(feature)); + return new TensorFunctionNode(Generate.bound(type, wrapScalar(tokenSequenceExpr))); + } + + /** + * Transforms a feature of the form + * + * tokenTypeIds(128, a, b, ...) + * + * to an expression that generates a tensor that has values 0 for "a" + * (including CLS and SEP tokens) and 1 for the rest of the sequence. + * + * Concretely, transforms to a tensor generation expression: + * + * tensor(d0[1],d1[128])( + * if (d1 < 1 + length_a + 1, + * 0, + * if (d1 < 1 + length_a + 1 + length_b + 1 + ..., + * 1, + * 0 + * ))) + */ + private ExpressionNode transformTokenTypeIds(ReferenceNode feature, RankProfileTransformContext context) { + checkArguments(feature); + + TensorType type = createTensorType(feature.getName(), feature.getArguments().expressions().get(0)); + + // we need to add functions calculating the token lengths of the arguments + createTokenLengthFunctions(feature, context); + + List tokenSequence = createTokenSequence(feature); + ExpressionNode queryLengthExpr = createLengthExpr(2, tokenSequence); + ExpressionNode restLengthExpr = createLengthExpr(tokenSequence.size() - 1, tokenSequence); + ExpressionNode expr = new IfNode( + new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, queryLengthExpr), + ZERO, + new IfNode( + new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, restLengthExpr), + ONE, + ZERO + ) + ); + return new TensorFunctionNode(Generate.bound(type, wrapScalar(expr))); + } + + /** + * Transforms a feature of the form + * + * tokenAttentionMask(128, a, b, ...) + * + * to an expression that generates a tensor that has values 1 for all + * arguments (including CLS and SEP tokens) and 0 for the rest of the + * sequence. + * + * Concretely, transforms to a tensor generation expression: + * + * tensor(d0[1],d1[128])(if(d1 < 1 + length_a + 1 + length_b + 1 + ..., 1, 0)) + * + */ + private ExpressionNode transformTokenAttentionMask(ReferenceNode feature, RankProfileTransformContext context) { + checkArguments(feature); + + TensorType type = createTensorType(feature.getName(), feature.getArguments().expressions().get(0)); + + // we need to add functions calculating the token lengths of the arguments + createTokenLengthFunctions(feature, context); + + List tokenSequence = createTokenSequence(feature); + ExpressionNode lengthExpr = createLengthExpr(tokenSequence.size() - 1, tokenSequence); + ComparisonNode comparison = new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, lengthExpr); + ExpressionNode expr = new IfNode(comparison, ONE, ZERO); + return new TensorFunctionNode(Generate.bound(type, wrapScalar(expr))); + } + + private boolean shouldTransform(ReferenceNode feature, RankProfileTransformContext context) { + if (context.rankProfile().getFunctions().containsKey(feature.getName())) + return false; + if (feature.getArguments().size() < 2) + return false; + return true; + } + + private void checkArguments(ReferenceNode feature) { + for (int i = 1; i < feature.getArguments().size(); ++i) { + ExpressionNode arg = feature.getArguments().expressions().get(i); + if ( ! (arg instanceof ReferenceNode)) { + throw new IllegalArgumentException("Invalid argument " + i + " to " + feature.getName() + ": " + + "the argument must be a reference. Got " + arg.toString()); + } + } + } + + public static TensorType createTensorType(String featureName, ExpressionNode argument) { + try { + int length = Integer.parseInt(argument.toString()); + return new TensorType.Builder(TensorType.Value.FLOAT).indexed("d0", 1).indexed("d1", length).build(); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException("Invalid argument to " + featureName + ": the first argument must be " + + "the length to the token sequence to generate. Got " + argument); + } + } + + private String lengthFunctionName(ReferenceNode arg) { + return "__token_length@" + arg.hashCode(); + } + + private List createTokenSequence(ReferenceNode feature) { + List sequence = new ArrayList<>(); + sequence.add(CLS); + for (int i = 1; i < feature.getArguments().size(); ++i) { + sequence.add(feature.getArguments().expressions().get(i)); + sequence.add(SEP); + } + return sequence; + } + + /** + * Adds functions for calculating the token length input. Assumes that + * token sequences are 0-padded, so this returns the number of non-0 + * tokens using a map and reduce-sum. + */ + private void createTokenLengthFunctions(ReferenceNode feature, RankProfileTransformContext context) { + for (int i = 1; i < feature.getArguments().size(); ++i) { + ExpressionNode arg = feature.getArguments().expressions().get(i); + if ( ! (arg instanceof ReferenceNode)) { + throw new IllegalArgumentException("Invalid argument " + i + " to " + feature.getName() + ": " + + "the argument must be a reference. Got " + arg.toString()); + } + ReferenceNode ref = (ReferenceNode) arg; + String functionName = lengthFunctionName(ref); + if ( ! context.rankProfile().getFunctions().containsKey(functionName)) { + context.rankProfile().addFunction(functionName, List.of(), "sum(map(" + ref + ", f(x)(x > 0)))", false); + } + } + } + + /** + * Recursively creates partial expressions of the form + * + * if (d1 < 1 + length_a, + * a{d0:(d1 - 1}, + * ... + * + * for each part of the token sequence. CLS and SEP are added directly, + * and we create a slice expression for each argument to extract the + * actual tokens. + */ + private ExpressionNode createTokenSequenceExpr(int iter, List sequence) { + ExpressionNode lengthExpr = createLengthExpr(iter, sequence); + ComparisonNode comparison = new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, lengthExpr); + + ExpressionNode trueExpr = sequence.get(iter); + if (sequence.get(iter) instanceof ReferenceNode) { + trueExpr = createTokenExtractExpr(iter, sequence); + } + + ExpressionNode falseExpr; + if (iter < sequence.size() - 1) { + falseExpr = createTokenSequenceExpr(iter + 1, sequence); + } else { + falseExpr = ZERO; // 0-padding for rest of sequence + } + + return new IfNode(comparison, trueExpr, falseExpr); + } + + /** + * Creates an expression for the length of the token sequence so far, where + * the lengths of CLS and SEP are 1, and the length of the arguments are + * calculated using auxiliary functions. + */ + private ExpressionNode createLengthExpr(int iter, List sequence) { + List factors = new ArrayList<>(); + List operators = new ArrayList<>(); + for (int i = 0; i < iter + 1; ++i) { + if (sequence.get(i) instanceof ConstantNode) { + factors.add(ONE); + } else if (sequence.get(i) instanceof ReferenceNode) { + factors.add(new ReferenceNode(lengthFunctionName((ReferenceNode) sequence.get(i)))); + } + if (i >= 1) { + operators.add(ArithmeticOperator.PLUS); + } + } + return new ArithmeticNode(factors, operators); + } + + /** + * Create the slice expression to extract the tokens from arguments + */ + private ExpressionNode createTokenExtractExpr(int iter, List sequence) { + ExpressionNode expr; + if (iter >= 1) { + ExpressionNode lengthExpr = new EmbracedNode(createLengthExpr(iter - 1, sequence)); + expr = new EmbracedNode(new ArithmeticNode(new ReferenceNode("d1"), ArithmeticOperator.MINUS, lengthExpr)); + } else { + expr = new ReferenceNode("d1"); + } + List> slices = List.of(new Slice.DimensionValue<>("d0", wrapScalar(expr)) ); + TensorFunction argument = new TensorFunctionNode.ExpressionTensorFunction(sequence.get(iter)); + return new TensorFunctionNode(new Slice<>(argument, slices)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/XgboostFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/XgboostFeatureConverter.java new file mode 100644 index 00000000000..b05f9ba9166 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/XgboostFeatureConverter.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the xgboost(model-path) + * pseudofeature with the native Vespa ranking expression implementing + * the same computation. + * + * @author grace-lam + * @author bratseth + */ +public class XgboostFeatureConverter extends ExpressionTransformer { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map convertedXGBoostModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("xgboost")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedXGBoostModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use XGBoost model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.size() != 1) + throw new IllegalArgumentException("An xgboost node must take a single argument pointing to " + + "the xgboost model directory under [application]/models"); + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/AliasOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AliasOperation.java new file mode 100644 index 00000000000..b5648dde4fc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AliasOperation.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class AliasOperation implements FieldOperation { + + private String aliasedName; + private String alias; + + public AliasOperation(String aliasedName, String alias) { + this.aliasedName = aliasedName; + this.alias = alias; + } + + public String getAliasedName() { + return aliasedName; + } + + public void setAliasedName(String aliasedName) { + this.aliasedName = aliasedName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public void apply(SDField field) { + if (aliasedName == null) { + aliasedName = field.getName(); + } + field.getAliasToName().put(alias, aliasedName); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AttributeOperation.java new file mode 100644 index 00000000000..3983137129d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AttributeOperation.java @@ -0,0 +1,172 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.tensor.TensorType; + +import java.util.Locale; +import java.util.Optional; + +/** + * @author Einar M R Rosenvinge + */ +public class AttributeOperation implements FieldOperation, FieldOperationContainer { + + private final String name; + private Boolean huge; + private Boolean fastSearch; + private Boolean fastAccess; + private Boolean mutable; + private Boolean paged; + private Boolean enableBitVectors; + private Boolean enableOnlyBitVector; + //TODO: Remember sorting!! + private boolean doAlias = false; + private String alias; + private String aliasedName; + private Optional tensorType = Optional.empty(); + private Optional distanceMetric = Optional.empty(); + + public AttributeOperation(String name) { + this.name = name; + } + + @Override + public void addOperation(FieldOperation op) { + //TODO: Implement this method: + + } + + @Override + public void applyOperations(SDField field) { + //TODO: Implement this method: + } + + @Override + public String getName() { + return name; + } + + public Boolean getHuge() { + return huge; + } + + public void setHuge(Boolean huge) { + this.huge = huge; + } + + public Boolean getFastSearch() { + return fastSearch; + } + + public void setFastSearch(Boolean fastSearch) { + this.fastSearch = fastSearch; + } + + public Boolean getFastAccess() { + return fastAccess; + } + + public void setFastAccess(Boolean fastAccess) { + this.fastAccess = fastAccess; + } + public void setMutable(Boolean mutable) { + this.mutable = mutable; + } + public void setPaged(Boolean paged) { + this.paged = paged; + } + + public Boolean getEnableBitVectors() { + return enableBitVectors; + } + + public void setEnableBitVectors(Boolean enableBitVectors) { + this.enableBitVectors = enableBitVectors; + } + + public Boolean getEnableOnlyBitVector() { + return enableOnlyBitVector; + } + + public void setEnableOnlyBitVector(Boolean enableOnlyBitVector) { + this.enableOnlyBitVector = enableOnlyBitVector; + } + + public void setDoAlias(boolean doAlias) { + this.doAlias = doAlias; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + + public void setAliasedName(String aliasedName) { + this.aliasedName = aliasedName; + } + + public void setTensorType(TensorType tensorType) { + this.tensorType = Optional.of(tensorType); + } + + public void setDistanceMetric(String value) { + this.distanceMetric = Optional.of(value); + } + + public void apply(SDField field) { + Attribute attribute = null; + if (attributeIsSuffixOfStructField(field.getName())) { + attribute = field.getAttributes().get(field.getName()); + } + if (attribute == null) { + attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } + } + + if (huge != null) { + attribute.setHuge(huge); + } + if (paged != null) { + attribute.setPaged(paged); + } + if (fastSearch != null) { + attribute.setFastSearch(fastSearch); + } + if (fastAccess != null) { + attribute.setFastAccess(fastAccess); + } + if (mutable != null) { + attribute.setMutable(mutable); + } + if (enableBitVectors != null) { + attribute.setEnableBitVectors(enableBitVectors); + } + if (enableOnlyBitVector != null) { + attribute.setEnableOnlyBitVector(enableOnlyBitVector); + } + if (doAlias) { + field.getAliasToName().put(alias, aliasedName); + } + if (tensorType.isPresent()) { + attribute.setTensorType(tensorType.get()); + } + if (distanceMetric.isPresent()) { + String upper = distanceMetric.get().toUpperCase(Locale.ENGLISH); + attribute.setDistanceMetric(Attribute.DistanceMetric.valueOf(upper)); + } + } + + private boolean attributeIsSuffixOfStructField(String fieldName) { + return ((fieldName.indexOf('.') != -1) && fieldName.endsWith(name)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/BoldingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/BoldingOperation.java new file mode 100644 index 00000000000..74e69e047ef --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/BoldingOperation.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +/** + * @author Einar M R Rosenvinge + */ +public class BoldingOperation implements FieldOperation { + + private final boolean bold; + + public BoldingOperation(boolean bold) { + this.bold = bold; + } + + public void apply(SDField field) { + SummaryField summaryField = field.getSummaryField(field.getName(), true); + summaryField.addSource(field.getName()); + summaryField.addDestination("default"); + summaryField.setTransform(bold ? summaryField.getTransform().bold() : summaryField.getTransform().unbold()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/DictionaryOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/DictionaryOperation.java new file mode 100644 index 00000000000..a9a2ce7cbb1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/DictionaryOperation.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.SDField; + +/** + * Represents operations controlling setup of dictionary used for queries + * + * @author baldersheim + */ +public class DictionaryOperation implements FieldOperation { + public enum Operation { HASH, BTREE, CASED, UNCASED } + private final Operation operation; + + public DictionaryOperation(Operation type) { + this.operation = type; + } + @Override + public void apply(SDField field) { + Dictionary dictionary = field.getOrSetDictionary(); + switch (operation) { + case HASH: + dictionary.updateType(Dictionary.Type.HASH); + break; + case BTREE: + dictionary.updateType(Dictionary.Type.BTREE); + break; + case CASED: + dictionary.updateMatch(Case.CASED); + break; + case UNCASED: + dictionary.updateMatch(Case.UNCASED); + break; + default: + throw new IllegalArgumentException("Unhandled operation " + operation); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java new file mode 100644 index 00000000000..126f594c371 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +/** + * An operation on a field. + * Operations has a natural order of execution. + * + * @author Einar M R Rosenvinge + */ +public interface FieldOperation extends Comparable { + + /** Apply this operation on the given field */ + void apply(SDField field); + + @Override + default int compareTo(FieldOperation other) { + return 0; // no order by default + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperationContainer.java new file mode 100644 index 00000000000..d5e52fadffa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperationContainer.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public interface FieldOperationContainer { + + /** Adds an operation */ + void addOperation(FieldOperation op); + + /** Apply all operations. Operations must be sorted in their natural order before applying each operation. */ + void applyOperations(SDField field); + + String getName(); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IdOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IdOperation.java new file mode 100644 index 00000000000..5e62742085f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IdOperation.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class IdOperation implements FieldOperation { + + private SDDocumentType document; + private int fieldId; + + public SDDocumentType getDocument() { + return document; + } + + public void setDocument(SDDocumentType document) { + this.document = document; + } + + public int getFieldId() { + return fieldId; + } + + public void setFieldId(int fieldId) { + this.fieldId = fieldId; + } + + public void apply(SDField field) { + document.setFieldId(field, fieldId); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexOperation.java new file mode 100644 index 00000000000..ab5ffa25f33 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexOperation.java @@ -0,0 +1,134 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.Index; +import com.yahoo.schema.Index.Type; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; + +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; + +/** + * @author Einar M R Rosenvinge + */ +public class IndexOperation implements FieldOperation { + + private String indexName; + private Optional prefix = Optional.empty(); + private List aliases = new LinkedList<>(); + private Optional stemming = Optional.empty(); + private Optional type = Optional.empty(); + + private OptionalInt arity = OptionalInt.empty(); // For predicate data type + private OptionalLong lowerBound = OptionalLong.empty(); + private OptionalLong upperBound = OptionalLong.empty(); + private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); + private Optional enableBm25 = Optional.empty(); + + private Optional hnswIndexParams = Optional.empty(); + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public boolean getPrefix() { + return prefix.get(); + } + + public void setPrefix(Boolean prefix) { + this.prefix = Optional.of(prefix); + } + + public void addAlias(String alias) { + aliases.add(alias); + } + + public String getStemming() { + return stemming.get(); + } + + public void setStemming(String stemming) { + this.stemming = Optional.of(stemming); + } + + public void apply(SDField field) { + Index index = field.getIndex(indexName); + + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + + applyToIndex(index); + } + + public void applyToIndex(Index index) { + if (prefix.isPresent()) { + index.setPrefix(prefix.get()); + } + for (String alias : aliases) { + index.addAlias(alias); + } + if (stemming.isPresent()) { + index.setStemming(Stemming.get(stemming.get())); + } + if (type.isPresent()) { + index.setType(type.get()); + } + if (arity.isPresent() || lowerBound.isPresent() || + upperBound.isPresent() || densePostingListThreshold.isPresent()) { + index.setBooleanIndexDefiniton( + new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold)); + } + if (enableBm25.isPresent()) { + index.setInterleavedFeatures(enableBm25.get()); + } + if (hnswIndexParams.isPresent()) { + index.setHnswIndexParams(hnswIndexParams.get().build()); + } + } + + public Type getType() { + return type.get(); + } + + public void setType(Type type) { + this.type = Optional.of(type); + } + + public void setArity(int arity) { + this.arity = OptionalInt.of(arity); + } + + public void setLowerBound(long value) { + this.lowerBound = OptionalLong.of(value); + } + + public void setUpperBound(long value) { + this.upperBound = OptionalLong.of(value); + } + + public void setDensePostingListThreshold(double densePostingListThreshold) { + this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); + } + + public void setEnableBm25(boolean value) { + enableBm25 = Optional.of(value); + } + + public void setHnswIndexParams(HnswIndexParams.Builder params) { + this.hnswIndexParams = Optional.of(params); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java new file mode 100644 index 00000000000..bb79a45831e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.parser.SimpleCharStream; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.yolean.Exceptions; + +import java.util.Map; + +/** + * @author Einar M R Rosenvinge + */ +public class IndexingOperation implements FieldOperation { + + private final ScriptExpression script; + + public IndexingOperation(ScriptExpression script) { + this.script = script; + } + + public ScriptExpression getScript() { return script; } + + public void apply(SDField field) { + field.setIndexingScript(script); + } + + /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ + public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException { + return fromStream(input, multiLine, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + } + + public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, + Linguistics linguistics, Map embedders) + throws ParseException { + ScriptParserContext config = new ScriptParserContext(linguistics, embedders); + config.setAnnotatorConfig(new AnnotatorConfig()); + config.setInputStream(input); + ScriptExpression exp; + try { + if (multiLine) { + exp = ScriptExpression.newInstance(config); + } else { + exp = new ScriptExpression(StatementExpression.newInstance(config)); + } + } catch (com.yahoo.vespa.indexinglanguage.parser.ParseException e) { + ParseException t = new ParseException("Could not parse indexing statement: " + Exceptions.toMessageString(e)); + t.initCause(e); + throw t; + } + return new IndexingOperation(exp); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/MatchOperation.java new file mode 100644 index 00000000000..a568b5b0f66 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/MatchOperation.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.MatchAlgorithm; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class MatchOperation implements FieldOperation { + + private MatchType matchingType; + private Case casing; + private Integer gramSize; + private Integer maxLength; + private MatchAlgorithm matchingAlgorithm; + private String exactMatchTerminator; + + public void setMatchingType(MatchType matchingType) { + this.matchingType = matchingType; + } + + public void setGramSize(Integer gramSize) { + this.gramSize = gramSize; + } + public void setMaxLength(Integer maxLength) { + this.maxLength = maxLength; + } + + public void setMatchingAlgorithm(MatchAlgorithm matchingAlgorithm) { + this.matchingAlgorithm = matchingAlgorithm; + } + + public void setExactMatchTerminator(String exactMatchTerminator) { + this.exactMatchTerminator = exactMatchTerminator; + } + + public void setCase(Case casing) { + this.casing = casing; + } + + public void apply(SDField field) { + if (matchingType != null) { + field.setMatchingType(matchingType); + } + if (casing != null) { + field.setMatchingCase(casing); + } + if (gramSize != null) { + field.getMatching().setGramSize(gramSize); + } + if (maxLength != null) { + field.getMatching().maxLength(maxLength); + } + if (matchingAlgorithm != null) { + field.setMatchingAlgorithm(matchingAlgorithm); + } + if (exactMatchTerminator != null) { + field.getMatching().setExactMatchTerminator(exactMatchTerminator); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/NormalizingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/NormalizingOperation.java new file mode 100644 index 00000000000..561c5b87899 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/NormalizingOperation.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class NormalizingOperation implements FieldOperation { + + private final NormalizeLevel.Level level; + + public NormalizingOperation(String setting) { + if ("none".equals(setting)) { + this.level = NormalizeLevel.Level.NONE; + } else if ("codepoint".equals(setting)) { + this.level = NormalizeLevel.Level.CODEPOINT; + } else if ("lowercase".equals(setting)) { + this.level = NormalizeLevel.Level.LOWERCASE; + } else if ("accent".equals(setting)) { + this.level = NormalizeLevel.Level.ACCENT; + } else if ("all".equals(setting)) { + this.level = NormalizeLevel.Level.ACCENT; + } else { + throw new IllegalArgumentException("invalid normalizing setting: " + setting); + } + } + + public void apply(SDField field) { + field.setNormalizing(new NormalizeLevel(level, true)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/QueryCommandOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/QueryCommandOperation.java new file mode 100644 index 00000000000..d0e9feb41e4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/QueryCommandOperation.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +import java.util.List; + +/** + * @author Einar M R Rosenvinge + */ +public class QueryCommandOperation implements FieldOperation { + + private final List queryCommands = new java.util.ArrayList<>(0); + + public void addQueryCommand(String name) { + queryCommands.add(name); + } + + public void apply(SDField field) { + for (String command : queryCommands) { + field.addQueryCommand(command); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankOperation.java new file mode 100644 index 00000000000..bbc6208ba1b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankOperation.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class RankOperation implements FieldOperation { + + private Boolean literal = null; + private Boolean filter = null; + private Boolean normal = null; + + public Boolean getLiteral() { return literal; } + public void setLiteral(Boolean literal) { this.literal = literal; } + + public Boolean getFilter() { return filter; } + public void setFilter(Boolean filter) { this.filter = filter; } + + public Boolean getNormal() { return normal; } + public void setNormal(Boolean n) { this.normal = n; } + + public void apply(SDField field) { + if (literal != null) { + field.getRanking().setLiteral(literal); + } + if (filter != null) { + field.getRanking().setFilter(filter); + } + if (normal != null) { + field.getRanking().setNormal(normal); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankTypeOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankTypeOperation.java new file mode 100644 index 00000000000..4a43a907549 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankTypeOperation.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Index; + +/** + * @author Einar M R Rosenvinge + */ +public class RankTypeOperation implements FieldOperation { + + private String indexName; + private RankType type; + + public String getIndexName() { + return indexName; + } + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public RankType getType() { + return type; + } + public void setType(RankType type) { + this.type = type; + } + + public void apply(SDField field) { + if (indexName == null) { + field.setRankType(type); // Set default if the index is not specified. + } else { + Index index = field.getIndex(indexName); + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + index.setRankType(type); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/SortingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SortingOperation.java new file mode 100644 index 00000000000..2e981a893ce --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SortingOperation.java @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Sorting; + +/** + * @author Einar M R Rosenvinge + */ +public class SortingOperation implements FieldOperation { + + private final String attributeName; + private Boolean ascending; + private Boolean descending; + private Sorting.Function function; + private Sorting.Strength strength; + private String locale; + + public SortingOperation(String attributeName) { + this.attributeName = attributeName; + } + + public String getAttributeName() { + return attributeName; + } + + public Boolean getAscending() { + return ascending; + } + + public void setAscending() { + this.ascending = true; + } + + public Boolean getDescending() { + return descending; + } + + public void setDescending() { + this.descending = true; + } + + public Sorting.Function getFunction() { + return function; + } + + public void setFunction(Sorting.Function function) { + this.function = function; + } + + public Sorting.Strength getStrength() { + return strength; + } + + public void setStrength(Sorting.Strength strength) { + this.strength = strength; + } + + public String getLocale() { + return locale; + } + + public void setLocale(String locale) { + this.locale = locale; + } + + public void apply(SDField field) { + Attribute attribute = field.getAttributes().get(attributeName); + if (attribute == null) { + attribute = new Attribute(attributeName, field.getDataType()); + field.addAttribute(attribute); + } + Sorting sorting = attribute.getSorting(); + + if (ascending != null) { + sorting.setAscending(); + } + if (descending != null) { + sorting.setDescending(); + } + if (function != null) { + sorting.setFunction(function); + } + if (strength != null) { + sorting.setStrength(strength); + } + if (locale != null) { + sorting.setLocale(locale); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/StemmingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StemmingOperation.java new file mode 100644 index 00000000000..a4bb00b0d07 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StemmingOperation.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; + +/** + * @author Einar M R Rosenvinge + */ +public class StemmingOperation implements FieldOperation { + + private String setting; + + public String getSetting() { + return setting; + } + + public void setSetting(String setting) { + this.setting = setting; + } + + public void apply(SDField field) { + field.setStemming(Stemming.get(setting)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/StructFieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StructFieldOperation.java new file mode 100644 index 00000000000..ac80f5023fc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StructFieldOperation.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; + +/** + * @author Einar M R Rosenvinge + */ +public class StructFieldOperation implements FieldOperation, FieldOperationContainer { + + private final String structFieldName; + private final List pendingOperations = new LinkedList<>(); + + public StructFieldOperation(String structFieldName) { + this.structFieldName = structFieldName; + } + + public void apply(SDField field) { + SDField structField = field.getStructField(structFieldName); + if (structField == null ) { + throw new IllegalArgumentException("Struct field '" + structFieldName + "' has not been defined in struct " + + "for field '" + field.getName() + "'."); + } + + applyOperations(structField); + } + + @Override + public void addOperation(FieldOperation op) { + pendingOperations.add(op); + } + + @Override + public void applyOperations(SDField field) { + if (pendingOperations.isEmpty()) return; + + Collections.sort(pendingOperations); + ListIterator ops = pendingOperations.listIterator(); + while (ops.hasNext()) { + FieldOperation op = ops.next(); + ops.remove(); + op.apply(field); + } + } + + @Override + public String getName() { + return structFieldName; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldLongOperation.java new file mode 100644 index 00000000000..4576b7a34fe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldLongOperation.java @@ -0,0 +1,70 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.document.DataType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.Iterator; +import java.util.Set; + +/** + * @author Einar M R Rosenvinge + */ +public class SummaryInFieldLongOperation extends SummaryInFieldOperation { + + private DataType type; + private Boolean bold; + private Set destinations = new java.util.LinkedHashSet<>(); + + public SummaryInFieldLongOperation(String name) { + super(name); + } + + public SummaryInFieldLongOperation() { + super(null); + } + + public void setType(DataType type) { + this.type = type; + } + + public void setBold(Boolean bold) { + this.bold = bold; + } + + public void addDestination(String destination) { + destinations.add(destination); + } + + public Iterator destinationIterator() { + return destinations.iterator(); + } + + public void apply(SDField field) { + if (type == null) { + type = field.getDataType(); + } + SummaryField summary = new SummaryField(name, type); + applyToSummary(summary); + field.addSummaryField(summary); + } + + public void applyToSummary(SummaryField summary) { + if (transform != null) { + summary.setTransform(transform); + } + + if (bold != null) { + summary.setTransform(bold ? summary.getTransform().bold() : summary.getTransform().unbold()); + } + + for (SummaryField.Source source : sources) { + summary.addSource(source); + } + + for (String destination : destinations) { + summary.addDestination(destination); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldOperation.java new file mode 100644 index 00000000000..dd06d920aac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldOperation.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import java.util.Set; + +/** + * @author Einar M R Rosenvinge + */ +public abstract class SummaryInFieldOperation implements FieldOperation { + + protected String name; + protected SummaryTransform transform; + protected Set sources = new java.util.LinkedHashSet<>(); + + public SummaryInFieldOperation(String name) { + this.name = name; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setTransform(SummaryTransform transform) { + this.transform = transform; + } + + public SummaryTransform getTransform() { + return transform; + } + + public void addSource(String name) { + sources.add(new SummaryField.Source(name)); + } + + public void addSource(SummaryField.Source source) { + sources.add(source); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldShortOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldShortOperation.java new file mode 100644 index 00000000000..ccc22719f25 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldShortOperation.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +/** + * @author Einar M R Rosenvinge + */ +public class SummaryInFieldShortOperation extends SummaryInFieldOperation { + + public SummaryInFieldShortOperation(String name) { + super(name); + } + + public void apply(SDField field) { + SummaryField ret = field.getSummaryField(name); + if (ret == null) { + ret = new SummaryField(name, field.getDataType()); + ret.addSource(field.getName()); + ret.addDestination("default"); + } + ret.setImplicit(false); + + ret.setTransform(transform); + for (SummaryField.Source source : sources) { + ret.addSource(source); + } + field.addSummaryField(ret); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryToOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryToOperation.java new file mode 100644 index 00000000000..2d9cf3acf4e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryToOperation.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.Set; + +/** + * @author Einar M R Rosenvinge + */ +public class SummaryToOperation implements FieldOperation { + + private Set destinations = new java.util.LinkedHashSet<>(); + private String name; + + public void setName(String name) { + this.name = name; + } + + public void addDestination(String destination) { + destinations.add(destination); + } + + public void apply(SDField field) { + SummaryField summary; + summary = field.getSummaryField(name); + if (summary == null) { + summary = new SummaryField(field); + summary.addSource(field.getName()); + summary.addDestination("default"); + field.addSummaryField(summary); + } + summary.setImplicit(false); + + for (String destination : destinations) { + summary.addDestination(destination); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightOperation.java new file mode 100644 index 00000000000..57c28d9bdb5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightOperation.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.SDField; + +/** + * @author Einar M R Rosenvinge + */ +public class WeightOperation implements FieldOperation { + + private int weight; + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } + + public void apply(SDField field) { + field.setWeight(weight); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightedSetOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightedSetOperation.java new file mode 100644 index 00000000000..8fb0cc9fcdb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightedSetOperation.java @@ -0,0 +1,70 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.document.Attribute; +import com.yahoo.document.DataType; +import com.yahoo.schema.document.SDField; +import com.yahoo.document.WeightedSetDataType; + +/** + * @author Einar M R Rosenvinge + */ +public class WeightedSetOperation implements FieldOperation { + + private Boolean createIfNonExistent; + private Boolean removeIfZero; + + public Boolean getCreateIfNonExistent() { + return createIfNonExistent; + } + + public void setCreateIfNonExistent(Boolean createIfNonExistent) { + this.createIfNonExistent = createIfNonExistent; + } + + public Boolean getRemoveIfZero() { + return removeIfZero; + } + + public void setRemoveIfZero(Boolean removeIfZero) { + this.removeIfZero = removeIfZero; + } + + public void apply(SDField field) { + WeightedSetDataType ctype = (WeightedSetDataType) field.getDataType(); + + if (createIfNonExistent != null) { + field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), createIfNonExistent, + ctype.removeIfZero())); + } + + ctype = (WeightedSetDataType) field.getDataType(); + if (removeIfZero != null) { + field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), + ctype.createIfNonExistent(), removeIfZero)); + } + + ctype = (WeightedSetDataType) field.getDataType(); + for (Object o : field.getAttributes().values()) { + Attribute attribute = (Attribute) o; + attribute.setRemoveIfZero(ctype.removeIfZero()); + attribute.setCreateIfNonExistent(ctype.createIfNonExistent()); + } + } + + @Override + public int compareTo(FieldOperation other) { + // this operation should be executed first because it modifies the type of weighted sets, and other + // operation depends on the type of the weighted set + return -1; + } + + @Override + public String toString() { + return "WeightedSetOperation{" + + "createIfNonExistent=" + createIfNonExistent + + ", removeIfZero=" + removeIfZero + + '}'; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java new file mode 100644 index 00000000000..fa656b72530 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -0,0 +1,331 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Sorting; +import com.yahoo.schema.document.annotation.SDAnnotationType; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import java.util.Locale; + +/** + * Helper for converting ParsedField etc to SDField with settings + * + * @author arnej27959 + **/ +public class ConvertParsedFields { + + private final TypeResolver context; + + ConvertParsedFields(TypeResolver context) { + this.context = context; + } + + static void convertMatchSettings(SDField field, ParsedMatchSettings parsed) { + parsed.getMatchType().ifPresent(matchingType -> field.setMatchingType(matchingType)); + parsed.getMatchCase().ifPresent(casing -> field.setMatchingCase(casing)); + parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); + parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); + parsed.getMatchAlgorithm().ifPresent + (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); + parsed.getExactTerminator().ifPresent + (exactMatchTerminator -> field.getMatching().setExactMatchTerminator(exactMatchTerminator)); + } + + void convertSorting(SDField field, ParsedSorting parsed, String name) { + Attribute attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } + Sorting sorting = attribute.getSorting(); + if (parsed.getAscending()) { + sorting.setAscending(); + } else { + sorting.setDescending(); + } + parsed.getFunction().ifPresent(function -> sorting.setFunction(function)); + parsed.getStrength().ifPresent(strength -> sorting.setStrength(strength)); + parsed.getLocale().ifPresent(locale -> sorting.setLocale(locale)); + } + + void convertAttribute(SDField field, ParsedAttribute parsed) { + String name = parsed.name(); + String fieldName = field.getName(); + Attribute attribute = null; + if (fieldName.endsWith("." + name)) { + attribute = field.getAttributes().get(field.getName()); + } + if (attribute == null) { + attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } + } + attribute.setHuge(parsed.getHuge()); + attribute.setPaged(parsed.getPaged()); + attribute.setFastSearch(parsed.getFastSearch()); + if (parsed.getFastRank()) { + attribute.setFastRank(parsed.getFastRank()); + } + attribute.setFastAccess(parsed.getFastAccess()); + attribute.setMutable(parsed.getMutable()); + attribute.setEnableBitVectors(parsed.getEnableBitVectors()); + attribute.setEnableOnlyBitVector(parsed.getEnableOnlyBitVector()); + + // attribute.setTensorType(?) + + for (String alias : parsed.getAliases()) { + field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); + } + var distanceMetric = parsed.getDistanceMetric(); + if (distanceMetric.isPresent()) { + String upper = distanceMetric.get().toUpperCase(Locale.ENGLISH); + attribute.setDistanceMetric(Attribute.DistanceMetric.valueOf(upper)); + } + var sorting = parsed.getSorting(); + if (sorting.isPresent()) { + convertSorting(field, sorting.get(), name); + } + } + + private void convertRankType(SDField field, String indexName, String rankType) { + RankType type = RankType.fromString(rankType); + if (indexName == null || indexName.equals("")) { + field.setRankType(type); // Set default if the index is not specified. + } else { + Index index = field.getIndex(indexName); + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + index.setRankType(type); + } + } + + private void convertNormalizing(SDField field, String setting) { + NormalizeLevel.Level level; + if ("none".equals(setting)) { + level = NormalizeLevel.Level.NONE; + } else if ("codepoint".equals(setting)) { + level = NormalizeLevel.Level.CODEPOINT; + } else if ("lowercase".equals(setting)) { + level = NormalizeLevel.Level.LOWERCASE; + } else if ("accent".equals(setting)) { + level = NormalizeLevel.Level.ACCENT; + } else if ("all".equals(setting)) { + level = NormalizeLevel.Level.ACCENT; + } else { + throw new IllegalArgumentException("invalid normalizing setting: " + setting); + } + field.setNormalizing(new NormalizeLevel(level, true)); + } + + // from grammar, things that can be inside struct-field block + private void convertCommonFieldSettings(SDField field, ParsedField parsed) { + convertMatchSettings(field, parsed.matchSettings()); + var indexing = parsed.getIndexing(); + if (indexing.isPresent()) { + field.setIndexingScript(indexing.get().script()); + } + parsed.getWeight().ifPresent(value -> field.setWeight(value)); + parsed.getStemming().ifPresent(value -> field.setStemming(value)); + parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); + for (var attribute : parsed.getAttributes()) { + convertAttribute(field, attribute); + } + for (var summaryField : parsed.getSummaryFields()) { + var dataType = field.getDataType(); + var otherType = summaryField.getType(); + if (otherType != null) { + dataType = context.resolveType(otherType); + } + convertSummaryField(field, summaryField, dataType); + } + for (String command : parsed.getQueryCommands()) { + field.addQueryCommand(command); + } + for (var structField : parsed.getStructFields()) { + convertStructField(field, structField); + } + if (parsed.hasLiteral()) { + field.getRanking().setLiteral(true); + } + if (parsed.hasFilter()) { + field.getRanking().setFilter(true); + } + if (parsed.hasNormal()) { + field.getRanking().setNormal(true); + } + } + + private void convertStructField(SDField field, ParsedField parsed) { + SDField structField = field.getStructField(parsed.name()); + if (structField == null ) { + throw new IllegalArgumentException("Struct field '" + parsed.name() + "' has not been defined in struct " + + "for field '" + field.getName() + "'."); + } + convertCommonFieldSettings(structField, parsed); + } + + private void convertExtraFieldSettings(SDField field, ParsedField parsed) { + String name = parsed.name(); + for (var dictOp : parsed.getDictionaryOptions()) { + var dictionary = field.getOrSetDictionary(); + switch (dictOp) { + case HASH: dictionary.updateType(Dictionary.Type.HASH); break; + case BTREE: dictionary.updateType(Dictionary.Type.BTREE); break; + case CASED: dictionary.updateMatch(Case.CASED); break; + case UNCASED: dictionary.updateMatch(Case.UNCASED); break; + } + } + for (var index : parsed.getIndexes()) { + convertIndex(field, index); + } + for (var alias : parsed.getAliases()) { + field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); + } + parsed.getRankTypes().forEach((indexName, rankType) -> convertRankType(field, indexName, rankType)); + parsed.getSorting().ifPresent(sortInfo -> convertSorting(field, sortInfo, name)); + if (parsed.hasBolding()) { + // TODO must it be so ugly: + SummaryField summaryField = field.getSummaryField(name, true); + summaryField.addSource(name); + summaryField.addDestination("default"); + summaryField.setTransform(summaryField.getTransform().bold()); + } + } + + static void convertSummaryFieldSettings(SummaryField summary, ParsedSummaryField parsed) { + var transform = SummaryTransform.NONE; + if (parsed.getMatchedElementsOnly()) { + transform = SummaryTransform.MATCHED_ELEMENTS_FILTER; + } else if (parsed.getDynamic()) { + transform = SummaryTransform.DYNAMICTEASER; + } + if (parsed.getBolded()) { + transform = transform.bold(); + } + summary.setTransform(transform); + for (String source : parsed.getSources()) { + summary.addSource(source); + } + for (String destination : parsed.getDestinations()) { + summary.addDestination(destination); + } + summary.setImplicit(false); + } + + private void convertSummaryField(SDField field, ParsedSummaryField parsed, DataType type) { + var summary = new SummaryField(parsed.name(), type); + convertSummaryFieldSettings(summary, parsed); + summary.addDestination("default"); + if (parsed.getSources().isEmpty()) { + summary.addSource(field.getName()); + } + field.addSummaryField(summary); + } + + private void convertIndex(SDField field, ParsedIndex parsed) { + String indexName = parsed.name(); + Index index = field.getIndex(indexName); + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + convertIndexSettings(index, parsed); + } + + private void convertIndexSettings(Index index, ParsedIndex parsed) { + parsed.getPrefix().ifPresent(prefix -> index.setPrefix(prefix)); + for (String alias : parsed.getAliases()) { + index.addAlias(alias); + } + parsed.getStemming().ifPresent(stemming -> index.setStemming(stemming)); + var arity = parsed.getArity(); + var lowerBound = parsed.getLowerBound(); + var upperBound = parsed.getUpperBound(); + var densePostingListThreshold = parsed.getDensePostingListThreshold(); + if (arity.isPresent() || + lowerBound.isPresent() || + upperBound.isPresent() || + densePostingListThreshold.isPresent()) + { + var bid = new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold); + index.setBooleanIndexDefiniton(bid); + } + parsed.getEnableBm25().ifPresent(enableBm25 -> index.setInterleavedFeatures(enableBm25)); + parsed.getHnswIndexParams().ifPresent + (hnswIndexParams -> index.setHnswIndexParams(hnswIndexParams)); + } + + SDField convertDocumentField(Schema schema, SDDocumentType document, ParsedField parsed) { + String name = parsed.name(); + DataType dataType = context.resolveType(parsed.getType()); + var field = new SDField(document, name, dataType); + convertCommonFieldSettings(field, parsed); + convertExtraFieldSettings(field, parsed); + document.addField(field); + return field; + } + + void convertExtraField(Schema schema, ParsedField parsed) { + String name = parsed.name(); + DataType dataType = context.resolveType(parsed.getType()); + var field = new SDField(schema.getDocument(), name, dataType); + convertCommonFieldSettings(field, parsed); + convertExtraFieldSettings(field, parsed); + schema.addExtraField(field); + } + + void convertExtraIndex(Schema schema, ParsedIndex parsed) { + Index index = new Index(parsed.name()); + convertIndexSettings(index, parsed); + schema.addIndex(index); + } + + SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) { + // TODO - can we cleanup this mess + var structProxy = new SDDocumentType(parsed.name(), schema); + for (var parsedField : parsed.getFields()) { + var fieldType = context.resolveType(parsedField.getType()); + var field = new SDField(document, parsedField.name(), fieldType); + convertCommonFieldSettings(field, parsedField); + structProxy.addField(field); + if (parsedField.hasIdOverride()) { + structProxy.setFieldId(field, parsedField.idOverride()); + } + } + for (String inherit : parsed.getInherited()) { + structProxy.inherit(new DataTypeName(inherit)); + } + structProxy.setStruct(context.resolveStruct(parsed)); + return structProxy; + } + + void convertAnnotation(Schema schema, SDDocumentType document, ParsedAnnotation parsed) { + SDAnnotationType annType = context.resolveAnnotation(parsed.name()); + var withStruct = parsed.getStruct(); + if (withStruct.isPresent()) { + ParsedStruct parsedStruct = withStruct.get(); + SDDocumentType structProxy = convertStructDeclaration(schema, document, parsedStruct); + structProxy.setStruct(context.resolveStruct(parsedStruct)); + annType.setSdDocType(structProxy); + } + document.addAnnotation(annType); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java new file mode 100644 index 00000000000..bd628779b24 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -0,0 +1,124 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; + +import java.util.List; + +/** + * Helper for converting ParsedRankProfile etc to RankProfile with settings + * + * @author arnej27959 + */ +public class ConvertParsedRanking { + + private final RankProfileRegistry rankProfileRegistry; + + // for unit test + ConvertParsedRanking() { + this(new RankProfileRegistry()); + } + + public ConvertParsedRanking(RankProfileRegistry rankProfileRegistry) { + this.rankProfileRegistry = rankProfileRegistry; + } + + private RankProfile makeRankProfile(Schema schema, String name) { + if (name.equals("default")) { + return rankProfileRegistry.get(schema, "default"); + } + return new RankProfile(name, schema, rankProfileRegistry); + } + + void convertRankProfile(Schema schema, ParsedRankProfile parsed) { + RankProfile profile = makeRankProfile(schema, parsed.name()); + for (String name : parsed.getInherited()) + profile.inherit(name); + + parsed.isStrict().ifPresent(value -> profile.setStrict(value)); + + for (var constant : parsed.getConstants().values()) + profile.add(constant); + + for (var onnxModel : parsed.getOnnxModels()) + profile.add(onnxModel); + + for (var input : parsed.getInputs().entrySet()) + profile.addInput(input.getKey(), input.getValue()); + + for (var func : parsed.getFunctions()) { + String name = func.name(); + List parameters = func.getParameters(); + String expression = func.getExpression(); + boolean inline = func.getInline(); + profile.addFunction(name, parameters, expression, inline); + } + + parsed.getRankScoreDropLimit().ifPresent + (value -> profile.setRankScoreDropLimit(value)); + parsed.getTermwiseLimit().ifPresent + (value -> profile.setTermwiseLimit(value)); + parsed.getPostFilterThreshold().ifPresent + (value -> profile.setPostFilterThreshold(value)); + parsed.getApproximateThreshold().ifPresent + (value -> profile.setApproximateThreshold(value)); + parsed.getKeepRankCount().ifPresent + (value -> profile.setKeepRankCount(value)); + parsed.getMinHitsPerThread().ifPresent + (value -> profile.setMinHitsPerThread(value)); + parsed.getNumSearchPartitions().ifPresent + (value -> profile.setNumSearchPartitions(value)); + parsed.getNumThreadsPerSearch().ifPresent + (value -> profile.setNumThreadsPerSearch(value)); + parsed.getReRankCount().ifPresent + (value -> profile.setRerankCount(value)); + + parsed.getMatchPhaseSettings().ifPresent + (value -> profile.setMatchPhaseSettings(value)); + + parsed.getFirstPhaseExpression().ifPresent + (value -> profile.setFirstPhaseRanking(value)); + parsed.getSecondPhaseExpression().ifPresent + (value -> profile.setSecondPhaseRanking(value)); + + for (var value : parsed.getMatchFeatures()) { + profile.addMatchFeatures(value); + } + for (var value : parsed.getRankFeatures()) { + profile.addRankFeatures(value); + } + for (var value : parsed.getSummaryFeatures()) { + profile.addSummaryFeatures(value); + } + + parsed.getInheritedMatchFeatures().ifPresent + (value -> profile.setInheritedMatchFeatures(value)); + parsed.getInheritedSummaryFeatures().ifPresent + (value -> profile.setInheritedSummaryFeatures(value)); + if (parsed.getIgnoreDefaultRankFeatures()) { + profile.setIgnoreDefaultRankFeatures(true); + } + + for (var mutateOp : parsed.getMutateOperations()) { + profile.addMutateOperation(mutateOp); + } + parsed.getFieldsWithRankFilter().forEach + ((fieldName, isFilter) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.PREFERBITVECTOR, isFilter)); + + parsed.getFieldsWithRankWeight().forEach + ((fieldName, weight) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.WEIGHT, weight)); + + parsed.getFieldsWithRankType().forEach + ((fieldName, rankType) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.fromString(rankType))); + + parsed.getRankProperties().forEach + ((key, values) -> {for (String value : values) profile.addRankProperty(key, value);}); + + // always? + rankProfileRegistry.add(profile); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java new file mode 100644 index 00000000000..f3289621ce1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -0,0 +1,221 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.schema.DefaultRankProfile; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.UnrankedRankProfile; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; +import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Class converting a collection of schemas from the intermediate format. + * + * @author arnej27959 + **/ +public class ConvertParsedSchemas { + + private final List orderedInput; + private final DocumentTypeManager docMan; + private final ApplicationPackage applicationPackage; + private final FileRegistry fileRegistry; + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + private final RankProfileRegistry rankProfileRegistry; + private final boolean documentsOnly; + private final ConvertParsedTypes typeConverter; + + // for unit test + ConvertParsedSchemas(List orderedInput, + DocumentTypeManager documentTypeManager) + { + this(orderedInput, documentTypeManager, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new BaseDeployLogger(), + new TestProperties(), + new RankProfileRegistry(), + true); + } + + public ConvertParsedSchemas(List orderedInput, + DocumentTypeManager documentTypeManager, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) + { + this.orderedInput = orderedInput; + this.docMan = documentTypeManager; + this.applicationPackage = applicationPackage; + this.fileRegistry = fileRegistry; + this.deployLogger = deployLogger; + this.properties = properties; + this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; + this.typeConverter = new ConvertParsedTypes(orderedInput, docMan); + } + + private final Map convertedDocuments = new LinkedHashMap<>(); + + public List convertToSchemas() { + typeConverter.convert(false); + var resultList = new ArrayList(); + for (var parsed : orderedInput) { + Optional inherited; + var inheritList = parsed.getInherited(); + if (inheritList.size() == 0) { + inherited = Optional.empty(); + } else if (inheritList.size() == 1) { + inherited = Optional.of(inheritList.get(0)); + } else { + throw new IllegalArgumentException("schema " + parsed.name() + "cannot inherit more than once"); + } + Schema schema = parsed.getDocumentWithoutSchema() + ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) + : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + convertSchema(schema, parsed); + resultList.add(schema); + } + return resultList; + } + + private void convertDocument(Schema schema, ParsedDocument parsed, + ConvertParsedFields fieldConverter) + { + SDDocumentType document = new SDDocumentType(parsed.name()); + for (var struct : parsed.getStructs()) { + var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); + document.addType(structProxy); + } + for (String inherit : parsed.getInherited()) { + var parent = convertedDocuments.get(inherit); + assert(parent != null); + document.inherit(parent); + } + for (var annotation : parsed.getAnnotations()) { + fieldConverter.convertAnnotation(schema, document, annotation); + } + for (var field : parsed.getFields()) { + var sdf = fieldConverter.convertDocumentField(schema, document, field); + if (field.hasIdOverride()) { + document.setFieldId(sdf, field.idOverride()); + } + } + convertedDocuments.put(parsed.name(), document); + schema.addDocument(document); + } + + private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { + var docsum = new DocumentSummary(parsed.name(), schema); + var inheritList = parsed.getInherited(); + if (inheritList.size() == 1) { + docsum.setInherited(inheritList.get(0)); + } else if (inheritList.size() != 0) { + throw new IllegalArgumentException("document-summary "+parsed.name()+" cannot inherit more than once"); + } + if (parsed.getFromDisk()) { + docsum.setFromDisk(true); + } + if (parsed.getOmitSummaryFeatures()) { + docsum.setOmitSummaryFeatures(true); + } + for (var parsedField : parsed.getSummaryFields()) { + DataType dataType = typeContext.resolveType(parsedField.getType()); + var summaryField = new SummaryField(parsedField.name(), dataType); + // XXX does not belong here: + summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); + ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); + docsum.add(summaryField); + } + schema.addSummary(docsum); + } + + private void convertImportField(Schema schema, ParsedSchema.ImportedField f) { + // needs rethinking + var importedFields = schema.temporaryImportedFields().get(); + if (importedFields.hasField(f.asFieldName)) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', import field as '" + f.asFieldName + + "': Field already imported"); + } + importedFields.add(new TemporaryImportedField(f.asFieldName, f.refFieldName, f.foreignFieldName)); + } + + private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { + String setName = parsed.name(); + for (String field : parsed.getFieldNames()) { + schema.fieldSets().addUserFieldSetItem(setName, field); + } + for (String command : parsed.getQueryCommands()) { + schema.fieldSets().userFieldSets().get(setName).queryCommands().add(command); + } + if (parsed.getMatchSettings().isPresent()) { + // same ugliness as SDParser.jj used to have: + var tmp = new SDField(setName, DataType.STRING); + ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); + schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); + } + } + + private void convertSchema(Schema schema, ParsedSchema parsed) { + if (parsed.hasStemming()) { + schema.setStemming(parsed.getStemming()); + } + parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); + var typeContext = typeConverter.makeContext(parsed.getDocument()); + var fieldConverter = new ConvertParsedFields(typeContext); + convertDocument(schema, parsed.getDocument(), fieldConverter); + for (var field : parsed.getFields()) { + fieldConverter.convertExtraField(schema, field); + } + for (var index : parsed.getIndexes()) { + fieldConverter.convertExtraIndex(schema, index); + } + for (var docsum : parsed.getDocumentSummaries()) { + convertDocumentSummary(schema, docsum, typeContext); + } + for (var importedField : parsed.getImportedFields()) { + convertImportField(schema, importedField); + } + for (var fieldSet : parsed.getFieldSets()) { + convertFieldSet(schema, fieldSet); + } + if (documentsOnly) { + return; // skip ranking-only content, not used for document type generation + } + for (var constant : parsed.getConstants()) + schema.add(constant); + for (var onnxModel : parsed.getOnnxModels()) + schema.add(onnxModel); + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); + var rankConverter = new ConvertParsedRanking(rankProfileRegistry); + for (var rankProfile : parsed.getRankProfiles()) { + rankConverter.convertRankProfile(schema, rankProfile); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java new file mode 100644 index 00000000000..9f1203ffc9f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java @@ -0,0 +1,337 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.documentmodel.OwnedStructDataType; +import com.yahoo.schema.document.annotation.SDAnnotationType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Helper class for converting ParsedType instances to DataType + * + * @author arnej27959 + **/ +public class ConvertParsedTypes { + + private final List orderedInput; + private final DocumentTypeManager docMan; + + ConvertParsedTypes(List input) { + this.orderedInput = input; + this.docMan = new DocumentTypeManager(); + } + + public ConvertParsedTypes(List input, DocumentTypeManager docMan) { + this.orderedInput = input; + this.docMan = docMan; + } + + public void convert(boolean andRegister) { + startDataTypes(); + fillDataTypes(); + if (andRegister) { + registerDataTypes(); + } + } + + private Map documentsFromSchemas = new HashMap<>(); + private Map structsFromSchemas = new HashMap<>(); + private Map annotationsFromSchemas = new HashMap<>(); + + private void startDataTypes() { + for (var schema : orderedInput) { + String name = schema.getDocument().name(); + documentsFromSchemas.put(name, new DocumentType(name)); + } + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var struct : doc.getStructs()) { + String structId = doc.name() + "->" + struct.name(); + var dt = new OwnedStructDataType(struct.name(), doc.name()); + structsFromSchemas.put(structId, dt); + } + for (var annotation : doc.getAnnotations()) { + String annId = doc.name() + "->" + annotation.name(); + var at = new SDAnnotationType(annotation.name()); + annotationsFromSchemas.put(annId, at); + for (String inherit : annotation.getInherited()) { + at.inherit(inherit); + } + var withStruct = annotation.getStruct(); + if (withStruct.isPresent()) { + ParsedStruct struct = withStruct.get(); + String structId = doc.name() + "->" + struct.name(); + var old = structsFromSchemas.put(structId, new OwnedStructDataType(struct.name(), doc.name())); + assert(old == null); + } + } + } + } + + void fillAnnotationStruct(ParsedAnnotation annotation) { + var withStruct = annotation.getStruct(); + if (withStruct.isPresent()) { + var doc = annotation.getOwnerDoc(); + var toFill = findStructFromParsed(withStruct.get()); + for (ParsedField field : withStruct.get().getFields()) { + var t = resolveFromContext(field.getType(), doc); + var f = field.hasIdOverride() + ? new com.yahoo.document.Field(field.name(), field.idOverride(), t) + : new com.yahoo.document.Field(field.name(), t); + toFill.addField(f); + } + for (var parent : annotation.getResolvedInherits()) { + parent.getStruct().ifPresent + (ps -> toFill.inherit(findStructFromParsed(ps))); + } + var at = findAnnotationFromParsed(annotation); + at.setDataType(toFill); + } + } + + private void fillDataTypes() { + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var annotation : doc.getAnnotations()) { + var at = findAnnotationFromParsed(annotation); + for (var parent : annotation.getResolvedInherits()) { + at.inherit(findAnnotationFromParsed(parent)); + } + fillAnnotationStruct(annotation); + } + for (var struct : doc.getStructs()) { + var toFill = findStructFromParsed(struct); + // evil ugliness + for (ParsedField field : struct.getFields()) { + if (! field.hasIdOverride()) { + var t = resolveFromContext(field.getType(), doc); + var f = new com.yahoo.document.Field(field.name(), t); + toFill.addField(f); + } + } + for (ParsedField field : struct.getFields()) { + if (field.hasIdOverride()) { + var t = resolveFromContext(field.getType(), doc); + var f = new com.yahoo.document.Field(field.name(), field.idOverride(), t); + toFill.addField(f); + } + } + for (var inherit : struct.getResolvedInherits()) { + var parent = findStructFromParsed(inherit); + // ensure a nice, compatible exception message + for (var field : toFill.getFields()) { + if (parent.hasField(field)) { + for (var base : parent.getInheritedTypes()) { + if (base.hasField(field)) { + parent = base; + } + } + throw new IllegalArgumentException + ("In document " + doc.name() + ": struct " + struct.name() + + " cannot inherit from " + parent.getName() + " and redeclare field " + field.getName()); + } + } + toFill.inherit(parent); + } + } + var docToFill = documentsFromSchemas.get(doc.name()); + Map> fieldSets = new HashMap<>(); + List inDocFields = new ArrayList<>(); + for (var docField : doc.getFields()) { + String name = docField.name(); + var t = resolveFromContext(docField.getType(), doc); + var f = new com.yahoo.document.Field(docField.name(), t); + docToFill.addField(f); + if (docField.hasIdOverride()) { + f.setId(docField.idOverride(), docToFill); + } + inDocFields.add(name); + } + fieldSets.put("[document]", inDocFields); + for (var extraField : schema.getFields()) { + String name = extraField.name(); + if (docToFill.hasField(name)) continue; + var t = resolveFromContext(extraField.getType(), doc); + var f = new com.yahoo.document.Field(name, t); + docToFill.addField(f); + } + for (var fieldset : schema.getFieldSets()) { + fieldSets.put(fieldset.name(), fieldset.getFieldNames()); + } + docToFill.addFieldSets(fieldSets); + for (String inherit : doc.getInherited()) { + docToFill.inherit(findDocFromSchemas(inherit)); + } + } + } + + private StructDataType findStructFromParsed(ParsedStruct resolved) { + String structId = resolved.getOwnerName() + "->" + resolved.name(); + var struct = structsFromSchemas.get(structId); + assert(struct != null); + return struct; + } + + private StructDataType findStructFromSchemas(String name, ParsedDocument context) { + var resolved = context.findParsedStruct(name); + if (resolved == null) { + throw new IllegalArgumentException("no struct named " + name + " in context " + context); + } + return findStructFromParsed(resolved); + } + + private SDAnnotationType findAnnotationFromSchemas(String name, ParsedDocument context) { + var resolved = context.findParsedAnnotation(name); + String annotationId = resolved.getOwnerName() + "->" + resolved.name(); + var annotation = annotationsFromSchemas.get(annotationId); + if (annotation == null) { + throw new IllegalArgumentException("no annotation named " + name + " in context " + context); + } + return annotation; + } + + private SDAnnotationType findAnnotationFromParsed(ParsedAnnotation resolved) { + String annotationId = resolved.getOwnerName() + "->" + resolved.name(); + var annotation = annotationsFromSchemas.get(annotationId); + if (annotation == null) { + throw new IllegalArgumentException("no annotation " + resolved.name() + " in " + resolved.getOwnerName()); + } + return annotation; + } + + private DataType createArray(ParsedType pType, ParsedDocument context) { + DataType nested = resolveFromContext(pType.nestedType(), context); + return DataType.getArray(nested); + } + + private DataType createWset(ParsedType pType, ParsedDocument context) { + DataType nested = resolveFromContext(pType.nestedType(), context); + boolean cine = pType.getCreateIfNonExistent(); + boolean riz = pType.getRemoveIfZero(); + return new WeightedSetDataType(nested, cine, riz); + } + + private DataType createMap(ParsedType pType, ParsedDocument context) { + DataType kt = resolveFromContext(pType.mapKeyType(), context); + DataType vt = resolveFromContext(pType.mapValueType(), context); + return DataType.getMap(kt, vt); + } + + private DocumentType findDocFromSchemas(String name) { + var dt = documentsFromSchemas.get(name); + if (dt == null) { + throw new IllegalArgumentException("missing document type for: " + name); + } + return dt; + } + + private DataType createAnnRef(ParsedType pType, ParsedDocument context) { + SDAnnotationType annotation = findAnnotationFromSchemas(pType.getNameOfReferencedAnnotation(), context); + return new AnnotationReferenceDataType(annotation); + } + + private DataType createDocRef(ParsedType pType) { + var ref = pType.getReferencedDocumentType(); + assert(ref.getVariant() == ParsedType.Variant.DOCUMENT); + return new NewDocumentReferenceDataType(findDocFromSchemas(ref.name())); + } + + private DataType getBuiltinType(String name) { + switch (name) { + case "bool": return DataType.BOOL; + case "byte": return DataType.BYTE; + case "int": return DataType.INT; + case "long": return DataType.LONG; + case "string": return DataType.STRING; + case "float": return DataType.FLOAT; + case "double": return DataType.DOUBLE; + case "uri": return DataType.URI; + case "predicate": return DataType.PREDICATE; + case "raw": return DataType.RAW; + case "tag": return DataType.TAG; + case "float16": return DataType.FLOAT16; + default: + throw new IllegalArgumentException("Unknown builtin type: "+name); + } + } + + private DataType resolveFromContext(ParsedType pType, ParsedDocument context) { + String name = pType.name(); + switch (pType.getVariant()) { + case NONE: return DataType.NONE; + case BUILTIN: return getBuiltinType(name); + case POSITION: return PositionDataType.INSTANCE; + case ARRAY: return createArray(pType, context); + case WSET: return createWset(pType, context); + case MAP: return createMap(pType, context); + case TENSOR: return DataType.getTensor(pType.getTensorType()); + case DOC_REFERENCE: return createDocRef(pType); + case ANN_REFERENCE: return createAnnRef(pType, context); + case DOCUMENT: return findDocFromSchemas(name); + case STRUCT: return findStructFromSchemas(name, context); + case UNKNOWN: + // fallthrough + } + // unknown is probably struct + var found = context.findParsedStruct(name); + if (found != null) { + pType.setVariant(ParsedType.Variant.STRUCT); + return findStructFromSchemas(name, context); + } + if (documentsFromSchemas.containsKey(name)) { + pType.setVariant(ParsedType.Variant.DOCUMENT); + return findDocFromSchemas(name); + } + throw new IllegalArgumentException("unknown type named '" + name + "' in context "+context); + } + + @SuppressWarnings("deprecation") + private void registerDataTypes() { + for (DataType t : structsFromSchemas.values()) { + docMan.register(t); + } + for (DocumentType t : documentsFromSchemas.values()) { + docMan.registerDocumentType(t); + } + for (SDAnnotationType t : annotationsFromSchemas.values()) { + docMan.getAnnotationTypeRegistry().register(t); + } + } + + public class TypeResolver { + private final ParsedDocument context; + public DataType resolveType(ParsedType parsed) { + return resolveFromContext(parsed, context); + } + public DataType resolveStruct(ParsedStruct parsed) { + String structId = context.name() + "->" + parsed.name(); + var r = structsFromSchemas.get(structId); + if (r == null) { + throw new IllegalArgumentException("no datatype found for struct: " + structId); + } + return r; + } + public SDAnnotationType resolveAnnotation(String name) { + return findAnnotationFromSchemas(name, context); + } + TypeResolver(ParsedDocument context) { + this.context = context; + } + } + + public TypeResolver makeContext(ParsedDocument doc) { + return new TypeResolver(doc); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java new file mode 100644 index 00000000000..5509d11885c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java @@ -0,0 +1,212 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class converting a collection of schemas from the intermediate format. + * + * @author arnej27959 + **/ +public class ConvertSchemaCollection { + + private final IntermediateCollection input; + private final List orderedInput = new ArrayList<>(); + private final DocumentTypeManager docMan; + private final ApplicationPackage applicationPackage; + private final FileRegistry fileRegistry; + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + private final RankProfileRegistry rankProfileRegistry; + private final boolean documentsOnly; + + // for unit test + ConvertSchemaCollection(IntermediateCollection input, + DocumentTypeManager documentTypeManager) + { + this(input, documentTypeManager, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new BaseDeployLogger(), + new TestProperties(), + new RankProfileRegistry(), + true); + } + + public ConvertSchemaCollection(IntermediateCollection input, + DocumentTypeManager documentTypeManager, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) + { + this.input = input; + this.docMan = documentTypeManager; + this.applicationPackage = applicationPackage; + this.fileRegistry = fileRegistry; + this.deployLogger = deployLogger; + this.properties = properties; + this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; + + input.resolveInternalConnections(); + order(); + pushTypesToDocuments(); + } + + void order() { + var map = input.getParsedSchemas(); + for (var schema : map.values()) { + findOrdering(schema); + } + } + + void findOrdering(ParsedSchema schema) { + if (orderedInput.contains(schema)) return; + for (var parent : schema.getAllResolvedInherits()) { + findOrdering(parent); + } + orderedInput.add(schema); + } + + void pushTypesToDocuments() { + for (var schema : orderedInput) { + for (var struct : schema.getStructs()) { + schema.getDocument().addStruct(struct); + } + for (var annotation : schema.getAnnotations()) { + schema.getDocument().addAnnotation(annotation); + } + } + } + + private ConvertParsedTypes typeConverter; + + public void convertTypes() { + typeConverter = new ConvertParsedTypes(orderedInput, docMan); + typeConverter.convert(true); + } + + public List convertToSchemas() { + resolveStructInheritance(); + resolveAnnotationInheritance(); + addMissingAnnotationStructs(); + var converter = new ConvertParsedSchemas(orderedInput, + docMan, + applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + documentsOnly); + return converter.convertToSchemas(); + } + + private void resolveStructInheritance() { + List all = new ArrayList<>(); + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var struct : doc.getStructs()) { + for (String inherit : struct.getInherited()) { + var parent = doc.findParsedStruct(inherit); + if (parent == null) { + throw new IllegalArgumentException("Can not find parent for "+struct+" in "+doc); + } + struct.resolveInherit(inherit, parent); + } + all.add(struct); + } + } + List seen = new ArrayList<>(); + for (ParsedStruct struct : all) { + inheritanceCycleCheck(struct, seen); + } + } + + private void resolveAnnotationInheritance() { + List all = new ArrayList(); + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var annotation : doc.getAnnotations()) { + for (String inherit : annotation.getInherited()) { + var parent = doc.findParsedAnnotation(inherit); + if (parent == null) { + throw new IllegalArgumentException("Can not find parent for "+annotation+" in "+doc); + } + annotation.resolveInherit(inherit, parent); + } + all.add(annotation); + } + } + List seen = new ArrayList<>(); + for (ParsedAnnotation annotation : all) { + inheritanceCycleCheck(annotation, seen); + } + } + + private void fixupAnnotationStruct(ParsedAnnotation parsed) { + for (var parent : parsed.getResolvedInherits()) { + fixupAnnotationStruct(parent); + parent.getStruct().ifPresent(ps -> { + var myStruct = parsed.ensureStruct(); + if (! myStruct.getInherited().contains(ps.name())) { + myStruct.inherit(ps.name()); + myStruct.resolveInherit(ps.name(), ps); + } + }); + } + } + + private void addMissingAnnotationStructs() { + for (var schema : orderedInput) { + var doc = schema.getDocument(); + for (var annotation : doc.getAnnotations()) { + fixupAnnotationStruct(annotation); + } + } + } + + private void inheritanceCycleCheck(ParsedStruct struct, List seen) { + String name = struct.name(); + if (seen.contains(name)) { + seen.add(name); + throw new IllegalArgumentException("Inheritance/reference cycle for structs: " + + String.join(" -> ", seen)); + } + seen.add(name); + for (ParsedStruct parent : struct.getResolvedInherits()) { + inheritanceCycleCheck(parent, seen); + } + seen.remove(name); + } + + private void inheritanceCycleCheck(ParsedAnnotation annotation, List seen) { + String name = annotation.name(); + if (seen.contains(name)) { + seen.add(name); + throw new IllegalArgumentException("Inheritance/reference cycle for annotations: " + + String.join(" -> ", seen)); + } + seen.add(name); + for (ParsedAnnotation parent : annotation.getResolvedInherits()) { + inheritanceCycleCheck(parent, seen); + } + seen.remove(name); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/DictionaryOption.java b/config-model/src/main/java/com/yahoo/schema/parser/DictionaryOption.java new file mode 100644 index 00000000000..3acb51ace3f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/DictionaryOption.java @@ -0,0 +1,5 @@ +package com.yahoo.schema.parser; + +public enum DictionaryOption { + HASH, BTREE, CASED, UNCASED +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/schema/parser/InheritanceResolver.java new file mode 100644 index 00000000000..ad9acf2f095 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/InheritanceResolver.java @@ -0,0 +1,130 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Class resolving some inheritance relationships. + * + * @author arnej27959 + **/ +public class InheritanceResolver { + + private final Map parsedSchemas; + private final Map parsedDocs = new HashMap<>(); + private final Map schemaForDocs = new HashMap<>(); + + public InheritanceResolver(Map parsedSchemas) { + this.parsedSchemas = parsedSchemas; + } + + private void inheritanceCycleCheck(ParsedSchema schema, List seen) { + String name = schema.name(); + if (seen.contains(name)) { + seen.add(name); + throw new IllegalArgumentException("Inheritance/reference cycle for schemas: " + + String.join(" -> ", seen)); + } + seen.add(name); + for (ParsedSchema parent : schema.getAllResolvedInherits()) { + inheritanceCycleCheck(parent, seen); + } + seen.remove(name); + } + + private void resolveSchemaInheritance() { + for (ParsedSchema schema : parsedSchemas.values()) { + for (String inherit : schema.getInherited()) { + var parent = parsedSchemas.get(inherit); + if (parent == null) { + throw new IllegalArgumentException("schema '" + schema.name() + "' inherits '" + inherit + "', but this schema does not exist"); + } + schema.resolveInherit(inherit, parent); + } + } + } + + private void checkSchemaCycles() { + List seen = new ArrayList<>(); + for (ParsedSchema schema : parsedSchemas.values()) { + inheritanceCycleCheck(schema, seen); + } + } + + private void resolveDocumentInheritance() { + for (ParsedSchema schema : parsedSchemas.values()) { + if (! schema.hasDocument()) { + throw new IllegalArgumentException("For schema '" + schema.name() + + "': A search specification must have an equally named document inside of it."); + } + ParsedDocument doc = schema.getDocument(); + var old = parsedDocs.put(doc.name(), doc); + if (old != null) { + throw new IllegalArgumentException("duplicate document declaration for " + doc.name()); + } + schemaForDocs.put(doc.name(), schema); + for (String docInherit : doc.getInherited()) { + schema.inheritByDocument(docInherit); + } + for (String docReferenced : doc.getReferencedDocuments()) { + schema.inheritByDocument(docReferenced); + } + } + for (ParsedDocument doc : parsedDocs.values()) { + for (String inherit : doc.getInherited()) { + var parentDoc = parsedDocs.get(inherit); + if (parentDoc == null) { + throw new IllegalArgumentException("document " + doc.name() + " inherits from unavailable document " + inherit); + } + doc.resolveInherit(inherit, parentDoc); + } + for (String docRefName : doc.getReferencedDocuments()) { + var refDoc = parsedDocs.get(docRefName); + if (refDoc == null) { + throw new IllegalArgumentException("document " + doc.name() + " references unavailable document " + docRefName); + } + doc.resolveReferenced(refDoc); + } + } + for (ParsedSchema schema : parsedSchemas.values()) { + for (String docName : schema.getInheritedByDocument()) { + var parent = schemaForDocs.get(docName); + assert(parent.hasDocument()); + assert(parent.getDocument().name().equals(docName)); + schema.resolveInheritByDocument(docName, parent); + } + } + } + + private void inheritanceCycleCheck(ParsedDocument document, List seen) { + String name = document.name(); + if (seen.contains(name)) { + seen.add(name); + throw new IllegalArgumentException("Inheritance/reference cycle for documents: " + + String.join(" -> ", seen)); + } + seen.add(name); + for (ParsedDocument parent : document.getAllResolvedParents()) { + inheritanceCycleCheck(parent, seen); + } + seen.remove(name); + } + + private void checkDocumentCycles() { + List seen = new ArrayList<>(); + for (ParsedDocument doc : parsedDocs.values()) { + inheritanceCycleCheck(doc, seen); + } + } + + public void resolveInheritance() { + resolveSchemaInheritance(); + resolveDocumentInheritance(); + checkDocumentCycles(); + checkSchemaCycles(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java new file mode 100644 index 00000000000..8bb9bca3249 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -0,0 +1,159 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.yolean.Exceptions; + +import java.io.File; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Class wrapping parsing of schema files and holding a collection of + * schemas in the intermediate format. + * + * @author arnej27959 + **/ +public class IntermediateCollection { + + private final DeployLogger deployLogger; + private final ModelContext.Properties modelProperties; + + private Map parsedSchemas = new LinkedHashMap<>(); + + IntermediateCollection() { + this.deployLogger = new BaseDeployLogger(); + this.modelProperties = new TestProperties(); + } + + public IntermediateCollection(DeployLogger logger, ModelContext.Properties properties) { + this.deployLogger = logger; + this.modelProperties = properties; + } + + public Map getParsedSchemas() { return Collections.unmodifiableMap(parsedSchemas); } + + public ParsedSchema getParsedSchema(String name) { return parsedSchemas.get(name); } + + public ParsedSchema addSchemaFromString(String input) throws ParseException { + var stream = new SimpleCharStream(input); + var parser = new SchemaParser(stream, deployLogger, modelProperties); + try { + var schema = parser.schema(); + if (parsedSchemas.containsKey(schema.name())) { + throw new IllegalArgumentException("Duplicate schemas named: " + schema.name()); + } + parsedSchemas.put(schema.name(), schema); + return schema; + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + } + } + + private String addSchemaFromStringWithFileName(String input, String fileName) throws ParseException { + var parsed = addSchemaFromString(input); + String nameFromFile = baseName(fileName); + if (! parsed.name().equals(nameFromFile)) { + throw new IllegalArgumentException("The file containing schema '" + + parsed.name() + "' must be named '" + + parsed.name() + ApplicationPackage.SD_NAME_SUFFIX + + "', was '" + stripDirs(fileName) + "'"); + } + return parsed.name(); + } + + private String baseName(String filename) { + int pos = filename.lastIndexOf('/'); + if (pos != -1) { + filename = filename.substring(pos + 1); + } + pos = filename.lastIndexOf('.'); + if (pos != -1) { + filename = filename.substring(0, pos); + } + return filename; + } + + private String stripDirs(String filename) { + int pos = filename.lastIndexOf('/'); + if (pos != -1) { + return filename.substring(pos + 1); + } + return filename; + } + + /** + * parse a schema from the given reader and add result to collection + **/ + public String addSchemaFromReader(NamedReader reader) throws ParseException { + try { + var nameParsed = addSchemaFromStringWithFileName(IOUtils.readAll(reader.getReader()), reader.getName()); + reader.close(); + return nameParsed; + } catch (ParseException ex) { + throw new ParseException("Failed parsing schema from " + reader.getName() + ": " + ex.getMessage()); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); + } + } + + /** for unit tests */ + public String addSchemaFromFile(String fileName) throws ParseException { + try { + // return addSchemaFromStringWithFileName(IOUtils.readFile(new File(fileName)), fileName); + var parsed = addSchemaFromString(IOUtils.readFile(new File(fileName))); + return parsed.name(); + } catch (ParseException ex) { + throw new ParseException("Failed parsing schema from " + fileName + ": " + ex.getMessage()); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); + } + } + + /** + * parse a rank profile from the given reader and add to the schema identified by name. + * note: the named schema must have been parsed already. + **/ + public void addRankProfileFile(String schemaName, NamedReader reader) throws ParseException { + try { + ParsedSchema schema = parsedSchemas.get(schemaName); + if (schema == null) { + throw new IllegalArgumentException("No schema named: " + schemaName); + } + var stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); + var parser = new SchemaParser(stream, deployLogger, modelProperties); + try { + parser.rankProfile(schema); + } catch (ParseException pe) { + throw new ParseException("Failed parsing rank-profile from " + reader.getName() + ": " + + stream.formatException(Exceptions.toMessageString(pe))); + } + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); + } + } + + // for unit test + void addRankProfileFile(String schemaName, String fileName) throws ParseException { + try { + var reader = IOUtils.createReader(fileName, "UTF-8"); + addRankProfileFile(schemaName, new NamedReader(fileName, reader)); + } catch (java.io.IOException ex) { + throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); + } + } + + void resolveInternalConnections() { + var resolver = new InheritanceResolver(parsedSchemas); + resolver.resolveInheritance(); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedAnnotation.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAnnotation.java new file mode 100644 index 00000000000..c36656838f7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAnnotation.java @@ -0,0 +1,57 @@ +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a + * "annotation" block, using simple data structures as far as + * possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedAnnotation extends ParsedBlock { + + private ParsedStruct wrappedStruct = null; + private final List inherited = new ArrayList<>(); + private final List resolvedInherits = new ArrayList<>(); + private ParsedDocument ownedBy = null; + + ParsedAnnotation(String name) { + super(name, "annotation"); + } + + public List getInherited() { return List.copyOf(inherited); } + public List getResolvedInherits() { + assert(inherited.size() == resolvedInherits.size()); + return List.copyOf(resolvedInherits); + } + + + public Optional getStruct() { return Optional.ofNullable(wrappedStruct); } + public ParsedDocument getOwnerDoc() { return ownedBy; } + public String getOwnerName() { return ownedBy.name(); } + + public ParsedStruct ensureStruct() { + if (wrappedStruct == null) { + wrappedStruct = new ParsedStruct("annotation." + name()); + wrappedStruct.tagOwner(ownedBy); + } + return wrappedStruct; + } + void setStruct(ParsedStruct struct) { this.wrappedStruct = struct; } + + void inherit(String other) { inherited.add(other); } + + void tagOwner(ParsedDocument owner) { + verifyThat(ownedBy == null, "already owned by", ownedBy); + this.ownedBy = owner; + getStruct().ifPresent(s -> s.tagOwner(owner)); + } + + void resolveInherit(String name, ParsedAnnotation parsed) { + verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); + resolvedInherits.add(parsed); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java new file mode 100644 index 00000000000..be8d20fbe93 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a + * "attribute" block, using simple data structures as far as + * possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedAttribute extends ParsedBlock { + + private boolean enableBitVectors = false; + private boolean enableOnlyBitVector = false; + private boolean enableFastAccess = false; + private boolean enableFastRank = false; + private boolean enableFastSearch = false; + private boolean enableHuge = false; + private boolean enableMutable = false; + private boolean enablePaged = false; + private final Map aliases = new LinkedHashMap<>(); + private ParsedSorting sortSettings = null; + private String distanceMetric = null; + + ParsedAttribute(String name) { + super(name, "attribute"); + } + + List getAliases() { return List.copyOf(aliases.keySet()); } + String lookupAliasedFrom(String alias) { return aliases.get(alias); } + Optional getDistanceMetric() { return Optional.ofNullable(distanceMetric); } + boolean getEnableBitVectors() { return this.enableBitVectors; } + boolean getEnableOnlyBitVector() { return this.enableOnlyBitVector; } + boolean getFastAccess() { return this.enableFastAccess; } + boolean getFastRank() { return this.enableFastRank; } + boolean getFastSearch() { return this.enableFastSearch; } + boolean getHuge() { return this.enableHuge; } + boolean getMutable() { return this.enableMutable; } + boolean getPaged() { return this.enablePaged; } + Optional getSorting() { return Optional.ofNullable(sortSettings); } + + void addAlias(String from, String to) { + verifyThat(! aliases.containsKey(to), "already has alias", to); + aliases.put(to, from); + } + + void setDistanceMetric(String value) { + verifyThat(distanceMetric == null, "already has distance-metric", distanceMetric); + this.distanceMetric = value; + } + + ParsedSorting sortInfo() { + if (sortSettings == null) sortSettings = new ParsedSorting(name(), "attribute.sorting"); + return this.sortSettings; + } + + void setEnableBitVectors(boolean value) { this.enableBitVectors = value; } + void setEnableOnlyBitVector(boolean value) { this.enableOnlyBitVector = value; } + void setFastAccess(boolean value) { this.enableFastAccess = true; } + void setFastRank(boolean value) { this.enableFastRank = true; } + void setFastSearch(boolean value) { this.enableFastSearch = true; } + void setHuge(boolean value) { this.enableHuge = true; } + void setMutable(boolean value) { this.enableMutable = true; } + void setPaged(boolean value) { this.enablePaged = true; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedBlock.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedBlock.java new file mode 100644 index 00000000000..c20abf52bf3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedBlock.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +/** + * Common methods for various Parsed* classes. + * @author arnej27959 + **/ +public class ParsedBlock { + private final String name; + private final String blockType; + + public ParsedBlock(String name, String blockType) { + this.name = name; + this.blockType = blockType; + } + + public final String name() { return name; } + public final String blockType() { return blockType; } + + protected void verifyThat(boolean check, String msg, Object ... msgDetails) { + if (check) return; + var buf = new StringBuilder(); + buf.append(blockType).append(" '").append(name).append("' error: "); + buf.append(msg); + for (Object detail : msgDetails) { + buf.append(" "); + buf.append(detail.toString()); + } + throw new IllegalArgumentException(buf.toString()); + } + + public String toString() { + return blockType + " '" + name + "'"; + } +} + diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocument.java new file mode 100644 index 00000000000..281e7989885 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocument.java @@ -0,0 +1,127 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * This class holds the extracted information after parsing a + * "document" block in a schema (.sd) file, using simple data + * structures as far as possible. Do not put advanced logic here! + * @author arnej27959 + **/ +public class ParsedDocument extends ParsedBlock { + private final List inherited = new ArrayList<>(); + private final Map resolvedInherits = new LinkedHashMap(); + private final Map resolvedReferences = new LinkedHashMap(); + private final Map docFields = new LinkedHashMap<>(); + private final Map docStructs = new LinkedHashMap<>(); + private final Map docAnnotations = new LinkedHashMap<>(); + + public ParsedDocument(String name) { + super(name, "document"); + } + + List getInherited() { return List.copyOf(inherited); } + List getAnnotations() { return List.copyOf(docAnnotations.values()); } + List getResolvedInherits() { + assert(inherited.size() == resolvedInherits.size()); + return List.copyOf(resolvedInherits.values()); + } + List getResolvedReferences() { + return List.copyOf(resolvedReferences.values()); + } + List getAllResolvedParents() { + List all = new ArrayList<>(); + all.addAll(getResolvedInherits()); + all.addAll(getResolvedReferences()); + return all; + } + List getFields() { return List.copyOf(docFields.values()); } + List getStructs() { return List.copyOf(docStructs.values()); } + ParsedStruct getStruct(String name) { return docStructs.get(name); } + ParsedAnnotation getAnnotation(String name) { return docAnnotations.get(name); } + + List getReferencedDocuments() { + var result = new ArrayList(); + for (var field : docFields.values()) { + var type = field.getType(); + if (type.getVariant() == ParsedType.Variant.DOC_REFERENCE) { + var docType = type.getReferencedDocumentType(); + assert(docType.getVariant() == ParsedType.Variant.DOCUMENT); + result.add(docType.name()); + } + } + return result; + } + + void inherit(String other) { inherited.add(other); } + + void addField(ParsedField field) { + String fieldName = field.name().toLowerCase(); + verifyThat(! docFields.containsKey(fieldName), + "Duplicate (case insensitively) " + field + " in document type '" + this.name() + "'"); + docFields.put(fieldName, field); + } + + void addStruct(ParsedStruct struct) { + String sName = struct.name(); + verifyThat(! docStructs.containsKey(sName), "already has struct", sName); + docStructs.put(sName, struct); + struct.tagOwner(this); + } + + void addAnnotation(ParsedAnnotation annotation) { + String annName = annotation.name(); + verifyThat(! docAnnotations.containsKey(annName), "already has annotation", annName); + docAnnotations.put(annName, annotation); + annotation.tagOwner(this); + } + + void resolveInherit(String name, ParsedDocument parsed) { + verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); + verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); + resolvedInherits.put(name, parsed); + } + + void resolveReferenced(ParsedDocument parsed) { + var old = resolvedReferences.put(parsed.name(), parsed); + assert(old == null || old == parsed); + } + + ParsedStruct findParsedStruct(String name) { + ParsedStruct found = getStruct(name); + if (found != null) return found; + for (var parent : getAllResolvedParents()) { + var fromParent = parent.findParsedStruct(name); + if (fromParent == null) continue; + if (fromParent == found) continue; + if (found == null) { + found = fromParent; + } else { + throw new IllegalArgumentException("conflicting values for struct " + name + " in " +this); + } + } + return found; + } + + ParsedAnnotation findParsedAnnotation(String name) { + ParsedAnnotation found = docAnnotations.get(name); + if (found != null) return found; + for (var parent : getResolvedInherits()) { + var fromParent = parent.findParsedAnnotation(name); + if (fromParent == null) continue; + if (fromParent == found) continue; + if (found == null) { + found = fromParent; + } else { + throw new IllegalArgumentException("conflicting values for annotation " + name + " in " +this); + } + } + return found; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java new file mode 100644 index 00000000000..93469a86fe3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java @@ -0,0 +1,49 @@ + +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * This class holds the extracted information after parsing a + * "document-summary" block, using simple data structures as far as + * possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedDocumentSummary extends ParsedBlock { + + private boolean omitSummaryFeatures; + private boolean fromDisk; + private final List inherited = new ArrayList<>(); + private final Map fields = new LinkedHashMap<>(); + + ParsedDocumentSummary(String name) { + super(name, "document-summary"); + } + + boolean getOmitSummaryFeatures() { return omitSummaryFeatures; } + boolean getFromDisk() { return fromDisk; } + List getSummaryFields() { return List.copyOf(fields.values()); } + List getInherited() { return List.copyOf(inherited); } + + ParsedSummaryField addField(ParsedSummaryField field) { + String fieldName = field.name(); + // TODO disallow this on Vespa 8 + // verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); + return fields.put(fieldName, field); + } + + void setFromDisk(boolean value) { + this.fromDisk = value; + } + + void setOmitSummaryFeatures(boolean value) { + this.omitSummaryFeatures = value; + } + + void inherit(String other) { + inherited.add(other); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedField.java new file mode 100644 index 00000000000..a4df2ac6dc2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedField.java @@ -0,0 +1,159 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.schema.document.Stemming; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a "field" + * block, using simple data structures as far as possible. Do not put + * advanced logic here! + * @author arnej27959 + **/ +class ParsedField extends ParsedBlock { + + private ParsedType type; + private boolean hasBolding = false; + private boolean isFilter = false; + private int overrideId = 0; + private boolean isLiteral = false; + private boolean isNormal = false; + private Integer weight; + private String normalizing = null; + private final ParsedMatchSettings matchInfo = new ParsedMatchSettings(); + private Stemming stemming = null; + private ParsedIndexingOp indexingOp = null; + private ParsedSorting sortSettings = null; + private final Map attributes = new LinkedHashMap<>(); + private final Map fieldIndexes = new LinkedHashMap<>(); + private final Map aliases = new LinkedHashMap<>(); + private final Map rankTypes = new LinkedHashMap<>(); + private final Map structFields = new LinkedHashMap<>(); + private final Map summaryFields = new LinkedHashMap<>(); + private final List dictionaryOptions = new ArrayList<>(); + private final List queryCommands = new ArrayList<>(); + + ParsedField(String name, ParsedType type) { + super(name, "field"); + this.type = type; + } + + ParsedType getType() { return this.type; } + boolean hasBolding() { return this.hasBolding; } + boolean hasFilter() { return this.isFilter; } + boolean hasLiteral() { return this.isLiteral; } + boolean hasNormal() { return this.isNormal; } + boolean hasIdOverride() { return overrideId != 0; } + int idOverride() { return overrideId; } + List getDictionaryOptions() { return List.copyOf(dictionaryOptions); } + List getAttributes() { return List.copyOf(attributes.values()); } + List getIndexes() { return List.copyOf(fieldIndexes.values()); } + List getSummaryFields() { return List.copyOf(summaryFields.values()); } + List getStructFields() { return List.copyOf(structFields.values()); } + List getAliases() { return List.copyOf(aliases.keySet()); } + List getQueryCommands() { return List.copyOf(queryCommands); } + String lookupAliasedFrom(String alias) { return aliases.get(alias); } + ParsedMatchSettings matchSettings() { return this.matchInfo; } + Optional getWeight() { return Optional.ofNullable(weight); } + Optional getStemming() { return Optional.ofNullable(stemming); } + Optional getNormalizing() { return Optional.ofNullable(normalizing); } + Optional getIndexing() { return Optional.ofNullable(indexingOp); } + Optional getSorting() { return Optional.ofNullable(sortSettings); } + Map getRankTypes() { return Collections.unmodifiableMap(rankTypes); } + + /** get an existing summary field for modification, or create it */ + ParsedSummaryField summaryFieldFor(String name) { + if (summaryFields.containsKey(name)) { + return summaryFields.get(name); + } + var sf = new ParsedSummaryField(name, getType()); + summaryFields.put(name, sf); + return sf; + } + + /** get an existing summary field for modification, or create it */ + ParsedSummaryField summaryFieldFor(String name, ParsedType type) { + if (summaryFields.containsKey(name)) { + var sf = summaryFields.get(name); + if (sf.getType() == null) { + sf.setType(type); + } else { + // TODO check that types are properly equal here + String oldName = sf.getType().name(); + String newName = type.name(); + verifyThat(newName.equals(oldName), "type mismatch for summary field", name, ":", oldName, "/", newName); + } + return sf; + } + var sf = new ParsedSummaryField(name, type); + summaryFields.put(name, sf); + return sf; + } + + void addAlias(String from, String to) { + verifyThat(! aliases.containsKey(to), "already has alias", to); + aliases.put(to, from); + } + + void addIndex(ParsedIndex index) { + String idxName = index.name(); + verifyThat(! fieldIndexes.containsKey(idxName), "already has index", idxName); + fieldIndexes.put(idxName, index); + } + + void addRankType(String index, String rankType) { + rankTypes.put(index, rankType); + } + + void dictionary(DictionaryOption option) { + dictionaryOptions.add(option); + } + + void setBolding(boolean value) { this.hasBolding = value; } + void setFilter(boolean value) { this.isFilter = value; } + void setId(int id) { this.overrideId = id; } + void setLiteral(boolean value) { this.isLiteral = value; } + void setNormal(boolean value) { this.isNormal = value; } + void setNormalizing(String value) { this.normalizing = value; } + void setStemming(Stemming stemming) { this.stemming = stemming; } + void setWeight(int weight) { this.weight = weight; } + + ParsedAttribute attributeFor(String attrName) { + return attributes.computeIfAbsent(attrName, n -> new ParsedAttribute(n)); + } + + void setIndexingOperation(ParsedIndexingOp idxOp) { + verifyThat(indexingOp == null, "already has indexing"); + indexingOp = idxOp; + } + + ParsedSorting sortInfo() { + if (sortSettings == null) sortSettings = new ParsedSorting(name(), "field.sorting"); + return this.sortSettings; + } + + void addQueryCommand(String command) { + queryCommands.add(command); + } + + void addStructField(ParsedField structField) { + String fieldName = structField.name(); + verifyThat(! structFields.containsKey(fieldName), "already has struct-field", fieldName); + structFields.put(fieldName, structField); + } + + void addSummaryField(ParsedSummaryField summaryField) { + String fieldName = summaryField.name(); + verifyThat(! summaryFields.containsKey(fieldName), "already has summary field", fieldName); + if (summaryField.getType() == null) { + summaryField.setType(getType()); + } + summaryFields.put(fieldName, summaryField); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedFieldSet.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedFieldSet.java new file mode 100644 index 00000000000..9e8906a41a4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedFieldSet.java @@ -0,0 +1,36 @@ +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a "fieldset" + * block, using simple data structures as far as possible. Do not put + * advanced logic here! + * @author arnej27959 + **/ +class ParsedFieldSet extends ParsedBlock { + + private final List fields = new ArrayList<>(); + private final List queryCommands = new ArrayList<>(); + private ParsedMatchSettings matchInfo = null; + + ParsedFieldSet(String name) { + super(name, "fieldset"); + } + + ParsedMatchSettings matchSettings() { + if (matchInfo == null) matchInfo = new ParsedMatchSettings(); + return this.matchInfo; + } + + List getQueryCommands() { return List.copyOf(queryCommands); } + List getFieldNames() { return List.copyOf(fields); } + Optional getMatchSettings() { + return Optional.ofNullable(this.matchInfo); + } + + void addField(String field) { fields.add(field); } + void addQueryCommand(String command) { queryCommands.add(command); } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndex.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndex.java new file mode 100644 index 00000000000..cf70168e8d2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndex.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.Stemming; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing an "index" + * block, using simple data structures as far as possible. Do not put + * advanced logic here! + * @author arnej27959 + **/ +class ParsedIndex extends ParsedBlock { + + private Boolean enableBm25 = null; + private Boolean isPrefix = null; + private HnswIndexParams hnswParams = null; + private final List aliases = new ArrayList<>(); + private Stemming stemming = null; + private Integer arity = null; + private Long lowerBound = null; + private Long upperBound = null; + private Double densePLT = null; + + ParsedIndex(String name) { + super(name, "index"); + } + + Optional getEnableBm25() { return Optional.ofNullable(this.enableBm25); } + Optional getPrefix() { return Optional.ofNullable(this.isPrefix); } + Optional getHnswIndexParams() { return Optional.ofNullable(this.hnswParams); } + List getAliases() { return List.copyOf(aliases); } + boolean hasStemming() { return stemming != null; } + Optional getStemming() { return Optional.ofNullable(stemming); } + Optional getArity() { return Optional.ofNullable(this.arity); } + Optional getLowerBound() { return Optional.ofNullable(this.lowerBound); } + Optional getUpperBound() { return Optional.ofNullable(this.upperBound); } + Optional getDensePostingListThreshold() { return Optional.ofNullable(this.densePLT); } + + void addAlias(String alias) { + aliases.add(alias); + } + + void setArity(int arity) { + this.arity = arity; + } + + void setDensePostingListThreshold(double threshold) { + this.densePLT = threshold; + } + + void setEnableBm25(boolean value) { + this.enableBm25 = value; + } + + void setHnswIndexParams(HnswIndexParams params) { + this.hnswParams = params; + } + + void setLowerBound(long value) { + this.lowerBound = value; + } + + void setPrefix(boolean value) { + this.isPrefix = value; + } + + void setStemming(Stemming stemming) { + this.stemming = stemming; + } + + void setUpperBound(long value) { + this.upperBound = value; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndexingOp.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndexingOp.java new file mode 100644 index 00000000000..3a2df2aac4c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndexingOp.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; + +/** + * This class wraps an indexing script expression, with some helper + * methods for extracting information from it + * @author arnej27959 + **/ +class ParsedIndexingOp { + + private final ScriptExpression script; + + ParsedIndexingOp(ScriptExpression script) { + this.script = script; + } + + ScriptExpression script() { return this.script; } + + public boolean doesAttributing() { return containsExpression(AttributeExpression.class); } + public boolean doesIndexing() { return containsExpression(IndexExpression.class); } + public boolean doesLowerCasing() { return containsExpression(LowerCaseExpression.class); } + public boolean doesSummarying() { return containsExpression(SummaryExpression.class); } + + private boolean containsExpression(Class searchFor) { + var searcher = new ExpressionSearcher<>(searchFor); + var expr = searcher.searchIn(script); + return (expr != null); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java new file mode 100644 index 00000000000..4d3c45ad67f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -0,0 +1,38 @@ +package com.yahoo.schema.parser; + +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.MatchAlgorithm; + +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a "match" + * block, using simple data structures as far as possible. Do not put + * advanced logic here! + * @author arnej27959 + **/ +public class ParsedMatchSettings { + + private MatchType matchType = null; + private Case matchCase = null; + private MatchAlgorithm matchAlgorithm = null; + private String exactTerminator = null; + private Integer gramSize = null; + private Integer maxLength = null; + + Optional getMatchType() { return Optional.ofNullable(matchType); } + Optional getMatchCase() { return Optional.ofNullable(matchCase); } + Optional getMatchAlgorithm() { return Optional.ofNullable(matchAlgorithm); } + Optional getExactTerminator() { return Optional.ofNullable(exactTerminator); } + Optional getGramSize() { return Optional.ofNullable(gramSize); } + Optional getMaxLength() { return Optional.ofNullable(maxLength); } + + // TODO - consider allowing each set only once: + void setType(MatchType value) { this.matchType = value; } + void setCase(Case value) { this.matchCase = value; } + void setAlgorithm(MatchAlgorithm value) { this.matchAlgorithm = value; } + void setExactTerminator(String value) { this.exactTerminator = value; } + void setGramSize(int value) { this.gramSize = value; } + void setMaxLength(int value) { this.maxLength = value; } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankFunction.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankFunction.java new file mode 100644 index 00000000000..73f1316d468 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankFunction.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.List; + +/** + * This class holds the extracted information after parsing a + * "function" block in a rank-profile, using simple data structures as + * far as possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedRankFunction extends ParsedBlock { + + private boolean inline; + private String expression; + private final List parameters = new ArrayList<>(); + + ParsedRankFunction(String name) { + super(name, "function"); + } + + boolean getInline() { return this.inline; } + String getExpression() { return this.expression; } + List getParameters() { return List.copyOf(parameters); } + + void addParameter(String param) { + verifyThat(! parameters.contains(param), "cannot have parameter", param, "twice"); + parameters.add(param); + } + + void setInline(boolean value) { + this.inline = value; + } + + void setExpression(String value) { + this.expression = value; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java new file mode 100644 index 00000000000..64dd8dd0ad4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -0,0 +1,220 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile.MutateOperation; +import com.yahoo.searchlib.rankingexpression.FeatureList; +import com.yahoo.searchlib.rankingexpression.Reference; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a + * rank-profile block in a schema (.sd) file, using simple data + * structures as far as possible. Do not put advanced logic here! + * + * @author arnej27959 + */ +class ParsedRankProfile extends ParsedBlock { + + private boolean ignoreDefaultRankFeatures = false; + private Double rankScoreDropLimit = null; + private Double termwiseLimit = null; + private Double postFilterThreshold = null; + private Double approximateThreshold = null; + private final List matchFeatures = new ArrayList<>(); + private final List rankFeatures = new ArrayList<>(); + private final List summaryFeatures = new ArrayList<>(); + private Integer keepRankCount = null; + private Integer minHitsPerThread = null; + private Integer numSearchPartitions = null; + private Integer numThreadsPerSearch = null; + private Integer reRankCount = null; + private MatchPhaseSettings matchPhaseSettings = null; + private String firstPhaseExpression = null; + private String inheritedSummaryFeatures = null; + private String inheritedMatchFeatures = null; + private String secondPhaseExpression = null; + private Boolean strict = null; + private final List mutateOperations = new ArrayList<>(); + private final List inherited = new ArrayList<>(); + private final Map fieldsRankFilter = new LinkedHashMap<>(); + private final Map fieldsRankWeight = new LinkedHashMap<>(); + private final Map functions = new LinkedHashMap<>(); + private final Map fieldsRankType = new LinkedHashMap<>(); + private final Map> rankProperties = new LinkedHashMap<>(); + private final Map constants = new LinkedHashMap<>(); + private final Map inputs = new LinkedHashMap<>(); + private final List onnxModels = new ArrayList<>(); + + ParsedRankProfile(String name) { + super(name, "rank-profile"); + } + + boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } + Optional getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } + Optional getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } + Optional getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } + Optional getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } + List getMatchFeatures() { return List.copyOf(this.matchFeatures); } + List getRankFeatures() { return List.copyOf(this.rankFeatures); } + List getSummaryFeatures() { return List.copyOf(this.summaryFeatures); } + Optional getKeepRankCount() { return Optional.ofNullable(this.keepRankCount); } + Optional getMinHitsPerThread() { return Optional.ofNullable(this.minHitsPerThread); } + Optional getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } + Optional getNumThreadsPerSearch() { return Optional.ofNullable(this.numThreadsPerSearch); } + Optional getReRankCount() { return Optional.ofNullable(this.reRankCount); } + Optional getMatchPhaseSettings() { return Optional.ofNullable(this.matchPhaseSettings); } + Optional getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } + Optional getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } + List getFunctions() { return List.copyOf(functions.values()); } + List getMutateOperations() { return List.copyOf(mutateOperations); } + List getInherited() { return List.copyOf(inherited); } + + Map getFieldsWithRankFilter() { return Collections.unmodifiableMap(fieldsRankFilter); } + Map getFieldsWithRankWeight() { return Collections.unmodifiableMap(fieldsRankWeight); } + Map getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } + Map> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } + Map getConstants() { return Collections.unmodifiableMap(constants); } + Map getInputs() { return Collections.unmodifiableMap(inputs); } + List getOnnxModels() { return List.copyOf(onnxModels); } + + Optional getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } + Optional getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } + Optional isStrict() { return Optional.ofNullable(this.strict); } + + void addSummaryFeatures(FeatureList features) { this.summaryFeatures.add(features); } + void addMatchFeatures(FeatureList features) { this.matchFeatures.add(features); } + void addRankFeatures(FeatureList features) { this.rankFeatures.add(features); } + + void inherit(String other) { inherited.add(other); } + + void setInheritedSummaryFeatures(String other) { + verifyThat(inheritedSummaryFeatures == null, "already inherits summary-features"); + this.inheritedSummaryFeatures = other; + } + + void add(RankProfile.Constant constant) { + verifyThat(! constants.containsKey(constant.name()), "already has constant", constant.name()); + constants.put(constant.name(), constant); + } + + void addInput(Reference name, RankProfile.Input input) { + verifyThat(! inputs.containsKey(name), "already has input", name); + inputs.put(name, input); + } + + void add(OnnxModel model) { + onnxModels.add(model); + } + + void addFieldRankFilter(String field, boolean filter) { + fieldsRankFilter.put(field, filter); + } + + void addFieldRankType(String field, String type) { + verifyThat(! fieldsRankType.containsKey(field), "already has rank type for field", field); + fieldsRankType.put(field, type); + } + + void addFieldRankWeight(String field, int weight) { + verifyThat(! fieldsRankType.containsKey(field), "already has weight for field", field); + fieldsRankWeight.put(field, weight); + } + + ParsedRankFunction addOrReplaceFunction(ParsedRankFunction func) { + // allowed with warning + // verifyThat(! functions.containsKey(func.name()), "already has function", func.name()); + return functions.put(func.name(), func); + } + + void addMutateOperation(MutateOperation.Phase phase, String attrName, String operation) { + mutateOperations.add(new MutateOperation(phase, attrName, operation)); + } + + void addRankProperty(String key, String value) { + List values = rankProperties.computeIfAbsent(key, k -> new ArrayList()); + values.add(value); + } + + void setFirstPhaseRanking(String expression) { + verifyThat(firstPhaseExpression == null, "already has first-phase expression"); + this.firstPhaseExpression = expression; + } + + void setIgnoreDefaultRankFeatures(boolean value) { + this.ignoreDefaultRankFeatures = value; + } + + void setInheritedMatchFeatures(String other) { + this.inheritedMatchFeatures = other; + } + + void setKeepRankCount(int count) { + verifyThat(keepRankCount == null, "already has rerank-count"); + this.keepRankCount = count; + } + + void setMatchPhaseSettings(MatchPhaseSettings settings) { + verifyThat(matchPhaseSettings == null, "already has match-phase"); + this.matchPhaseSettings = settings; + } + + void setMinHitsPerThread(int minHits) { + verifyThat(minHitsPerThread == null, "already has min-hits-per-thread"); + this.minHitsPerThread = minHits; + } + + void setNumSearchPartitions(int numParts) { + verifyThat(numSearchPartitions == null, "already has num-search-partitions"); + this.numSearchPartitions = numParts; + } + + void setNumThreadsPerSearch(int threads) { + verifyThat(numThreadsPerSearch == null, "already has num-threads-per-search"); + this.numThreadsPerSearch = threads; + } + + void setRankScoreDropLimit(double limit) { + verifyThat(rankScoreDropLimit == null, "already has rank-score-drop-limit"); + this.rankScoreDropLimit = limit; + } + + void setRerankCount(int count) { + verifyThat(reRankCount == null, "already has rerank-count"); + this.reRankCount = count; + } + + void setSecondPhaseRanking(String expression) { + verifyThat(secondPhaseExpression == null, "already has second-phase expression"); + this.secondPhaseExpression = expression; + } + + void setStrict(boolean strict) { + verifyThat(this.strict == null, "already has strict"); + this.strict = strict; + } + + void setTermwiseLimit(double limit) { + verifyThat(termwiseLimit == null, "already has termwise-limit"); + this.termwiseLimit = limit; + } + + void setPostFilterThreshold(double threshold) { + verifyThat(postFilterThreshold == null, "already has post-filter-threshold"); + this.postFilterThreshold = threshold; + } + + void setApproximateThreshold(double threshold) { + verifyThat(approximateThreshold == null, "already has approximate-threshold"); + this.approximateThreshold = threshold; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSchema.java new file mode 100644 index 00000000000..5ee483db044 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSchema.java @@ -0,0 +1,176 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.document.Stemming; +import com.yahoo.searchlib.rankingexpression.Reference; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * This class holds the extracted information after parsing + * one schema (.sd) file, using simple data structures + * as far as possible. + * + * Do not put complicated logic here! + * + * @author arnej27959 + */ +public class ParsedSchema extends ParsedBlock { + + public static class ImportedField { + public final String asFieldName; + public final String refFieldName; + public final String foreignFieldName; + public ImportedField(String asField, String refField, String foreignField) { + this.asFieldName = asField; + this.refFieldName = refField; + this.foreignFieldName = foreignField; + } + } + + private boolean documentWithoutSchema = false; + private Boolean rawAsBase64 = null; + private ParsedDocument myDocument = null; + private Stemming defaultStemming = null; + private final List importedFields = new ArrayList<>(); + private final List onnxModels = new ArrayList<>(); + private final Map constants = new LinkedHashMap<>(); + private final List inherited = new ArrayList<>(); + private final List inheritedByDocument = new ArrayList<>(); + private final Map resolvedInherits = new LinkedHashMap<>(); + private final Map allResolvedInherits = new LinkedHashMap<>(); + private final Map extraAnnotations = new LinkedHashMap<>(); + private final Map docSums = new LinkedHashMap<>(); + private final Map extraFields = new LinkedHashMap<>(); + private final Map fieldSets = new LinkedHashMap<>(); + private final Map extraIndexes = new LinkedHashMap<>(); + private final Map rankProfiles = new LinkedHashMap<>(); + private final Map extraStructs = new LinkedHashMap<>(); + + public ParsedSchema(String name) { + super(name, "schema"); + } + + boolean getDocumentWithoutSchema() { return documentWithoutSchema; } + Optional getRawAsBase64() { return Optional.ofNullable(rawAsBase64); } + boolean hasDocument() { return myDocument != null; } + ParsedDocument getDocument() { return myDocument; } + boolean hasStemming() { return defaultStemming != null; } + Stemming getStemming() { return defaultStemming; } + List getImportedFields() { return List.copyOf(importedFields); } + List getOnnxModels() { return List.copyOf(onnxModels); } + List getAnnotations() { return List.copyOf(extraAnnotations.values()); } + List getDocumentSummaries() { return List.copyOf(docSums.values()); } + List getFields() { return List.copyOf(extraFields.values()); } + List getFieldSets() { return List.copyOf(fieldSets.values()); } + List getIndexes() { return List.copyOf(extraIndexes.values()); } + List getStructs() { return List.copyOf(extraStructs.values()); } + List getInherited() { return List.copyOf(inherited); } + List getInheritedByDocument() { return List.copyOf(inheritedByDocument); } + List getRankProfiles() { return List.copyOf(rankProfiles.values()); } + List getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } + List getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); } + List getConstants() { return List.copyOf(constants.values()); } + + void addAnnotation(ParsedAnnotation annotation) { + String annName = annotation.name(); + verifyThat(! extraAnnotations.containsKey(annName), "already has annotation", annName); + extraAnnotations.put(annName, annotation); + } + + void addDocument(ParsedDocument document) { + verifyThat(myDocument == null, + "already has", myDocument, "so cannot add", document); + // TODO - disallow? + // verifyThat(name().equals(document.name()), + // "schema " + name() + " can only contain document named " + name() + ", was: "+ document.name()); + this.myDocument = document; + } + + void setDocumentWithoutSchema() { this.documentWithoutSchema = true; } + + void addDocumentSummary(ParsedDocumentSummary docsum) { + String dsName = docsum.name(); + verifyThat(! docSums.containsKey(dsName), "already has document-summary", dsName); + docSums.put(dsName, docsum); + } + + void addField(ParsedField field) { + String fieldName = field.name(); + verifyThat(! extraFields.containsKey(fieldName), "already has field", fieldName); + extraFields.put(fieldName, field); + } + + void addFieldSet(ParsedFieldSet fieldSet) { + String fsName = fieldSet.name(); + verifyThat(! fieldSets.containsKey(fsName), "already has fieldset", fsName); + fieldSets.put(fsName, fieldSet); + } + + void addImportedField(String asFieldName, String refFieldName, String foregnFieldName) { + importedFields.add(new ImportedField(asFieldName, refFieldName, foregnFieldName)); + } + + void addIndex(ParsedIndex index) { + String idxName = index.name(); + verifyThat(! extraIndexes.containsKey(idxName), "already has index", idxName); + extraIndexes.put(idxName, index); + } + + void add(OnnxModel model) { + onnxModels.add(model); + } + + void addRankProfile(ParsedRankProfile profile) { + String rpName = profile.name(); + verifyThat(! rankProfiles.containsKey(rpName), "already has rank-profile", rpName); + rankProfiles.put(rpName, profile); + } + + void add(RankProfile.Constant constant) { + constants.put(constant.name(), constant); + } + + void addStruct(ParsedStruct struct) { + String sName = struct.name(); + verifyThat(! extraStructs.containsKey(sName), "already has struct", sName); + extraStructs.put(sName, struct); + } + + void enableRawAsBase64(boolean value) { + this.rawAsBase64 = value; + } + + void inherit(String other) { inherited.add(other); } + + void inheritByDocument(String other) { inheritedByDocument.add(other); } + + void setStemming(Stemming value) { + verifyThat((defaultStemming == null) || (defaultStemming == value), + "already has stemming", defaultStemming, "cannot also set", value); + defaultStemming = value; + } + + void resolveInherit(String name, ParsedSchema parsed) { + verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); + verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); + resolvedInherits.put(name, parsed); + var old = allResolvedInherits.put("schema " + name, parsed); + verifyThat(old == null || old == parsed, "conflicting resolveInherit for", name); + } + + void resolveInheritByDocument(String name, ParsedSchema parsed) { + verifyThat(inheritedByDocument.contains(name), + "resolveInheritByDocument for non-inherited name", name); + var old = allResolvedInherits.put("document " + name, parsed); + verifyThat(old == null || old == parsed, "conflicting resolveInheritByDocument for", name); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSorting.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSorting.java new file mode 100644 index 00000000000..af84bbbb5bd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSorting.java @@ -0,0 +1,48 @@ + +package com.yahoo.schema.parser; + +import com.yahoo.schema.document.Sorting.Function; +import com.yahoo.schema.document.Sorting.Strength; + +import java.util.Optional; + +/** + * This class holds the extracted information after parsing a "sorting" + * block, using simple data structures as far as possible. Do not put + * advanced logic here! + * @author arnej27959 + **/ +class ParsedSorting extends ParsedBlock { + + private boolean ascending = true; + private Function sortFunction = null; + private Strength sortStrength = null; + private String sortLocale = null; + + ParsedSorting(String blockName, String blockType) { + super(blockName, blockType); + } + + boolean getAscending() { return this.ascending; } + boolean getDescending() { return ! this.ascending; } + Optional getFunction() { return Optional.ofNullable(sortFunction); } + Optional getStrength() { return Optional.ofNullable(sortStrength); } + Optional getLocale() { return Optional.ofNullable(sortLocale); } + + void setAscending() { this.ascending = true; } + + void setDescending() { this.ascending = false; } + + void setLocale(String value) { + verifyThat(sortLocale == null, "sorting already has locale", sortLocale); + this.sortLocale = value; + } + void setFunction(Function value) { + verifyThat(sortFunction == null, "sorting already has function", sortFunction); + this.sortFunction = value; + } + void setStrength(Strength value) { + verifyThat(sortStrength == null, "sorting already has strength", sortStrength); + this.sortStrength = value; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java new file mode 100644 index 00000000000..abe14b3689f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * This class holds the extracted information after parsing a "struct" + * block, using simple data structures as far as possible. Do not put + * advanced logic here! + * @author arnej27959 + **/ +public class ParsedStruct extends ParsedBlock { + private final List inherited = new ArrayList<>(); + private final List resolvedInherits = new ArrayList<>(); + private final Map fields = new LinkedHashMap<>(); + private final ParsedType asParsedType; + private ParsedDocument ownedBy = null; + + public ParsedStruct(String name) { + super(name, "struct"); + this.asParsedType = ParsedType.fromName(name); + asParsedType.setVariant(ParsedType.Variant.STRUCT); + } + + List getFields() { return List.copyOf(fields.values()); } + List getInherited() { return List.copyOf(inherited); } + ParsedDocument getOwnerDoc() { return ownedBy; } + String getOwnerName() { return ownedBy.name(); } + List getResolvedInherits() { + assert(inherited.size() == resolvedInherits.size()); + return List.copyOf(resolvedInherits); + } + + void addField(ParsedField field) { + String fieldName = field.name(); + verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); + fields.put(fieldName, field); + } + + void inherit(String other) { + verifyThat(! name().equals(other), "cannot inherit from itself"); + inherited.add(other); + } + + void tagOwner(ParsedDocument document) { + verifyThat(ownedBy == null, "already owned by document "+ownedBy); + this.ownedBy = document; + } + + void resolveInherit(String name, ParsedStruct parsed) { + verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); + verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); + resolvedInherits.add(parsed); + } + +} + diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java new file mode 100644 index 00000000000..38ee52c9d06 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import java.util.ArrayList; +import java.util.List; + +/** + * This class holds the extracted information after parsing a summary + * field declaration, either from "field" inside "document-summary" or + * "summary" inside "field". Using simple data structures as far as + * possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedSummaryField extends ParsedBlock { + + private ParsedType type; + private boolean isDyn = false; + private boolean isMEO = false; + private boolean isFull = false; + private boolean isBold = false; + private final List sources = new ArrayList<>(); + private final List destinations = new ArrayList<>(); + + ParsedSummaryField(String name) { + this(name, null); + } + + ParsedSummaryField(String name, ParsedType type) { + super(name, "summary field"); + this.type = type; + } + + ParsedType getType() { return type; } + List getDestinations() { return List.copyOf(destinations); } + List getSources() { return List.copyOf(sources); } + boolean getBolded() { return isBold; } + boolean getDynamic() { return isDyn; } + boolean getFull() { return isFull; } + boolean getMatchedElementsOnly() { return isMEO; } + + void addDestination(String dst) { destinations.add(dst); } + void addSource(String src) { sources.add(src); } + void setBold(boolean value) { this.isBold = value; } + void setDynamic() { this.isDyn = true; } + void setFull() { this.isFull = true; } + void setMatchedElementsOnly() { this.isMEO = true; } + void setType(ParsedType value) { + verifyThat(type == null, "Cannot change type from ", type, "to", value); + this.type = value; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java new file mode 100644 index 00000000000..9c3206a333a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java @@ -0,0 +1,224 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.tensor.TensorType; + +/** + * This class holds the extracted information after parsing a type + * declaration (typically for a field). Since types can be complex, + * struct names (known or unknown), or even document names, this class + * is somewhat complicated. + * @author arnej27959 + **/ +class ParsedType { + public enum Variant { + NONE, + BUILTIN, + POSITION, + TENSOR, + ARRAY, WSET, MAP, + DOC_REFERENCE, + ANN_REFERENCE, + STRUCT, + DOCUMENT, + UNKNOWN + } + + private final String name; + private final ParsedType keyType; + private final ParsedType valType; + private final TensorType tensorType; + private Variant variant; + private boolean createIfNonExistent = false; + private boolean removeIfZero = false; + + public String toString() { + var buf = new StringBuilder(); + buf.append("[type ").append(variant).append("] {"); + switch (variant) { + case NONE: + break; + case BUILTIN: + buf.append(name); + break; + case POSITION: + buf.append(name); + break; + case TENSOR: + buf.append(tensorType.toString()); + break; + case ARRAY: buf + .append(" array<") + .append(valType.toString()) + .append("> "); + break; + case WSET: buf + .append(" weightedset<") + .append(valType.toString()) + .append(">"); + if (createIfNonExistent) buf.append(",createIfNonExistent"); + if (removeIfZero) buf.append(",removeIfZero"); + buf.append(" "); + break; + case MAP: buf + .append(" map<") + .append(keyType.toString()) + .append(",") + .append(valType.toString()) + .append("> "); + break; + case DOC_REFERENCE: buf + .append(" reference<") + .append(valType.toString()) + .append("> "); + break; + case ANN_REFERENCE: buf + .append(" ") + .append(toString()) + .append(" "); + break; + case STRUCT: + case DOCUMENT: + case UNKNOWN: + buf.append(" ").append(name).append(" "); + break; + } + buf.append("}"); + return buf.toString(); + } + + private static Variant guessVariant(String name) { + switch (name) { + case "bool": return Variant.BUILTIN; + case "byte": return Variant.BUILTIN; + case "int": return Variant.BUILTIN; + case "long": return Variant.BUILTIN; + case "string": return Variant.BUILTIN; + case "float": return Variant.BUILTIN; + case "double": return Variant.BUILTIN; + case "uri": return Variant.BUILTIN; + case "predicate": return Variant.BUILTIN; + case "raw": return Variant.BUILTIN; + case "tag": return Variant.BUILTIN; + case "position": return Variant.POSITION; + case "float16": return Variant.BUILTIN; + } + return Variant.UNKNOWN; + } + + public String name() { return name; } + public Variant getVariant() { return variant; } + public ParsedType mapKeyType() { assert(variant == Variant.MAP); return keyType; } + public ParsedType mapValueType() { assert(variant == Variant.MAP); return valType; } + public ParsedType nestedType() { assert(variant == Variant.ARRAY || variant == Variant.WSET); assert(valType != null); return valType; } + public boolean getCreateIfNonExistent() { assert(variant == Variant.WSET); return this.createIfNonExistent; } + public boolean getRemoveIfZero() { assert(variant == Variant.WSET); return this.removeIfZero; } + public ParsedType getReferencedDocumentType() { assert(variant == Variant.DOC_REFERENCE); return valType; } + public TensorType getTensorType() { assert(variant == Variant.TENSOR); return tensorType; } + + public String getNameOfReferencedAnnotation() { + assert(variant == Variant.ANN_REFERENCE); + String prefix = "annotationreference<"; + int fromPos = prefix.length(); + int toPos = name.length() - 1; + return name.substring(fromPos, toPos); + } + + private ParsedType(String name, Variant variant) { + this(name, variant, null, null, null); + } + private ParsedType(String name, Variant variant, ParsedType vt) { + this(name, variant, null, vt, null); + } + private ParsedType(String name, Variant variant, ParsedType kt, ParsedType vt) { + this(name, variant, kt, vt, null); + } + private ParsedType(String name, Variant variant, ParsedType kt, ParsedType vt, TensorType tType) { + this.name = name; + this.variant = variant; + this.keyType = kt; + this.valType = vt; + this.tensorType = tType; + } + + static ParsedType mapType(ParsedType kt, ParsedType vt) { + assert(kt != null); + assert(vt != null); + String name = "map<" + kt.name() + "," + vt.name() + ">"; + return new ParsedType(name, Variant.MAP, kt, vt); + } + static ParsedType arrayOf(ParsedType vt) { + assert(vt != null); + return new ParsedType("array<" + vt.name() + ">", Variant.ARRAY, vt); + } + static ParsedType wsetOf(ParsedType vt) { + assert(vt != null); + if (vt.getVariant() != Variant.BUILTIN) { + throw new IllegalArgumentException("weightedset of complex type '" + vt + "' is not supported"); + } + switch (vt.name()) { + // allowed types: + case "bool": + case "byte": + case "int": + case "long": + case "string": + case "uri": + break; + case "predicate": + case "raw": + case "tag": + throw new IllegalArgumentException("weightedset of complex type '" + vt + "' is not supported"); + case "float16": + case "float": + case "double": + /* TODO Vespa 8: + throw new IllegalArgumentException("weightedset of inexact type '" + vt + "' is not supported"); + */ + break; + default: + throw new IllegalArgumentException("weightedset of unknown type '" + vt + "' is not supported"); + } + return new ParsedType("weightedset<" + vt.name() + ">", Variant.WSET, vt); + } + static ParsedType documentRef(ParsedType docType) { + assert(docType != null); + return new ParsedType("reference<" + docType.name + ">", Variant.DOC_REFERENCE, docType); + } + static ParsedType annotationRef(String name) { + return new ParsedType("annotationreference<" + name + ">", Variant.ANN_REFERENCE); + } + static ParsedType tensorType(TensorType tType) { + assert(tType != null); + return new ParsedType(tType.toString(), Variant.TENSOR, null, null, tType); + } + static ParsedType fromName(String name) { + return new ParsedType(name, guessVariant(name)); + } + static ParsedType documentType(String name) { + return new ParsedType(name, Variant.DOCUMENT); + } + + void setCreateIfNonExistent(boolean value) { + if (variant != Variant.WSET) { + throw new IllegalArgumentException("CreateIfNonExistent only valid for weightedset, not " + variant); + } + this.createIfNonExistent = value; + } + + void setRemoveIfZero(boolean value) { + if (variant != Variant.WSET) { + throw new IllegalArgumentException("RemoveIfZero only valid for weightedset, not " + variant); + } + this.removeIfZero = value; + } + + void setVariant(Variant value) { + if (variant == value) return; // already OK + if (variant != Variant.UNKNOWN) { + throw new IllegalArgumentException("setVariant(" + value + ") only valid for UNKNOWN, not: " + variant); + } + // maybe even more checking would be useful + this.variant = value; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/schema/parser/SimpleCharStream.java new file mode 100644 index 00000000000..0a53e0477ac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/SimpleCharStream.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.javacc.FastCharStream; + +/** + * @author Simon Thoresen Hult + */ +public class SimpleCharStream extends FastCharStream implements com.yahoo.schema.parser.CharStream, + com.yahoo.vespa.indexinglanguage.parser.CharStream { + + public SimpleCharStream(String input) { + super(input); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/Utils.java b/config-model/src/main/java/com/yahoo/schema/parser/Utils.java new file mode 100644 index 00000000000..cdb299c92df --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/parser/Utils.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +/** + * @author bjorncs + */ +class Utils { + + private Utils() {} + + // Separate class since javacc does not accept Java code using lambdas + static int count(String str, char ch) { + return (int) str.chars().filter(c -> c == ch).count(); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java new file mode 100644 index 00000000000..d96cd88f6be --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableImportedComplexSDField; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.stream.Stream; + +/** + * Adds the attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported + * field as source. + * + * @author bjorncs + */ +public class AddAttributeTransformToSummaryOfImportedFields extends Processor { + + public AddAttributeTransformToSummaryOfImportedFields(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + schema.allImportedFields() + .forEach(field -> setTransform(field)); + } + + private Stream getSummaryFieldsForImportedField(ImmutableSDField importedField) { + return schema.getSummaryFields(importedField).stream(); + } + + private void setTransform(ImmutableSDField field) { + if (field instanceof ImmutableImportedComplexSDField) { + getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); + } else { + getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); + } + } + + private static void setAttributeTransform(SummaryField summaryField) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + } + + private static void setAttributeCombinerTransform(SummaryField summaryField) { + if (summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER) { + summaryField.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } else { + summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java new file mode 100644 index 00000000000..ca81301da73 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.Field; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * This processor creates a {@link com.yahoo.schema.document.SDDocumentType} for each {@link Schema} + * object which holds all the data that search + * associates with a document described in a search definition file. This includes all extra fields, summary fields and + * implicit fields. All non-indexed and non-summary fields are discarded. + */ +public class AddExtraFieldsToDocument extends Processor { + + AddExtraFieldsToDocument(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + SDDocumentType document = schema.getDocument(); + if (document != null) { + for (SDField field : schema.extraFieldList()) { + addSdField(schema, document, field, validate); + } + for (var docsum : schema.getSummaries().values()) { + for (var summaryField : docsum.getSummaryFields().values()) { + switch (summaryField.getTransform()) { + case NONE: + case BOLDED: + case DYNAMICBOLDED: + case DYNAMICTEASER: + case TEXTEXTRACTOR: + addSummaryField(schema, document, summaryField, validate); + break; + default: + // skip: generated from attribute or similar, + // so does not need to be included as an extra + // field in the document type + } + } + } + } + } + + private void addSdField(Schema schema, SDDocumentType document, SDField field, boolean validate) { + if (! field.hasIndex() && field.getAttributes().isEmpty()) { + return; + } + for (Attribute atr : field.getAttributes().values()) { + if (!atr.getName().equals(field.getName())) { + addField(schema, document, new SDField(document, atr.getName(), atr.getDataType()), validate); + } + } + addField(schema, document, field, validate); + } + + private void addSummaryField(Schema schema, SDDocumentType document, SummaryField field, boolean validate) { + Field docField = document.getField(field.getName()); + if (docField == null) { + ImmutableSDField existingField = schema.getField(field.getName()); + if (existingField == null) { + SDField newField = new SDField(document, field.getName(), field.getDataType()); + newField.setIsExtraField(true); + document.addField(newField); + } else if (!existingField.isImportedField()) { + document.addField(existingField.asField()); + } + } else if (!docField.getDataType().equals(field.getDataType())) { + if (validate) + throw newProcessException(schema, field, "Summary field has conflicting type."); + } + } + + private void addField(Schema schema, SDDocumentType document, Field field, boolean validate) { + if (document.getField(field.getName()) != null && !(document.getField(field.getName()) == field)) { + if (validate) + throw newProcessException(schema, field, "Field shadows another."); + } + document.addField(field); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java new file mode 100644 index 00000000000..6c2d62f37cb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java @@ -0,0 +1,135 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryField.Source; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/* + * Adjusts position summary fields by adding derived summary fields (.distance and .position) and setting summary + * transform and source. + */ +public class AdjustPositionSummaryFields extends Processor { + + public AdjustPositionSummaryFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + private boolean useV8GeoPositions = false; + + @Override + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); + process(validate, documentsOnly); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (DocumentSummary summary : schema.getSummaries().values()) { + scanSummary(summary); + } + } + + private void scanSummary(DocumentSummary summary) { + for (SummaryField summaryField : summary.getSummaryFields().values()) { + if ( ! GeoPos.isAnyPos(summaryField.getDataType())) continue; + + String originalSource = summaryField.getSingleSource(); + if (originalSource.indexOf('.') == -1) { // Eliminate summary fields with pos.x or pos.y as source + ImmutableSDField sourceField = schema.getField(originalSource); + if (sourceField != null) { + String zCurve = null; + if (sourceField.getDataType().equals(summaryField.getDataType())) { + zCurve = PositionDataType.getZCurveFieldName(originalSource); + } else if (sourceField.getDataType().equals(makeZCurveDataType(summaryField.getDataType())) && + hasZCurveSuffix(originalSource)) { + zCurve = originalSource; + } + if (zCurve != null) { + if (hasPositionAttribute(zCurve)) { + Source source = new Source(zCurve); + adjustPositionField(summary, summaryField, source); + } else if (sourceField.isImportedField() || !summaryField.getName().equals(originalSource)) { + fail(summaryField, "No position attribute '" + zCurve + "'"); + } + } + } + } + } + } + + private void adjustPositionField(DocumentSummary summary, SummaryField summaryField, Source source) { + summaryField.setTransform(SummaryTransform.GEOPOS); + summaryField.getSources().clear(); + summaryField.addSource(source); + ensureSummaryField(summary, + PositionDataType.getPositionSummaryFieldName(summaryField.getName()), + DataType.getArray(DataType.STRING), + source, + SummaryTransform.POSITIONS); + ensureSummaryField(summary, + PositionDataType.getDistanceSummaryFieldName(summaryField.getName()), + DataType.INT, + source, + SummaryTransform.DISTANCE); + } + + private void ensureSummaryField(DocumentSummary summary, String fieldName, DataType dataType, Source source, SummaryTransform transform) { + SummaryField oldField = schema.getSummaryField(fieldName); + if (oldField == null) { + if (useV8GeoPositions) return; + SummaryField newField = new SummaryField(fieldName, dataType, transform); + newField.addSource(source); + summary.add(newField); + return; + } + if (!oldField.getDataType().equals(dataType)) { + fail(oldField, "exists with type '" + oldField.getDataType().toString() + "', should be of type '" + dataType.toString() + "'"); + } + if (oldField.getTransform() != transform) { + fail(oldField, "has summary transform '" + oldField.getTransform().toString() + "', should have transform '" + transform.toString() + "'"); + } + if (oldField.getSourceCount() != 1 || !oldField.getSingleSource().equals(source.getName())) { + fail(oldField, "has source '" + oldField.getSources().toString() + "', should have source '" + source + "'"); + } + if (useV8GeoPositions) return; + summary.add(oldField); + } + + private boolean hasPositionAttribute(String name) { + Attribute attribute = schema.getAttribute(name); + if (attribute == null) { + ImmutableSDField field = schema.getField(name); + if (field != null && field.isImportedField()) { + attribute = field.getAttribute(); + } + } + return attribute != null && attribute.isPosition(); + } + + private static boolean hasZCurveSuffix(String name) { + String suffix = PositionDataType.getZCurveFieldName(""); + return name.length() > suffix.length() && name.substring(name.length() - suffix.length()).equals(suffix); + } + + private static DataType makeZCurveDataType(DataType dataType) { + return dataType instanceof ArrayDataType ? DataType.getArray(DataType.LONG) : DataType.LONG; + } + + private void fail(SummaryField summaryField, String msg) { + throw newProcessException(schema.getName(), summaryField.getName(), msg); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributeProperties.java new file mode 100644 index 00000000000..6c7dbaecbfb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributeProperties.java @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Checks that attribute properties only are set for attributes that have data (are created by an indexing statement). + * + * @author hmusum + */ +public class AttributeProperties extends Processor { + + public AttributeProperties(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (ImmutableSDField field : schema.allConcreteFields()) { + String fieldName = field.getName(); + + // For each attribute, check if the attribute has been created + // by an indexing statement. + for (Attribute attribute : field.getAttributes().values()) { + if (attributeCreated(field, attribute.getName())) { + continue; + } + // Check other fields or statements that may have created this attribute. + boolean created = false; + for (SDField f : schema.allConcreteFields()) { + // Checking against the field we are looking at + if (!f.getName().equals(fieldName)) { + if (attributeCreated(f, attribute.getName())) { + created = true; + break; + } + } + } + if (validate && !created) { + throw new IllegalArgumentException("Attribute '" + attribute.getName() + "' in field '" + + field.getName() + "' is not created by the indexing statement"); + } + } + } + } + + /** + * Checks if the attribute has been created bye an indexing statement in this field. + * + * @param field a searchdefinition field + * @param attributeName name of the attribute + * @return true if the attribute has been created by this field, else false + */ + static boolean attributeCreated(ImmutableSDField field, String attributeName) { + if ( ! field.doesAttributing()) { + return false; + } + for (Attribute attribute : field.getAttributes().values()) { + if (attribute.getName().equals(attributeName)) { + return true; + } + } + return false; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java new file mode 100644 index 00000000000..415f23f2786 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.MatchType; +import com.yahoo.document.NumericDataType; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Fields that derive to attribute(s) and no indices should use the WORD indexing form, + * in a feeble attempt to match the most peoples expectations as closely as possible. + * + * @author Vegard Havdal + */ +public class AttributesImplicitWord extends Processor { + + public AttributesImplicitWord(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (ImmutableSDField field : schema.allConcreteFields()) { + processFieldRecursive(field); + } + } + + private void processFieldRecursive(ImmutableSDField field) { + processField(field); + for (ImmutableSDField structField : field.getStructFields()) { + processFieldRecursive(structField); + } + } + + private void processField(ImmutableSDField field) { + if (fieldImplicitlyWordMatch(field)) { + field.getMatching().setType(MatchType.WORD); + } + } + + private boolean fieldImplicitlyWordMatch(ImmutableSDField field) { + // numeric types should not trigger exact-match query parsing + DataType dt = field.getDataType().getPrimitiveType(); + if (dt != null && dt instanceof NumericDataType) { + return false; + } + return (! field.hasIndex() + && !field.getAttributes().isEmpty() + && field.getIndices().isEmpty() + && !field.getMatching().isTypeUserSet()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java b/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java new file mode 100644 index 00000000000..53a3d462d54 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Checks that bolding or dynamic summary is turned on only for text fields. Throws exception if it is turned on for any + * other fields (otherwise will cause indexing failure) + * + * @author hmusum + */ +public class Bolding extends Processor { + + public Bolding(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + for (ImmutableSDField field : schema.allConcreteFields()) { + for (SummaryField summary : field.getSummaryFields().values()) { + if (summary.getTransform().isBolded() && + !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) + { + throw new IllegalArgumentException("'bolding: on' for non-text field " + + "'" + field.getName() + "'" + + " (" + summary.getDataType() + ")" + + " is not allowed"); + } else if (summary.getTransform().isDynamic() && + !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) + { + throw new IllegalArgumentException("'summary: dynamic' for non-text field " + + "'" + field.getName() + "'" + + " (" + summary.getDataType() + ")" + + " is not allowed"); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java new file mode 100644 index 00000000000..bdb1eed4b10 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Validates attribute fields using bool type, ensuring the collection type is supported. + * + * Currently, only the single value bool type is supported. + * + * @author geirst + */ +public class BoolAttributeValidator extends Processor { + + public BoolAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var field : schema.allConcreteFields()) { + var attribute = field.getAttribute(); + if (attribute == null) { + continue; + } + if (attribute.getType().equals(Attribute.Type.BOOL) && + !attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE)) { + fail(schema, field, "Only single value bool attribute fields are supported"); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/schema/processing/BuiltInFieldSets.java new file mode 100644 index 00000000000..514cbf225fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/BuiltInFieldSets.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DocumentType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.Field; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document + * + * @author Vegard Havdal + */ +public class BuiltInFieldSets extends Processor { + + public static final String SEARCH_FIELDSET_NAME = "[search]"; // Public due to oddities in position handling. + public static final String INTERNAL_FIELDSET_NAME = "[internal]"; // This one populated from misc places + + public BuiltInFieldSets(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + addDocumentFieldSet(); + addSearchFieldSet(); + // "Hook" the field sets on search onto the document types, since we will include them + // on the document configs + schema.getDocument().setFieldSets(schema.fieldSets()); + } + + private void addSearchFieldSet() { + for (SDField searchField : schema.extraFieldList()) { + schema.fieldSets().addBuiltInFieldSetItem(SEARCH_FIELDSET_NAME, searchField.getName()); + } + } + + private void addDocumentFieldSet() { + for (Field docField : schema.getDocument().fieldSet()) { + if (docField instanceof SDField && ((SDField) docField).isExtraField()) { + continue; // skip + } + schema.fieldSets().addBuiltInFieldSetItem(DocumentType.DOCUMENT, docField.getName()); + } + } + + + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java new file mode 100644 index 00000000000..5bb5079fab6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -0,0 +1,216 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; + +/** + * Adds a "fieldName_zcurve" long attribute and "fieldName.distance" and "FieldName.position" summary fields to all position type fields. + * + * @author bratseth + */ +public class CreatePositionZCurve extends Processor { + + private final SDDocumentType repo; + + public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + this.repo = schema.getDocument(); + } + + private boolean useV8GeoPositions = false; + + @Override + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); + process(validate, documentsOnly); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + DataType fieldType = field.getDataType(); + if ( ! isSupportedPositionType(fieldType)) continue; + + if (validate && field.doesIndexing()) { + fail(schema, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " + + "replace 'index' statement with 'attribute'."); + } + + if ( ! field.doesAttributing()) continue; + + boolean doesSummary = field.doesSummarying(); + + String fieldName = field.getName(); + field.getAttributes().remove(fieldName); + + String zName = PositionDataType.getZCurveFieldName(fieldName); + SDField zCurveField = createZCurveField(field, zName, validate); + schema.addExtraField(zCurveField); + schema.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName()); + + // configure summary + Collection summaryTo = removeSummaryTo(field); + if (! useV8GeoPositions) { + ensureCompatibleSummary(field, zName, + PositionDataType.getPositionSummaryFieldName(fieldName), + DataType.getArray(DataType.STRING), // will become "xmlstring" + SummaryTransform.POSITIONS, summaryTo, validate); + ensureCompatibleSummary(field, zName, + PositionDataType.getDistanceSummaryFieldName(fieldName), + DataType.INT, + SummaryTransform.DISTANCE, summaryTo, validate); + } + // clear indexing script + field.setIndexingScript(null); + SDField posX = field.getStructField(PositionDataType.FIELD_X); + if (posX != null) { + posX.setIndexingScript(null); + } + SDField posY = field.getStructField(PositionDataType.FIELD_Y); + if (posY != null) { + posY.setIndexingScript(null); + } + if (doesSummary) ensureCompatibleSummary(field, zName, + field.getName(), + field.getDataType(), + SummaryTransform.GEOPOS, summaryTo, validate); + } + } + + private SDField createZCurveField(SDField inputField, String fieldName, boolean validate) { + if (validate && schema.getConcreteField(fieldName) != null || schema.getAttribute(fieldName) != null) { + throw newProcessException(schema, null, "Incompatible position attribute '" + fieldName + + "' already created."); + } + boolean isArray = inputField.getDataType() instanceof ArrayDataType; + SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); + Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY : + Attribute.CollectionType.SINGLE); + attribute.setPosition(true); + attribute.setFastSearch(true); + field.addAttribute(attribute); + + ScriptExpression script = inputField.getIndexingScript(); + script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script); + script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script); + field.setIndexingScript(script); + return field; + } + + private void ensureCompatibleSummary(SDField field, String sourceName, String summaryName, DataType summaryType, + SummaryTransform summaryTransform, Collection summaryTo, boolean validate) { + SummaryField summary = schema.getSummaryField(summaryName); + if (summary == null) { + summary = new SummaryField(summaryName, summaryType, summaryTransform); + summary.addDestination("default"); + summary.addDestinations(summaryTo); + field.addSummaryField(summary); + } else if (!summary.getDataType().equals(summaryType)) { + if (validate) + fail(schema, field, "Incompatible summary field '" + summaryName + "' type " + summary.getDataType() + " already created."); + } else if (summary.getTransform() == SummaryTransform.NONE) { + summary.setTransform(summaryTransform); + summary.addDestination("default"); + summary.addDestinations(summaryTo); + } else if (summary.getTransform() != summaryTransform) { + deployLogger.logApplicationPackage(Level.WARNING, "Summary field " + summaryName + " has wrong transform: " + summary.getTransform()); + return; + } + SummaryField.Source source = new SummaryField.Source(sourceName); + summary.getSources().clear(); + summary.addSource(source); + } + + private Set removeSummaryTo(SDField field) { + Set summaryTo = new HashSet<>(); + Collection summaryFields = field.getSummaryFields().values(); + for (SummaryField summary : summaryFields) { + summaryTo.addAll(summary.getDestinations()); + } + field.removeSummaryFields(); + return summaryTo; + } + + private static boolean isSupportedPositionType(DataType dataType) { + return GeoPos.isAnyPos(dataType); + } + + private static class RemoveSummary extends ExpressionConverter { + + final String find; + + RemoveSummary(String find) { + this.find = find; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (!(exp instanceof SummaryExpression)) { + return false; + } + String fieldName = ((SummaryExpression)exp).getFieldName(); + return fieldName == null || fieldName.equals(find); + } + + @Override + protected Expression doConvert(Expression exp) { + return null; + } + } + + private static class PerformZCurve extends ExpressionConverter { + + final String find; + final String replace; + final boolean isArray; + + PerformZCurve(SDField find, String replace) { + this.find = find.getName(); + this.replace = replace; + this.isArray = find.getDataType() instanceof ArrayDataType; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (!(exp instanceof AttributeExpression)) { + return false; + } + String fieldName = ((AttributeExpression)exp).getFieldName(); + return fieldName == null || fieldName.equals(find); + } + + @Override + protected Expression doConvert(Expression exp) { + return new StatementExpression( + isArray ? new ForEachExpression(new ZCurveExpression()) : + new ZCurveExpression(), new AttributeExpression(replace)); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java new file mode 100644 index 00000000000..3209fd1703d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Propagates dictionary settings from field level to attribute level. + * Only applies to numeric fields with fast-search enabled. + * + * @author baldersheim + */ +public class DictionaryProcessor extends Processor { + public DictionaryProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + Attribute attribute = field.getAttribute(); + if (attribute == null) continue; + attribute.setCase(field.getMatching().getCase()); + Dictionary dictionary = field.getDictionary(); + if (dictionary == null) continue; + if (attribute.getDataType().getPrimitiveType() instanceof NumericDataType ) { + if (attribute.isFastSearch()) { + attribute.setDictionary(dictionary); + } else { + fail(schema, field, "You must specify 'attribute:fast-search' to allow dictionary control"); + } + } else if (attribute.getDataType().getPrimitiveType() == PrimitiveDataType.STRING) { + attribute.setDictionary(dictionary); + if (dictionary.getType() == Dictionary.Type.HASH) { + if (dictionary.getMatch() != Case.CASED) { + fail(schema, field, "hash dictionary require cased match"); + } + } + if (! dictionary.getMatch().equals(attribute.getCase())) { + fail(schema, field, "Dictionary casing '" + dictionary.getMatch() + "' does not match field match casing '" + attribute.getCase() + "'"); + } + } else { + fail(schema, field, "You can only specify 'dictionary:' for numeric or string fields"); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypes.java new file mode 100644 index 00000000000..a5b4ca9a71f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.Field; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Non-primitive key types for map and weighted set forbidden (though OK in document model) + * + * @author Vegard Havdal + */ +public class DisallowComplexMapAndWsetKeyTypes extends Processor { + + public DisallowComplexMapAndWsetKeyTypes(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + // TODO also traverse struct types to search for bad map or wset types. + // Do this after document manager is fixed, do not start using the static stuff on SDDocumentTypes any more. + for (SDField field : schema.allConcreteFields()) { + checkFieldType(field, field.getDataType()); + } + } + + private void checkFieldType(Field field, DataType dataType) { + if (dataType instanceof ArrayDataType) { + DataType nestedType = ((ArrayDataType) dataType).getNestedType(); + checkFieldType(field, nestedType); + } else if (dataType instanceof WeightedSetDataType) { + DataType nestedType = ((WeightedSetDataType) dataType).getNestedType(); + if ( ! (nestedType instanceof PrimitiveDataType)) { + fail(schema, field, "Weighted set must have a primitive key type."); + } + } else if (dataType instanceof MapDataType) { + DataType keyType = ((MapDataType) dataType).getKeyType(); + if ( ! (keyType instanceof PrimitiveDataType)) { + fail(schema, field, "Map key type must be a primitive type."); + } + checkFieldType(field, ((MapDataType) dataType).getValueType()); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java new file mode 100644 index 00000000000..0400292c7e5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * @author baldersheim + */ +public class DiversitySettingsValidator extends Processor { + + public DiversitySettingsValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + if (documentsOnly) return; + + for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { + if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { + validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); + } + } + } + private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { + String attributeName = settings.getAttribute(); + new AttributeValidator(schema.getName(), rankProfile.name(), + schema.getAttribute(attributeName), attributeName).validate(); + } + + private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { + + public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { + super(searchName, rankProfileName, attribute, attributeName); + } + + protected void validateThatAttributeIsSingleAndNotPredicate() { + if ( ! attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || + attribute.getType().equals(Attribute.Type.PREDICATE)) + { + failValidation("must be single value numeric, or enumerated attribute, but it is '" + + attribute.getDataType().getName() + "'"); + } + } + + @Override + public void validate() { + validateThatAttributeExists(); + validateThatAttributeIsSingleAndNotPredicate(); + } + + @Override + public String getValidationType() { + return "diversity"; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java new file mode 100644 index 00000000000..aa2d8293cac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -0,0 +1,109 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; +import com.yahoo.vespa.indexinglanguage.expressions.ExactExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of exact matching + * + * @author bratseth + */ +public class ExactMatch extends Processor { + + public static final String DEFAULT_EXACT_TERMINATOR = "@@"; + + ExactMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + processField(field, schema); + } + } + + private void processField(SDField field, Schema schema) { + MatchType matching = field.getMatching().getType(); + if (matching.equals(MatchType.EXACT) || matching.equals(MatchType.WORD)) { + implementExactMatch(field, schema); + } else if (field.getMatching().getExactMatchTerminator() != null) { + warn(schema, field, "exact-terminator requires 'exact' matching to have any effect."); + } + for (var structField : field.getStructFields()) { + processField(structField, schema); + } + } + + private void implementExactMatch(SDField field, Schema schema) { + field.setStemming(Stemming.NONE); + field.getNormalizing().inferLowercase(); + + if (field.getMatching().getType().equals(MatchType.WORD)) { + field.addQueryCommand("word"); + } else { // exact + String exactTerminator = DEFAULT_EXACT_TERMINATOR; + if (field.getMatching().getExactMatchTerminator() != null + && ! field.getMatching().getExactMatchTerminator().equals("")) { + exactTerminator = field.getMatching().getExactMatchTerminator(); + } else { + info(schema, field, + "With 'exact' matching, an exact-terminator is needed," + + " using default value '" + exactTerminator +"' as terminator"); + } + field.addQueryCommand("exact " + exactTerminator); + + // The following part illustrates how nice it would have been with canonical representation of indices + if (field.doesIndexing()) { + exactMatchSettingsForField(field); + } + } + ScriptExpression script = field.getIndexingScript(); + if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { + field.setIndexingScript((ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + } + } + + private void exactMatchSettingsForField(SDField field) { + field.getRanking().setFilter(true); + } + + private static class MyProvider extends TypedTransformProvider { + + MyProvider(Schema schema) { + super(ExactExpression.class, schema); + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression newTransform(DataType fieldType) { + Expression exp = new ExactExpression(); + if (fieldType instanceof CollectionDataType) { + exp = new ForEachExpression(exp); + } + return exp; + } + + } + +} + diff --git a/config-model/src/main/java/com/yahoo/schema/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/FastAccessValidator.java new file mode 100644 index 00000000000..224000e6b64 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/FastAccessValidator.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.stream.Collectors; + +/** + * Validates the use of the fast-access property. + * + * @author bjorncs + */ +public class FastAccessValidator extends Processor { + + public FastAccessValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + String invalidAttributes = schema.allFields() + .flatMap(field -> field.getAttributes().values().stream()) + .filter(FastAccessValidator::isIncompatibleAttribute) + .map(Attribute::getName) + .collect(Collectors.joining(", ")); + if ( ! invalidAttributes.isEmpty()) { + throw new IllegalArgumentException( + "For " + schema + ": The following attributes have a type that is incompatible with fast-access: " + + invalidAttributes + ". Predicate, tensor and reference attributes are incompatible with fast-access."); + } + } + + private static boolean isIncompatibleAttribute(Attribute attribute) { + return attribute.isFastAccess() && isTypeIncompatibleWithFastAccess(attribute.getType()); + } + + private static boolean isTypeIncompatibleWithFastAccess(Attribute.Type type) { + switch (type) { + case PREDICATE: + case TENSOR: + case REFERENCE: + return true; + default: + return false; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/FieldSetSettings.java b/config-model/src/main/java/com/yahoo/schema/processing/FieldSetSettings.java new file mode 100644 index 00000000000..f0c59ece1bf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/FieldSetSettings.java @@ -0,0 +1,107 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.Stemming; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Computes the right "index commands" for each fieldset in a search definition. + * + * @author vegardh + * @author bratseth + */ +// See also IndexInfo.addFieldSetCommands, which does more of this in a complicated way. +// That should be moved here, and done in the way the match setting is done below +// (this requires adding normalizing and stemming settings to FieldSet). +public class FieldSetSettings extends Processor { + + public FieldSetSettings(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (FieldSet fieldSet : schema.fieldSets().userFieldSets().values()) { + if (validate) + checkFieldNames(schema, fieldSet); + checkMatching(schema, fieldSet); + checkNormalization(schema, fieldSet); + checkStemming(schema, fieldSet); + } + } + + private void checkFieldNames(Schema schema, FieldSet fieldSet) { + for (String field : fieldSet.getFieldNames()) { + if (schema.getField(field) == null) + throw new IllegalArgumentException("For " + schema + ": Field '" + field + "' in " + + fieldSet + " does not exist."); + } + } + + private void checkMatching(Schema schema, FieldSet fieldSet) { + Matching matching = fieldSet.getMatching(); + for (String fieldName : fieldSet.getFieldNames()) { + ImmutableSDField field = schema.getField(fieldName); + Matching fieldMatching = field.getMatching(); + if (matching == null) { + matching = fieldMatching; + } else { + if ( ! matching.equals(fieldMatching)) { + warn(schema, field.asField(), + "The matching settings for the fields in " + fieldSet + " are inconsistent " + + "(explicitly or because of field type). This may lead to recall and ranking issues."); + return; + } + } + } + fieldSet.setMatching(matching); // Assign the uniquely determined matching to the field set + } + + private void checkNormalization(Schema schema, FieldSet fieldSet) { + NormalizeLevel.Level normalizing = null; + for (String fieldName : fieldSet.getFieldNames()) { + ImmutableSDField field = schema.getField(fieldName); + NormalizeLevel.Level fieldNorm = field.getNormalizing().getLevel(); + if (normalizing == null) { + normalizing = fieldNorm; + } else { + if ( ! normalizing.equals(fieldNorm)) { + warn(schema, field.asField(), + "The normalization settings for the fields in " + fieldSet + " are inconsistent " + + "(explicitly or because of field type). This may lead to recall and ranking issues."); + return; + } + } + } + } + + private void checkStemming(Schema schema, FieldSet fieldSet) { + Stemming stemming = null; + for (String fieldName : fieldSet.getFieldNames()) { + ImmutableSDField field = schema.getField(fieldName); + Stemming fieldStemming = field.getStemming(); + if (stemming == null) { + stemming = fieldStemming; + } else { + if ( ! stemming.equals(fieldStemming)) { + warn(schema, field.asField(), + "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+ + "' are inconsistent (explicitly or because of field type). " + + "This may lead to recall and ranking issues."); + return; + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/schema/processing/FilterFieldNames.java new file mode 100644 index 00000000000..28973c82d42 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/FilterFieldNames.java @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.RankProfile; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.logging.Level; + +/** + * Takes the fields and indexes that are of type rank filter, and stores those names on all rank profiles + * + * @author Vegard Havdal + */ +public class FilterFieldNames extends Processor { + + public FilterFieldNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + + for (SDField f : schema.allConcreteFields()) { + if (f.getRanking().isFilter()) { + filterField(f.getName()); + } + } + + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + Set filterFields = new LinkedHashSet<>(); + findFilterFields(schema, profile, filterFields); + for (Iterator itr = filterFields.iterator(); itr.hasNext(); ) { + String fieldName = itr.next(); + profile.filterFields().add(fieldName); + profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.EMPTY); + } + } + } + + private void filterField(String f) { + for (RankProfile rp : rankProfileRegistry.rankProfilesOf(schema)) { + rp.filterFields().add(f); + } + } + + private void findFilterFields(Schema schema, RankProfile profile, Set filterFields) { + for (Iterator itr = profile.declaredRankSettingIterator(); itr.hasNext(); ) { + RankProfile.RankSetting setting = itr.next(); + if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) && ((Boolean)setting.getValue())) + { + String fieldName = setting.getFieldName(); + if (schema.getConcreteField(fieldName) != null) { + if ( ! profile.filterFields().contains(fieldName)) { + filterFields.add(fieldName); + } + } else { + deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.name() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java new file mode 100644 index 00000000000..4080e37003f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java @@ -0,0 +1,232 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import java.util.logging.Level; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + +/** + * Makes implicitly defined summaries into explicit summaries + * + * @author bratseth + */ +public class ImplicitSummaries extends Processor { + + public ImplicitSummaries(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + DocumentSummary defaultSummary = schema.getSummariesInThis().get("default"); + if (defaultSummary == null) { + defaultSummary = new DocumentSummary("default", schema); + defaultSummary.setFromDisk(true); // As we add documentid to this + schema.addSummary(defaultSummary); + } + + for (SDField field : schema.allConcreteFields()) { + collectSummaries(field, schema, validate); + } + for (DocumentSummary documentSummary : schema.getSummaries().values()) { + documentSummary.purgeImplicits(); + } + } + + private void addSummaryFieldSources(SummaryField summaryField, SDField sdField) { + sdField.addSummaryFieldSources(summaryField); + } + + private void collectSummaries(SDField field, Schema schema, boolean validate) { + SummaryField addedSummaryField = null; + + // Implicit + String fieldName = field.getName(); + SummaryField fieldSummaryField = field.getSummaryField(fieldName); + if (fieldSummaryField == null && field.doesSummarying()) { + fieldSummaryField = new SummaryField(fieldName, field.getDataType()); + fieldSummaryField.setImplicit(true); + addSummaryFieldSources(fieldSummaryField, field); + fieldSummaryField.addDestination("default"); + field.addSummaryField(fieldSummaryField); + addedSummaryField = fieldSummaryField; + } + if (fieldSummaryField != null) { + for (String dest : fieldSummaryField.getDestinations()) { + DocumentSummary summary = schema.getSummariesInThis().get(dest); + if (summary != null) { + summary.add(fieldSummaryField); + } + } + } + + // Attribute prefetch + for (Attribute attribute : field.getAttributes().values()) { + if (attribute.getName().equals(fieldName)) { + if (addedSummaryField != null) { + addedSummaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + if (attribute.isPrefetch()) { + addPrefetchAttribute(attribute, field, schema); + } + } + } + + if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) { + addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + + // Position attributes + if (field.doesSummarying()) { + for (Attribute attribute : field.getAttributes().values()) { + if ( ! attribute.isPosition()) continue; + var distField = field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)); + if (distField != null) { + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); + attributePrefetchSummary.add(distField); + } + var posField = field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)); + if (posField != null) { + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); + attributePrefetchSummary.add(posField); + } + } + } + + // Explicits + for (SummaryField summaryField : field.getSummaryFields().values()) { + // Make sure we fetch from attribute here too + Attribute attribute = field.getAttributes().get(fieldName); + if (attribute != null && summaryField.getTransform() == SummaryTransform.NONE) { + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + if (isValid(summaryField, schema, validate)) { + addToDestinations(summaryField, schema); + } + } + + } + + private DocumentSummary getOrCreateAttributePrefetchSummary(Schema schema) { + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); + if (summary == null) { + summary = new DocumentSummary("attributeprefetch", schema); + schema.addSummary(summary); + } + return summary; + } + + + private void addPrefetchAttribute(Attribute attribute, SDField field, Schema schema) { + if (attribute.getPrefetchValue() == null) { // Prefetch by default - unless any summary makes this dynamic + // Check if there is an implicit dynamic definition + SummaryField fieldSummaryField = field.getSummaryField(attribute.getName()); + if (fieldSummaryField != null && fieldSummaryField.getTransform().isDynamic()) return; + + // Check if an explicit class makes it dynamic (first is enough, as all must be the same, checked later) + SummaryField explicitSummaryField = schema.getExplicitSummaryField(attribute.getName()); + if (explicitSummaryField != null && explicitSummaryField.getTransform().isDynamic()) return; + } + + DocumentSummary summary = getOrCreateAttributePrefetchSummary(schema); + SummaryField attributeSummaryField = new SummaryField(attribute.getName(), attribute.getDataType()); + attributeSummaryField.addSource(attribute.getName()); + attributeSummaryField.addDestination("attributeprefetch"); + attributeSummaryField.setTransform(SummaryTransform.ATTRIBUTE); + summary.add(attributeSummaryField); + } + + // Returns whether this is valid. Warns if invalid and ignorable. Throws if not ignorable. + private boolean isValid(SummaryField summaryField, Schema schema, boolean validate) { + if (summaryField.getTransform() == SummaryTransform.DISTANCE || + summaryField.getTransform() == SummaryTransform.POSITIONS) { + int sourceCount = summaryField.getSourceCount(); + if (validate && sourceCount != 1) { + throw newProcessException(schema.getName(), summaryField.getName(), + "Expected 1 source field, got " + sourceCount + "."); + } + String sourceName = summaryField.getSingleSource(); + if (validate && schema.getAttribute(sourceName) == null) { + throw newProcessException(schema.getName(), summaryField.getName(), + "Summary source attribute '" + sourceName + "' not found."); + } + return true; + } + + String fieldName = summaryField.getSourceField(); + SDField sourceField = schema.getConcreteField(fieldName); + if (validate && sourceField == null) { + throw newProcessException(schema, summaryField, "Source field '" + fieldName + "' does not exist."); + } + if (! sourceField.doesSummarying() && + summaryField.getTransform() != SummaryTransform.ATTRIBUTE && + summaryField.getTransform() != SummaryTransform.GEOPOS) + { + // Summary transform attribute may indicate that the ilscript was rewritten to remove summary + // by another search that uses this same field in inheritance. + deployLogger.logApplicationPackage(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField + + " is not creating a summary value in its indexing statement"); + return false; + } + + if (summaryField.getTransform().isDynamic() + && summaryField.getName().equals(sourceField.getName()) + && sourceField.doesAttributing()) { + Attribute attribute = sourceField.getAttributes().get(sourceField.getName()); + if (attribute != null) { + String destinations = "document summary 'default'"; + if (summaryField.getDestinations().size() >0) { + destinations = "document summaries " + summaryField.getDestinations(); + } + deployLogger.logApplicationPackage(Level.WARNING, + "Will fetch the disk summary value of " + sourceField + " in " + destinations + + " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " + + "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " + + "to this attribute, remove any bolding and dynamic snippeting from this field"); + // Note: The dynamic setting has already overridden the attribute map setting, + // so we do not need to actually do attribute.setSummary(false) here + // Also, we can not do this, since it makes it impossible to fetch this attribute + // in another summary + } + } + + return true; + } + + private void addToDestinations(SummaryField summaryField, Schema schema) { + if (summaryField.getDestinations().size() == 0) { + addToDestination("default", summaryField, schema); + } + else { + for (String destinationName : summaryField.getDestinations()) { + addToDestination(destinationName, summaryField, schema); + } + } + } + + private void addToDestination(String destinationName, SummaryField summaryField, Schema schema) { + DocumentSummary destination = schema.getSummariesInThis().get(destinationName); + if (destination == null) { + destination = new DocumentSummary(destinationName, schema); + schema.addSummary(destination); + destination.add(summaryField); + } + else { + SummaryField existingField= destination.getSummaryField(summaryField.getName()); + SummaryField merged = summaryField.mergeWith(existingField); + destination.add(merged); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaryFields.java new file mode 100644 index 00000000000..b17efbfe8e8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaryFields.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * This processor adds all implicit summary fields to all registered document summaries. If another field has already + * been registered with one of the implicit names, this processor will throw an {@link IllegalStateException}. + */ +public class ImplicitSummaryFields extends Processor { + + public ImplicitSummaryFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (DocumentSummary docsum : schema.getSummariesInThis().values()) { + if (docsum.inherited().isPresent()) continue; // Implicit fields are added to inheriting summaries through their parent + addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); + addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); + } + } + + private void addField(DocumentSummary docsum, SummaryField field, boolean validate) { + if (validate && docsum.getSummaryField(field.getName()) != null) { + throw new IllegalArgumentException("Summary class '" + docsum.getName() + "' uses reserved field name '" + + field.getName() + "'."); + } + docsum.add(field); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java new file mode 100644 index 00000000000..ee465be44f2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java @@ -0,0 +1,207 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.TemporaryImportedField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; + +/** + * Iterates all imported fields from schema parsing and validates and resolves them into concrete fields from referenced document types. + * + * @author geirst + */ +public class ImportedFieldsResolver extends Processor { + + private final Map importedFields = new LinkedHashMap<>(); + private final Optional references; + + public ImportedFieldsResolver(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + references = schema.getDocument().getDocumentReferences(); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + schema.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); + schema.setImportedFields(new ImportedFields(importedFields)); + } + + private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { + DocumentReference reference = validateDocumentReference(importedField); + ImmutableSDField targetField = getTargetField(importedField, reference); + if (GeoPos.isAnyPos(targetField)) { + resolveImportedPositionField(importedField, reference, targetField, validate); + } else if (isArrayOfSimpleStruct(targetField)) { + resolveImportedArrayOfStructField(importedField, reference, targetField, validate); + } else if (isMapOfSimpleStruct(targetField)) { + resolveImportedMapOfStructField(importedField, reference, targetField, validate); + } else if (isMapOfPrimitiveType(targetField)) { + resolveImportedMapOfPrimitiveField(importedField, reference, targetField, validate); + } else { + resolveImportedNormalField(importedField, reference, targetField, validate); + } + } + + private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), + reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); + ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); + resolveImportedNormalField(importedZCurveField, reference, targetZCurveField, validate); + ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + registerImportedField(importedField, null, importedStructField); + } + + private void resolveImportedArrayOfStructField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + resolveImportedNestedStructField(importedField, reference, importedStructField, targetField, validate); + registerImportedField(importedField, null, importedStructField); + } + + private void resolveImportedMapOfStructField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + ImportedComplexField importedMapField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName() + ".value", reference, targetField.getStructField("value")); + importedMapField.addNestedField(importedStructField); + resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("key"), validate); + resolveImportedNestedStructField(importedField, reference, importedStructField, importedStructField.targetField(), validate); + registerImportedField(importedField, null, importedMapField); + } + + private void makeImportedNormalField(TemporaryImportedField importedField, ImportedComplexField owner, String name, DocumentReference reference, ImmutableSDField targetField) { + ImportedField importedSimpleField = new ImportedSimpleField(name, reference, targetField); + registerImportedField(importedField, owner, importedSimpleField); + } + + private void registerImportedField(TemporaryImportedField temporaryImportedField, ImportedComplexField owner, ImportedField importedField) { + if (owner != null) { + owner.addNestedField(importedField); + } else { + if (importedFields.get(importedField.fieldName()) != null) { + fail(temporaryImportedField, importedField.fieldName(), targetFieldAsString(importedField.targetField().getName(), importedField.reference()) + ": Field already imported"); + } + importedFields.put(importedField.fieldName(), importedField); + } + } + + private static String makeImportedNestedFieldName(TemporaryImportedField importedField, ImmutableSDField targetNestedField) { + return importedField.fieldName() + targetNestedField.getName().substring(importedField.targetFieldName().length()); + } + + private boolean resolveImportedNestedField(TemporaryImportedField importedField, DocumentReference reference, + ImportedComplexField owner, ImmutableSDField targetNestedField, boolean requireAttribute) { + Attribute attribute = targetNestedField.getAttribute(); + String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); + if (attribute != null) { + makeImportedNormalField(importedField, owner, importedNestedFieldName, reference, targetNestedField); + } else if (requireAttribute) { + fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + + ": Is not an attribute field. Only attribute fields supported"); + } + return attribute != null; + } + + private void resolveImportedNestedStructField(TemporaryImportedField importedField, DocumentReference reference, + ImportedComplexField ownerField, ImmutableSDField targetNestedField, boolean validate) { + boolean foundAttribute = false; + for (ImmutableSDField targetStructField : targetNestedField.getStructFields()) { + if (resolveImportedNestedField(importedField, reference, ownerField, targetStructField, false)) { + foundAttribute = true; + }; + } + if (validate && !foundAttribute) { + String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); + fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + + ": Is not a struct containing an attribute field."); + } + } + + private void resolveImportedMapOfPrimitiveField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + ImportedComplexField importedMapField = new ImportedComplexField(importedField.fieldName(), reference, targetField); + resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("key"), validate); + resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("value"), validate); + registerImportedField(importedField, null, importedMapField); + } + + private void resolveImportedNormalField(TemporaryImportedField importedField, DocumentReference reference, + ImmutableSDField targetField, boolean validate) { + if (validate) { + validateTargetField(importedField, targetField, reference); + } + makeImportedNormalField(importedField, null, importedField.fieldName(), reference, targetField); + } + + private DocumentReference validateDocumentReference(TemporaryImportedField importedField) { + String referenceFieldName = importedField.referenceFieldName(); + DocumentReference reference = references.get().referenceMap().get(referenceFieldName); + if (reference == null) { + fail(importedField, "Reference field '" + referenceFieldName + "' not found"); + } + return reference; + } + + private ImmutableSDField getTargetField(TemporaryImportedField importedField, + DocumentReference reference) { + String targetFieldName = importedField.targetFieldName(); + Schema targetSchema = reference.targetSearch(); + ImmutableSDField targetField = targetSchema.getField(targetFieldName); + if (targetField == null) { + fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Not found"); + } + return targetField; + } + + private void validateTargetField(TemporaryImportedField importedField, + ImmutableSDField targetField, DocumentReference reference) { + if (!targetField.doesAttributing()) { + fail(importedField, targetFieldAsString(targetField.getName(), reference) + + ": Is not an attribute field. Only attribute fields supported"); + } else if (targetField.doesIndexing()) { + fail(importedField, targetFieldAsString(targetField.getName(), reference) + + ": Is an index field. Not supported"); + } else if (targetField.getDataType().equals(DataType.PREDICATE)) { + fail(importedField, targetFieldAsString(targetField.getName(), reference) + + ": Is of type 'predicate'. Not supported"); + } + } + + private static String targetFieldAsString(String targetFieldName, DocumentReference reference) { + return "Field '" + targetFieldName + "' via reference field '" + reference.referenceField().getName() + "'"; + } + + private void fail(TemporaryImportedField importedField, String msg) { + throw new IllegalArgumentException("For " + schema + ", import field '" + + importedField.fieldName() + "': " + msg); + } + + private void fail(TemporaryImportedField importedField, String importedNestedFieldName, String msg) { + if (importedField.fieldName().equals(importedNestedFieldName)) { + fail(importedField, msg); + } + throw new IllegalArgumentException("For " + schema + ", import field '" + + importedField.fieldName() + "' (nested to '" + importedNestedFieldName + "'): " + msg); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java new file mode 100644 index 00000000000..27101c47c7a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Because of the way the parser works (allowing any token as identifier), + * it is not practical to limit the syntax of field names there, do it here. + * Important to disallow dash, has semantic in IL. + * + * @author Vehard Havdal + */ +public class IndexFieldNames extends Processor { + + private static final String FIELD_NAME_REGEXP = "[a-zA-Z]\\w*"; + + public IndexFieldNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (SDField field : schema.allConcreteFields()) { + if ( ! field.getName().matches(FIELD_NAME_REGEXP) && ! legalDottedPositionField(field)) { + fail(schema, field, " Not a legal field name. Legal expression: " + FIELD_NAME_REGEXP); + } + } + } + + /** + * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. + * TODO Vespa 8: Rename to _position and _distance and delete this method. + * + * @param field an {@link com.yahoo.schema.document.SDField} + * @return true if allowed + */ + private boolean legalDottedPositionField(SDField field) { + return field.getName().endsWith(".position") || field.getName().endsWith(".distance"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java new file mode 100644 index 00000000000..88e84d5289f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * This processor modifies all indexing scripts so that they input the value of the owning field by default. It also + * ensures that all fields used as input exist. + * + * @author Simon Thoresen Hult + */ +public class IndexingInputs extends Processor { + + public IndexingInputs(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + ScriptExpression script = field.getIndexingScript(); + if (script == null) continue; + + String fieldName = field.getName(); + script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script); + script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script); + if (validate) + new VerifyInputExpression(schema, field).visit(script); + + field.setIndexingScript(script); + } + } + + private static class DefaultToCurrentField extends ExpressionConverter { + + final String fieldName; + + DefaultToCurrentField(String fieldName) { + this.fieldName = fieldName; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof InputExpression && ((InputExpression)exp).getFieldName() == null; + } + + @Override + protected Expression doConvert(Expression exp) { + return new InputExpression(fieldName); + } + } + + private static class EnsureInputExpression extends ExpressionConverter { + + final String fieldName; + + EnsureInputExpression(String fieldName) { + this.fieldName = fieldName; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof StatementExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + if (exp.requiredInputType() != null) { + return new StatementExpression(new InputExpression(fieldName), exp); + } else { + return exp; + } + } + } + + private class VerifyInputExpression extends ExpressionVisitor { + + private final Schema schema; + private final SDField field; + + public VerifyInputExpression(Schema schema, SDField field) { + this.schema = schema; + this.field = field; + } + + @Override + protected void doVisit(Expression exp) { + if ( ! (exp instanceof InputExpression)) return; + String inputField = ((InputExpression)exp).getFieldName(); + if (schema.getField(inputField).hasFullIndexingDocprocRights()) return; + + fail(schema, field, "Indexing script refers to field '" + inputField + "' which does not exist " + + "in document type '" + schema.getDocument().getName() + "', and is not a mutable attribute."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java new file mode 100644 index 00000000000..ea65a223686 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -0,0 +1,144 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.*; + +/** + * This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents + * any output expression from writing to any field except for the owning field. Finally, for SummaryExpression, + * this processor expands to write all appropriate summary fields. + * + * @author Simon Thoresen Hult + */ +public class IndexingOutputs extends Processor { + + public IndexingOutputs(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + ScriptExpression script = field.getIndexingScript(); + if (script == null) continue; + + Set summaryFields = new TreeSet<>(); + findSummaryTo(schema, field, summaryFields, summaryFields); + MyConverter converter = new MyConverter(schema, field, summaryFields, validate); + field.setIndexingScript((ScriptExpression)converter.convert(script)); + } + } + + public void findSummaryTo(Schema schema, SDField field, Set dynamicSummary, Set staticSummary) { + var summaryFields = schema.getSummaryFields(field); + if (summaryFields.isEmpty()) { + fillSummaryToFromField(field, dynamicSummary, staticSummary); + } else { + fillSummaryToFromSearch(schema, field, summaryFields, dynamicSummary, staticSummary); + } + } + + private void fillSummaryToFromSearch(Schema schema, SDField field, List summaryFields, + Set dynamicSummary, Set staticSummary) { + for (SummaryField summaryField : summaryFields) { + fillSummaryToFromSummaryField(schema, field, summaryField, dynamicSummary, staticSummary); + } + } + + private void fillSummaryToFromSummaryField(Schema schema, SDField field, SummaryField summaryField, + Set dynamicSummary, Set staticSummary) { + SummaryTransform summaryTransform = summaryField.getTransform(); + String summaryName = summaryField.getName(); + if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) { + // Avoid writing to summary fields that have more than a single input field, as that is handled by the + // summary rewriter in the search core. + return; + } + if (summaryTransform.isDynamic()) { + DataType fieldType = field.getDataType(); + if (fieldType != DataType.URI && fieldType != DataType.STRING) { + warn(schema, field, "Dynamic summaries are only supported for fields of type " + + "string, ignoring summary field '" + summaryField.getName() + + "' for sd field '" + field.getName() + "' of type " + + fieldType.getName() + "."); + return; + } + dynamicSummary.add(summaryName); + } else if (summaryTransform != SummaryTransform.ATTRIBUTE) { + staticSummary.add(summaryName); + } + } + + private static void fillSummaryToFromField(SDField field, Set dynamicSummary, Set staticSummary) { + for (SummaryField summaryField : field.getSummaryFields().values()) { + String summaryName = summaryField.getName(); + if (summaryField.getTransform().isDynamic()) { + dynamicSummary.add(summaryName); + } else { + staticSummary.add(summaryName); + } + } + } + + private class MyConverter extends ExpressionConverter { + + final Schema schema; + final Field field; + final Set summaryFields; + final boolean validate; + + MyConverter(Schema schema, Field field, Set summaryFields, boolean validate) { + this.schema = schema; + this.field = field; + this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields; + this.validate = validate; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if ( ! (exp instanceof OutputExpression)) { + return false; + } + String fieldName = ((OutputExpression)exp).getFieldName(); + if (fieldName == null) { + return true; // inject appropriate field name + } + if ( validate && ! fieldName.equals(field.getName())) { + fail(schema, field, "Indexing expression '" + exp + "' attempts to write to a field other than '" + + field.getName() + "'."); + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + List ret = new LinkedList<>(); + if (exp instanceof AttributeExpression) { + ret.add(new AttributeExpression(field.getName())); + } else if (exp instanceof IndexExpression) { + ret.add(new IndexExpression(field.getName())); + } else if (exp instanceof SummaryExpression) { + for (String fieldName : summaryFields) { + ret.add(new SummaryExpression(fieldName)); + } + } else { + throw new UnsupportedOperationException(exp.getClass().getName()); + } + return new StatementExpression(ret); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java new file mode 100644 index 00000000000..d8c1fb3125f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -0,0 +1,164 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldTypeAdapter; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingValidation extends Processor { + + IndexingValidation(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + VerificationContext context = new VerificationContext(new MyAdapter(schema)); + for (SDField field : schema.allConcreteFields()) { + ScriptExpression script = field.getIndexingScript(); + try { + script.verify(context); + MyConverter converter = new MyConverter(); + for (StatementExpression exp : script) { + converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch + } + } catch (VerificationException e) { + fail(schema, field, "For expression '" + e.getExpression() + "': " + e.getMessage()); + } + } + } + + private static class MyConverter extends ExpressionConverter { + + final Set outputs = new HashSet<>(); + final Set prevNames = new HashSet<>(); + + @Override + protected ExpressionConverter branch() { + MyConverter ret = new MyConverter(); + ret.outputs.addAll(outputs); + ret.prevNames.addAll(prevNames); + return ret; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (exp instanceof OutputExpression) { + String fieldName = ((OutputExpression)exp).getFieldName(); + if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) { + throw new VerificationException(exp, "Attempting to assign conflicting values to field '" + + fieldName + "'."); + } + outputs.add(fieldName); + prevNames.add(fieldName); + } + if (exp.createdOutputType() != null) { + prevNames.clear(); + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + throw new UnsupportedOperationException(); + } + } + + private static class MyAdapter implements FieldTypeAdapter { + + final Schema schema; + + MyAdapter(Schema schema) { + this.schema = schema; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + SDField field = schema.getDocumentField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Input field '" + fieldName + "' not found."); + } + return field.getDataType(); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + String fieldDesc; + DataType fieldType; + if (exp instanceof AttributeExpression) { + Attribute attribute = schema.getAttribute(fieldName); + if (attribute == null) { + throw new VerificationException(exp, "Attribute '" + fieldName + "' not found."); + } + fieldDesc = "attribute"; + fieldType = attribute.getDataType(); + } else if (exp instanceof IndexExpression) { + SDField field = schema.getConcreteField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Index field '" + fieldName + "' not found."); + } + fieldDesc = "index field"; + fieldType = field.getDataType(); + } else if (exp instanceof SummaryExpression) { + SummaryField field = schema.getSummaryField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Summary field '" + fieldName + "' not found."); + } + fieldDesc = "summary field"; + fieldType = field.getDataType(); + } else { + throw new UnsupportedOperationException(); + } + if ( ! fieldType.isAssignableFrom(valueType) && + ! fieldType.isAssignableFrom(createCompatType(valueType))) { + throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + + " '" + fieldName + "' which is " + fieldType.getName() + "."); + } + } + + private static DataType createCompatType(DataType origType) { + if (origType instanceof ArrayDataType) { + return DataType.getArray(createCompatType(((ArrayDataType)origType).getNestedType())); + } else if (origType instanceof MapDataType) { + MapDataType mapType = (MapDataType)origType; + return DataType.getMap(createCompatType(mapType.getKeyType()), + createCompatType(mapType.getValueType())); + } else if (origType instanceof WeightedSetDataType) { + return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType())); + } else if (GeoPos.isPos(origType)) { + return DataType.LONG; + } else { + return origType; + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValues.java new file mode 100644 index 00000000000..fa4b7d2bc40 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValues.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.Field; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingValues extends Processor { + + public IndexingValues(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (Field field : schema.getDocument().fieldSet()) { + SDField sdField = (SDField)field; + if ( ! sdField.isExtraField()) { + new RequireThatDocumentFieldsAreImmutable(field).convert(sdField.getIndexingScript()); + } + } + } + + private class RequireThatDocumentFieldsAreImmutable extends ExpressionConverter { + + final Field field; + Expression mutatedBy; + + RequireThatDocumentFieldsAreImmutable(Field field) { + this.field = field; + } + + @Override + public ExpressionConverter branch() { + return clone(); + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (exp instanceof OutputExpression && mutatedBy != null) { + throw newProcessException(schema, field, + "Indexing expression '" + mutatedBy + "' attempts to modify the value of the " + + "document field '" + field.getName() + "'. Use a field outside the document " + + "block instead."); + } + if (exp instanceof InputExpression && ((InputExpression)exp).getFieldName().equals(field.getName())) { + mutatedBy = null; + } else if (exp.createdOutputType() != null) { + mutatedBy = exp; + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java new file mode 100644 index 00000000000..1d8480a8e99 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.NumericDataType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Set; + +/** + * Replaces the 'index' statement of all numerical fields to 'attribute' because we no longer support numerical indexes. + * + * @author baldersheim + */ +public class IntegerIndex2Attribute extends Processor { + + public IntegerIndex2Attribute(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { + if (field.getIndex(field.getName()) != null + && ! (field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue; + ScriptExpression script = field.getIndexingScript(); + Set attributeNames = new HashSet<>(); + new MyVisitor(attributeNames).visit(script); + field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); + warn(schema, field, "Changed to attribute because numerical indexes (field has type " + + field.getDataType().getName() + ") is not currently supported." + + " Index-only settings may fail. Ignore this warning for streaming search."); + } + } + } + + private static class MyVisitor extends ExpressionVisitor { + + final Set attributeNames; + + public MyVisitor(Set attributeNames) { + this.attributeNames = attributeNames; + } + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof AttributeExpression) { + attributeNames.add(((AttributeExpression)exp).getFieldName()); + } + } + } + + private static class MyConverter extends ExpressionConverter { + + final Set attributeNames; + + public MyConverter(Set attributeNames) { + this.attributeNames = attributeNames; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof IndexExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + String indexName = ((IndexExpression)exp).getFieldName(); + if (attributeNames.contains(indexName)) { + return null; + } + return new AttributeExpression(indexName); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/schema/processing/LiteralBoost.java new file mode 100644 index 00000000000..a84f895100a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/LiteralBoost.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.RankProfile; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; + +/** + * Expresses literal boosts in terms of extra indices with rank boost. + * One extra index named indexname_exact is added for each index having + * a fields with literal-boosts of zero or more (zero to support other + * rank profiles setting a literal boost). Complete boost values in to fields + * are translated to rank boosts to the implementation indices. + * These indices has no positional + * or phrase support and contains concatenated versions of each field value + * of complete-boosted fields indexed to indexname. A search for indexname + * will be rewritten to also search indexname_exaxt + * + * @author bratseth + */ +public class LiteralBoost extends Processor { + + public LiteralBoost(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + /** Adds extra search fields and indices to express literal boosts */ + @Override + public void process(boolean validate, boolean documentsOnly) { + checkRankModifierRankType(schema); + addLiteralBoostsToFields(schema); + reduceFieldLiteralBoosts(schema); + } + + /** Checks if literal boost is given using rank: , and set the actual literal boost accordingly. */ + private void checkRankModifierRankType(Schema schema) { + for (SDField field : schema.allConcreteFields()) { + if (field.getLiteralBoost() > -1) continue; // Let explicit value take precedence + if (field.getRanking().isLiteral()) + field.setLiteralBoost(100); + } + } + + /** + * Ensures there are field boosts for all literal boosts mentioned in rank profiles. + * This is required because boost indices will only be generated by looking + * at field boosts + */ + private void addLiteralBoostsToFields(Schema schema) { + Iterator i = matchingRankSettingsIterator(schema, RankProfile.RankSetting.Type.LITERALBOOST); + while (i.hasNext()) { + RankProfile.RankSetting setting = (RankProfile.RankSetting)i.next(); + SDField field = schema.getConcreteField(setting.getFieldName()); + if (field == null) continue; + if (field.getLiteralBoost() < 0) + field.setLiteralBoost(0); + } + } + + private void reduceFieldLiteralBoosts(Schema schema) { + for (SDField field : schema.allConcreteFields()) { + if (field.getLiteralBoost() < 0) continue; + reduceFieldLiteralBoost(field, schema); + } + } + + private void reduceFieldLiteralBoost(SDField field, Schema schema) { + SDField literalField = addField(schema, field, "literal", + "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }", + "literal-boost"); + literalField.setWeight(field.getWeight() + field.getLiteralBoost()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeAliases.java new file mode 100644 index 00000000000..7093242d0ac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeAliases.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Index; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Takes the aliases set on field by parser and sets them on correct Index or Attribute + * + * @author vegardh + */ +public class MakeAliases extends Processor { + + public MakeAliases(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + List usedAliases = new ArrayList<>(); + for (SDField field : schema.allConcreteFields()) { + for (Map.Entry e : field.getAliasToName().entrySet()) { + String alias = e.getKey(); + String name = e.getValue(); + String errMsg = "For " + schema + ": alias '" + alias + "' "; + if (validate && schema.existsIndex(alias)) { + throw new IllegalArgumentException(errMsg + "is illegal since it is the name of an index."); + } + if (validate && schema.getAttribute(alias) != null) { + throw new IllegalArgumentException(errMsg + "is illegal since it is the name of an attribute."); + } + if (validate && usedAliases.contains(alias)) { + throw new IllegalArgumentException(errMsg + "specified more than once."); + } + usedAliases.add(alias); + + Index index = field.getIndex(name); + Attribute attribute = field.getAttributes().get(name); + if (index != null) { + index.addAlias(alias); // alias will be for index in this case, since it is the one used in a search + } else if (attribute != null && ! field.doesIndexing()) { + attribute.getAliases().add(alias); + } else { + index = new Index(name); + index.addAlias(alias); + field.addIndex(index); + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java new file mode 100644 index 00000000000..ea24bf0569d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * All summary fields which are not attributes + * must currently be present in the default summary class, + * since the default summary class also defines the docsum.dat format. + * This processor adds any missing summaries to the default summary. + * When that is decoupled from the actual summaries returned, this + * processor can be removed. Note: the StreamingSummary also takes advantage of + * the fact that default is the superset. + * + * All other summary logic should work unchanged without this processing step + * except that IndexStructureValidator.validateSummaryFields must be changed to + * consider all summaries, not just the default, i.e change to + * if (search.getSummaryField(expr.getFieldName()) == null) + * + * This must be done after other summary processors. + * + * @author bratseth + */ +public class MakeDefaultSummaryTheSuperSet extends Processor { + + public MakeDefaultSummaryTheSuperSet(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + DocumentSummary defaultSummary= schema.getSummariesInThis().get("default"); + for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values() ) { + if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; + if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; + if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue; + if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue; + + defaultSummary.add(summaryField.clone()); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchConsistency.java new file mode 100644 index 00000000000..5fb59e53ba9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchConsistency.java @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashMap; +import java.util.Map; + +/** + * Warn on inconsistent match settings for any index + * + * @author vegardh + */ +public class MatchConsistency extends Processor { + + public MatchConsistency(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + Map types = new HashMap<>(); + for (SDField field : schema.allConcreteFields()) { + new MyVisitor(schema, field, types).visit(field.getIndexingScript()); + } + } + + private void checkMatching(Schema schema, SDField field, Map types, String indexTo) { + MatchType prevType = types.get(indexTo); + if (prevType == null) { + types.put(indexTo, field.getMatching().getType()); + } else if ( ! field.getMatching().getType().equals(prevType)) { + warn(schema, field, "The matching type for index '" + indexTo + "' (got " + field.getMatching().getType() + + ") is inconsistent with that given for the same index in a previous field (had " + + prevType + ")."); + } + } + + private class MyVisitor extends ExpressionVisitor { + + final Schema schema; + final SDField field; + final Map types; + + MyVisitor(Schema schema, SDField field, Map types) { + this.schema = schema; + this.field = field; + this.types = types; + } + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof IndexExpression) { + checkMatching(schema, field, types, ((IndexExpression)exp).getFieldName()); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java new file mode 100644 index 00000000000..7c1c255097f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Validates the match phase settings for all registered rank profiles. + * + * @author geirst + */ +public class MatchPhaseSettingsValidator extends Processor { + + public MatchPhaseSettingsValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + if (documentsOnly) return; + + for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { + RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); + if (settings != null) { + validateMatchPhaseSettings(rankProfile, settings); + } + } + } + + private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { + String attributeName = settings.getAttribute(); + new AttributeValidator(schema.getName(), + rankProfile.name(), + schema.getAttribute(attributeName), attributeName).validate(); + } + + public static class AttributeValidator { + + private final String searchName; + private final String rankProfileName; + protected final Attribute attribute; + private final String attributeName; + + public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { + this.searchName = searchName; + this.rankProfileName = rankProfileName; + this.attribute = attribute; + this.attributeName = attributeName; + } + + public void validate() { + validateThatAttributeExists(); + validateThatAttributeIsSingleNumeric(); + validateThatAttributeIsFastSearch(); + } + + protected void validateThatAttributeExists() { + if (attribute == null) { + failValidation("does not exists"); + } + } + + protected void validateThatAttributeIsSingleNumeric() { + if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || + attribute.getType().equals(Attribute.Type.STRING) || + attribute.getType().equals(Attribute.Type.PREDICATE)) + { + failValidation("must be single value numeric, but it is '" + + attribute.getDataType().getName() + "'"); + } + } + + protected void validateThatAttributeIsFastSearch() { + if ( ! attribute.isFastSearch()) { + failValidation("must be fast-search, but it is not"); + } + } + + protected void failValidation(String what) { + throw new IllegalArgumentException(createMessagePrefix() + what); + } + + public String getValidationType() { return "match-phase"; } + + private String createMessagePrefix() { + return "In search definition '" + searchName + + "', rank-profile '" + rankProfileName + + "': " + getValidationType() + " attribute '" + attributeName + "' "; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchedElementsOnlyResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchedElementsOnlyResolver.java new file mode 100644 index 00000000000..ed95f87d7d6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchedElementsOnlyResolver.java @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ComplexAttributeFieldUtils; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedComplexField; + +/** + * Iterates all summary fields with 'matched-elements-only' and adjusts transform (if all struct-fields are attributes) + * and validates that the field type is supported. + * + * @author geirst + */ +public class MatchedElementsOnlyResolver extends Processor { + + public MatchedElementsOnlyResolver(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var entry : schema.getSummaries().entrySet()) { + var summary = entry.getValue(); + for (var field : summary.getSummaryFields().values()) { + if (field.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER) { + processSummaryField(summary, field, validate); + } + } + } + } + + private void processSummaryField(DocumentSummary summary, SummaryField field, boolean validate) { + var sourceField = schema.getField(field.getSingleSource()); + if (sourceField != null) { + if (isSupportedComplexField(sourceField)) { + if (isComplexFieldWithOnlyStructFieldAttributes(sourceField)) { + field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + } else if (isSupportedMultiValueField(sourceField)) { + if (sourceField.doesAttributing()) { + field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + } else if (validate) { + fail(summary, field, "'matched-elements-only' is not supported for this field type. " + + "Supported field types are: array of primitive, weighted set of primitive, " + + "array of simple struct, map of primitive type to simple struct, " + + "and map of primitive type to primitive type"); + } + } + // else case is handled in SummaryFieldsMustHaveValidSource + } + + private boolean isSupportedMultiValueField(ImmutableSDField sourceField) { + var type = sourceField.getDataType(); + return (isArrayOfPrimitiveType(type) || isWeightedsetOfPrimitiveType(type)); + } + + private boolean isArrayOfPrimitiveType(DataType type) { + if (type instanceof ArrayDataType) { + var arrayType = (ArrayDataType) type; + return ComplexAttributeFieldUtils.isPrimitiveType(arrayType.getNestedType()); + } + return false; + } + + private boolean isWeightedsetOfPrimitiveType(DataType type) { + if (type instanceof WeightedSetDataType) { + var wsetType = (WeightedSetDataType) type; + return ComplexAttributeFieldUtils.isPrimitiveType(wsetType.getNestedType()); + } + return false; + } + + private void fail(DocumentSummary summary, SummaryField field, String msg) { + throw new IllegalArgumentException(formatError(schema, summary, field, msg)); + } + + private String formatError(Schema schema, DocumentSummary summary, SummaryField field, String msg) { + return "For " + schema + ", document summary '" + summary.getName() + + "', summary field '" + field.getName() + "': " + msg; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/schema/processing/MultifieldIndexHarmonizer.java new file mode 100644 index 00000000000..3a889085871 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MultifieldIndexHarmonizer.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.multifieldresolver.IndexCommandResolver; +import com.yahoo.schema.processing.multifieldresolver.RankTypeResolver; +import com.yahoo.schema.processing.multifieldresolver.StemmingResolver; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.List; +import java.util.Map; + +/** + * Ensures that there are no conflicting types or field settings + * in multifield indices, either by changing settings or by splitting + * conflicting fields in multiple ones with different settings. + * + * @author bratseth + */ +public class MultifieldIndexHarmonizer extends Processor { + + /** A map from index names to a List of fields going to that index */ + private Map> indexToFields=new java.util.HashMap<>(); + + public MultifieldIndexHarmonizer(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + populateIndexToFields(schema); + resolveAllConflicts(schema); + } + + private void populateIndexToFields(Schema schema) { + for (SDField field : schema.allConcreteFields() ) { + if ( ! field.doesIndexing()) continue; + addIndexField(field.getName(), field); + } + } + + private void addIndexField(String indexName,SDField field) { + List fields = indexToFields.get(indexName); + if (fields == null) { + fields = new java.util.ArrayList<>(); + indexToFields.put(indexName, fields); + } + fields.add(field); + } + + private void resolveAllConflicts(Schema schema) { + for (Map.Entry> entry : indexToFields.entrySet()) { + String indexName = entry.getKey(); + List fields = entry.getValue(); + if (fields.size() == 1) continue; // It takes two to make a conflict + resolveConflicts(indexName, fields, schema); + } + } + + /** + * Resolves all conflicts for one index + * + * @param indexName the name of the index in question + * @param fields all the fields indexed to this index + * @param schema the search definition having this + */ + private void resolveConflicts(String indexName, List fields, Schema schema) { + new StemmingResolver(indexName, fields, schema, deployLogger).resolve(); + new IndexCommandResolver(indexName, fields, schema, deployLogger).resolve(); + new RankTypeResolver(indexName, fields, schema, deployLogger).resolve(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/schema/processing/MutableAttributes.java new file mode 100644 index 00000000000..854f6b2dddb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/MutableAttributes.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +public class MutableAttributes extends Processor { + + public MutableAttributes(Schema schema, DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) + { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) { + if (field.getAttributes().get(field.getName()).isMutable()) { + throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + schema.getDocument().getName() + + "' can not be marked mutable as it is inside the document clause."); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java new file mode 100644 index 00000000000..f1ff910be43 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of "gram" matching - splitting the incoming text and the queries into + * n-grams for matching. This will also validate the gram settings. + * + * @author bratseth + */ +public class NGramMatch extends Processor { + + public static final int DEFAULT_GRAM_SIZE = 2; + + public NGramMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.getMatching().getType().equals(MatchType.GRAM)) + implementGramMatch(schema, field, validate); + else if (validate && field.getMatching().getGramSize() >= 0) + throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); + } + } + + private void implementGramMatch(Schema schema, SDField field, boolean validate) { + if (validate && field.doesAttributing() && ! field.doesIndexing()) + throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' in indexing"); + + int n = field.getMatching().getGramSize(); + if (n < 0) + n = DEFAULT_GRAM_SIZE; // not set - use default gram size + if (validate && n == 0) + throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); + field.getNormalizing().inferCodepoint(); + field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing + field.addQueryCommand("ngram " + n); + field.setIndexingScript((ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); + } + + private static class MyProvider extends TypedTransformProvider { + + final int ngram; + + MyProvider(Schema schema, int ngram) { + super(NGramExpression.class, schema); + this.ngram = ngram; + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression newTransform(DataType fieldType) { + Expression exp = new NGramExpression(null, ngram); + if (fieldType instanceof CollectionDataType) + exp = new ForEachExpression(exp); + return exp; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelConfigGenerator.java b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelConfigGenerator.java new file mode 100644 index 00000000000..ce56a4320d3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelConfigGenerator.java @@ -0,0 +1,96 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.expressiontransforms.OnnxModelTransformer; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.ml.OnnxModelInfo; + +import java.util.Map; + +/** + * Processes ONNX ranking features of the form: + * + * onnx("files/model.onnx", "path/to/output:1") + * + * And generates an "onnx-model" configuration as if it was defined in the profile: + * + * onnx-model files_model_onnx { + * file: "files/model.onnx" + * } + * + * Inputs and outputs are resolved in OnnxModelTypeResolver, which must be + * processed after this. + * + * @author lesters + */ +public class OnnxModelConfigGenerator extends Processor { + + public OnnxModelConfigGenerator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + if (profile.getFirstPhaseRanking() != null) { + process(profile.getFirstPhaseRanking().getRoot(), profile); + } + if (profile.getSecondPhaseRanking() != null) { + process(profile.getSecondPhaseRanking().getRoot(), profile); + } + for (Map.Entry function : profile.getFunctions().entrySet()) { + process(function.getValue().function().getBody().getRoot(), profile); + } + for (ReferenceNode feature : profile.getSummaryFeatures()) { + process(feature, profile); + } + } + } + + private void process(ExpressionNode node, RankProfile profile) { + if (node instanceof ReferenceNode) { + process((ReferenceNode)node, profile); + } else if (node instanceof CompositeNode) { + for (ExpressionNode child : ((CompositeNode) node).children()) { + process(child, profile); + } + } + } + + private void process(ReferenceNode feature, RankProfile profile) { + if (feature.getName().equals("onnxModel") || feature.getName().equals("onnx")) { + if (feature.getArguments().size() > 0) { + if (feature.getArguments().expressions().get(0) instanceof ConstantNode) { + ConstantNode node = (ConstantNode) feature.getArguments().expressions().get(0); + String path = OnnxModelTransformer.stripQuotes(node.toString()); + String modelConfigName = OnnxModelTransformer.asValidIdentifier(path); + + // Only add the configuration if the model can actually be found. + if ( ! OnnxModelInfo.modelExists(path, schema.applicationPackage())) { + path = ApplicationPackage.MODELS_DIR.append(path).toString(); + if ( ! OnnxModelInfo.modelExists(path, schema.applicationPackage())) { + return; + } + } + + OnnxModel onnxModel = profile.onnxModels().get(modelConfigName); + if (onnxModel == null) + profile.add(new OnnxModel(modelConfigName, path)); + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelTypeResolver.java new file mode 100644 index 00000000000..32229ea635b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelTypeResolver.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.ml.OnnxModelInfo; + +/** + * Processes every "onnx-model" element in the schema. Associates model type + * information by retrieving from either the ONNX model file directly or from + * preprocessed information in ZK. Adds missing input and output mappings + * (assigning default names). + * + * Must be processed before RankingExpressingTypeResolver. + * + * @author lesters + */ +public class OnnxModelTypeResolver extends Processor { + + public OnnxModelTypeResolver(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + for (OnnxModel onnxModel : schema.declaredOnnxModels().values()) + onnxModel.setModelInfo(OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage())); + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + for (OnnxModel onnxModel : profile.declaredOnnxModels().values()) + onnxModel.setModelInfo(OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage())); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java new file mode 100644 index 00000000000..a3b026fb724 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionOptimizer; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Run ExpressionOptimizer on all scripts, to get rid of expressions that have no effect. + */ +public class OptimizeIlscript extends Processor { + + public OptimizeIlscript(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + ScriptExpression script = field.getIndexingScript(); + if (script == null) continue; + + field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); + if ( ! field.getIndexingScript().toString().equals(script.toString())) { + info(schema, field, "Rewrote ilscript from:\n" + script.toString() + + "\nto\n" + field.getIndexingScript().toString()); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java new file mode 100644 index 00000000000..34bb6e1db2e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java @@ -0,0 +1,66 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.Field; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Optional; + +/** + * Validates the 'paged' attribute setting and throws if specified on unsupported types. + * + * @author geirst + */ +public class PagedAttributeValidator extends Processor { + + public PagedAttributeValidator(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (!validate) { + return; + } + for (var field : schema.allConcreteFields()) { + for (var attribute : field.getAttributes().values()) { + if (attribute.isPaged()) { + validatePagedSetting(field, attribute); + } + } + } + } + + private void validatePagedSetting(Field field, Attribute attribute) { + if (!isSupportedType(attribute)) { + fail(schema, field, "The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types"); + } + } + + private boolean isSupportedType(Attribute attribute) { + var type = attribute.getType(); + return (type != Attribute.Type.PREDICATE) && + (type != Attribute.Type.REFERENCE) && + (isSupportedTensorType(attribute.tensorType())); + } + + private boolean isSupportedTensorType(Optional tensorType) { + if (tensorType.isPresent()) { + return isDenseTensorType(tensorType.get()); + } + return true; + } + + private boolean isDenseTensorType(TensorType type) { + return type.dimensions().stream().allMatch(d -> d.isIndexed()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java new file mode 100644 index 00000000000..280eae3d88b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -0,0 +1,144 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.ArrayList; +import java.util.List; + +/** + * Validates the predicate fields. + * + * @author Lester Solbakken + */ +public class PredicateProcessor extends Processor { + + public PredicateProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.getDataType() == DataType.PREDICATE) { + if (validate && field.doesIndexing()) { + fail(schema, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); + } + if (field.doesAttributing()) { + Attribute attribute = field.getAttributes().get(field.getName()); + for (Index index : field.getIndices().values()) { + BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton(); + if (validate && (booleanDefinition == null || ! booleanDefinition.hasArity())) { + fail(schema, field, "Missing arity value in predicate field."); + } + if (validate && (booleanDefinition.getArity() < 2)) { + fail(schema, field, "Invalid arity value in predicate field, must be greater than 1."); + } + double threshold = booleanDefinition.getDensePostingListThreshold(); + if (validate && (threshold <= 0 || threshold > 1)) { + fail(schema, field, "Invalid dense-posting-list-threshold value in predicate field. " + + "Value must be in range (0..1]."); + } + + attribute.setArity(booleanDefinition.getArity()); + attribute.setLowerBound(booleanDefinition.getLowerBound()); + attribute.setUpperBound(booleanDefinition.getUpperBound()); + + attribute.setDensePostingListThreshold(threshold); + addPredicateOptimizationIlScript(field, booleanDefinition); + } + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); + if (summary != null) { + summary.remove(attribute.getName()); + } + for (SummaryField summaryField : schema.getSummaryFields(field)) { + summaryField.setTransform(SummaryTransform.NONE); + } + } + } else if (validate && field.getDataType().getPrimitiveType() == DataType.PREDICATE) { + fail(schema, field, "Collections of predicates are not allowed."); + } else if (validate && field.getDataType() == DataType.RAW && field.doesIndexing()) { + fail(schema, field, "Indexing of RAW fields is not supported."); + } else if (validate) { + // if field is not a predicate, disallow predicate-related index parameters + for (Index index : field.getIndices().values()) { + if (index.getBooleanIndexDefiniton() != null) { + BooleanIndexDefinition def = index.getBooleanIndexDefiniton(); + if (def.hasArity()) { + fail(schema, field, "Arity parameter is used only for predicate type fields."); + } else if (def.hasLowerBound() || def.hasUpperBound()) { + fail(schema, field, "Parameters lower-bound and upper-bound are used only for predicate type fields."); + } else if (def.hasDensePostingListThreshold()) { + fail(schema, field, "Parameter dense-posting-list-threshold is used only for predicate type fields."); + } + } + } + } + } + } + + private void addPredicateOptimizationIlScript(SDField field, BooleanIndexDefinition booleanIndexDefiniton) { + Expression script = field.getIndexingScript(); + if (script == null) return; + + script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); + + ExpressionConverter converter = new PredicateOutputTransformer(schema); + field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script))); + } + + private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { + List expressions = new ArrayList<>(); + expressions.add(new SetValueExpression(new IntegerFieldValue(options.getArity()))); + expressions.add(new SetVarExpression("arity")); + if (options.hasLowerBound()) { + expressions.add(new SetValueExpression(new LongFieldValue(options.getLowerBound()))); + expressions.add(new SetVarExpression("lower_bound")); + } + if (options.hasUpperBound()) { + expressions.add(new SetValueExpression(new LongFieldValue(options.getUpperBound()))); + expressions.add(new SetVarExpression("upper_bound")); + } + return new StatementExpression(expressions); + } + + private static class PredicateOutputTransformer extends TypedTransformProvider { + + PredicateOutputTransformer(Schema schema) { + super(OptimizePredicateExpression.class, schema); + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression && fieldType == DataType.PREDICATE; + } + + @Override + protected Expression newTransform(DataType fieldType) { + return new OptimizePredicateExpression(); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java new file mode 100644 index 00000000000..63eca2121c1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -0,0 +1,152 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.processing.multifieldresolver.RankProfileTypeSettingsProcessor; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.TestProperties; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; + +/** + * Executor of processors. This defines the right order of processor execution. + * + * @author bratseth + * @author bjorncs + */ +public class Processing { + + private final ModelContext.Properties properties; + + public Processing() { this.properties = new TestProperties(); } + + public Processing(ModelContext.Properties properties) { this.properties = properties; } + + private Collection processors() { + return Arrays.asList( + SearchMustHaveDocument::new, + UrlFieldValidator::new, + BuiltInFieldSets::new, + ReservedDocumentNames::new, + IndexFieldNames::new, + IntegerIndex2Attribute::new, + MakeAliases::new, + UriHack::new, + LiteralBoost::new, + TagType::new, + ValidateFieldTypesDocumentsOnly::new, + IndexingInputs::new, + OptimizeIlscript::new, + ValidateFieldWithIndexSettingsCreatesIndex::new, + AttributesImplicitWord::new, + MutableAttributes::new, + CreatePositionZCurve::new, + DictionaryProcessor::new, + WordMatch::new, + ImportedFieldsResolver::new, + ImplicitSummaries::new, + ImplicitSummaryFields::new, + AdjustPositionSummaryFields::new, + SummaryConsistency::new, + SummaryNamesFieldCollisions::new, + SummaryFieldsMustHaveValidSource::new, + MatchedElementsOnlyResolver::new, + AddAttributeTransformToSummaryOfImportedFields::new, + MakeDefaultSummaryTheSuperSet::new, + Bolding::new, + AttributeProperties::new, + SetRankTypeEmptyOnFilters::new, + SummaryDynamicStructsArrays::new, + StringSettingsOnNonStringFields::new, + IndexingOutputs::new, + ExactMatch::new, + NGramMatch::new, + TextMatch::new, + MultifieldIndexHarmonizer::new, + FilterFieldNames::new, + MatchConsistency::new, + ValidateStructTypeInheritance::new, + ValidateFieldTypes::new, + SummaryDiskAccessValidator::new, + DisallowComplexMapAndWsetKeyTypes::new, + SortingSettings::new, + FieldSetSettings::new, + AddExtraFieldsToDocument::new, + PredicateProcessor::new, + MatchPhaseSettingsValidator::new, + DiversitySettingsValidator::new, + TensorFieldProcessor::new, + RankProfileTypeSettingsProcessor::new, + ReferenceFieldsProcessor::new, + FastAccessValidator::new, + ReservedFunctionNames::new, + OnnxModelConfigGenerator::new, + OnnxModelTypeResolver::new, + RankingExpressionTypeResolver::new, + BoolAttributeValidator::new, + PagedAttributeValidator::new, + // These should be last: + IndexingValidation::new, + IndexingValues::new); + } + + /** Processors of rank profiles only (those who tolerate and do something useful when the search field is null) */ + private Collection rankProfileProcessors() { + return Arrays.asList( + RankProfileTypeSettingsProcessor::new, + ReservedFunctionNames::new, + RankingExpressionTypeResolver::new); + } + + private void runProcessor(Processor processor, boolean validate, boolean documentsOnly) { + processor.process(validate, documentsOnly, properties); + } + + /** + * Runs all search processors on the given {@link Schema} object. These will modify the search object, possibly + * exchanging it with another, as well as its document types. + * + * @param schema the search to process + * @param deployLogger the log to log messages and warnings for application deployment to + * @param rankProfileRegistry a {@link com.yahoo.schema.RankProfileRegistry} + * @param queryProfiles the query profiles contained in the application this search is part of + * @param processorsToSkip a set of processor classes we should not invoke in this. Useful for testing. + */ + public void process(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, boolean validate, boolean documentsOnly, + Set> processorsToSkip) + { + Collection factories = processors(); + factories.stream() + .map(factory -> factory.create(schema, deployLogger, rankProfileRegistry, queryProfiles)) + .filter(processor -> ! processorsToSkip.contains(processor.getClass())) + .forEach(processor -> runProcessor(processor, validate, documentsOnly)); + } + + /** + * Runs rank profiles processors only. + * + * @param deployLogger the log to log messages and warnings for application deployment to + * @param rankProfileRegistry a {@link com.yahoo.schema.RankProfileRegistry} + * @param queryProfiles the query profiles contained in the application this search is part of + */ + public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { + Collection factories = rankProfileProcessors(); + factories.stream() + .map(factory -> factory.create(null, deployLogger, rankProfileRegistry, queryProfiles)) + .forEach(processor -> runProcessor(processor, validate, documentsOnly)); + } + + @FunctionalInterface + public interface ProcessorFactory { + Processor create(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java new file mode 100644 index 00000000000..9768f33c27d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -0,0 +1,157 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Index; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; + +/** + * Abstract superclass of all search definition processors. + * + * @author bratseth + */ +public abstract class Processor { + + protected final Schema schema; + protected final DeployLogger deployLogger; + protected final RankProfileRegistry rankProfileRegistry; + protected final QueryProfiles queryProfiles; + + /** + * Base constructor + * + * @param schema the search to process + * @param deployLogger Logger du use when logging deploy output. + * @param rankProfileRegistry Registry with all rank profiles, used for lookup and insertion. + * @param queryProfiles The query profiles contained in the application this search is part of. + */ + public Processor(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + this.schema = schema; + this.deployLogger = deployLogger; + this.rankProfileRegistry = rankProfileRegistry; + this.queryProfiles = queryProfiles; + } + + /** + * Processes the input search definition by modifying the input search and its documents, and returns the + * input search definition. + * + * @param validate true to throw exceptions on validation errors, false to make the best possible effort + * at completing processing without throwing an exception. + * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. + * @param documentsOnly true to skip processing (including validation, regardless of the validate setting) + * of aspects not relating to document definitions (e.g rank profiles) + */ + public abstract void process(boolean validate, boolean documentsOnly); + + /** + * As above, possibly with properties from a context. Override if needed. + **/ + public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { + process(validate, documentsOnly); + } + + /** + * Convenience method for adding a no-strings-attached implementation field for a regular field + * + * @param schema the search definition in question + * @param field the field to add an implementation field for + * @param suffix the suffix of the added implementation field (without the underscore) + * @param indexing the indexing statement of the field + * @param queryCommand the query command of the original field, or null if none + * @return the implementation field which is added to the search + */ + protected SDField addField(Schema schema, SDField field, String suffix, String indexing, String queryCommand) { + SDField implementationField = schema.getConcreteField(field.getName() + "_" + suffix); + if (implementationField != null) { + deployLogger.logApplicationPackage(Level.WARNING, "Implementation field " + implementationField + " added twice"); + } else { + implementationField = new SDField(schema.getDocument(), field.getName() + "_" + suffix, DataType.STRING); + } + implementationField.setRankType(RankType.EMPTY); + implementationField.setStemming(Stemming.NONE); + implementationField.getNormalizing().inferCodepoint(); + implementationField.parseIndexingScript(indexing); + String indexName = field.getName(); + String implementationIndexName = indexName + "_" + suffix; + Index implementationIndex = new Index(implementationIndexName); + schema.addIndex(implementationIndex); + if (queryCommand != null) { + field.addQueryCommand(queryCommand); + } + schema.addExtraField(implementationField); + schema.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, implementationField.getName()); + return implementationField; + } + + /** + * Returns an iterator of all the rank settings with given type in all the rank profiles in this search + * definition. + */ + protected Iterator matchingRankSettingsIterator( + Schema schema, RankProfile.RankSetting.Type type) + { + List someRankSettings = new java.util.ArrayList<>(); + + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) { + RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next(); + if (setting.getType().equals(type)) { + someRankSettings.add(setting); + } + } + } + return someRankSettings.iterator(); + } + + protected String formatError(String schemaName, String fieldName, String msg) { + return "For schema '" + schemaName + "', field '" + fieldName + "': " + msg; + } + + protected RuntimeException newProcessException(String schemaName, String fieldName, String msg) { + return new IllegalArgumentException(formatError(schemaName, fieldName, msg)); + } + + protected RuntimeException newProcessException(Schema schema, Field field, String msg) { + return newProcessException(schema.getName(), field.getName(), msg); + } + + public void fail(Schema schema, Field field, String msg) { + throw newProcessException(schema, field, msg); + } + + protected void warn(String schemaName, String fieldName, String message) { + String fullMsg = formatError(schemaName, fieldName, message); + deployLogger.logApplicationPackage(Level.WARNING, fullMsg); + } + + protected void warn(Schema schema, Field field, String message) { + warn(schema.getName(), field.getName(), message); + } + + protected void info(String schemaName, String fieldName, String message) { + String fullMsg = formatError(schemaName, fieldName, message); + deployLogger.logApplicationPackage(Level.INFO, fullMsg); + } + + protected void info(Schema schema, Field field, String message) { + info(schema.getName(), field.getName(), message); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java new file mode 100644 index 00000000000..07f79f16334 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java @@ -0,0 +1,135 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.MapEvaluationTypeContext; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; + +/** + * Resolves and assigns types to all functions in a ranking expression, and + * validates the types of all ranking expressions under a search instance: + * Some operators constrain the types of inputs, and first-and second-phase expressions + * must return scalar values. + * + * In addition, the existence of all referred attribute, query and constant + * features is ensured. + * + * @author bratseth + */ +public class RankingExpressionTypeResolver extends Processor { + + private final QueryProfileRegistry queryProfiles; + + public RankingExpressionTypeResolver(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + this.queryProfiles = queryProfiles.getRegistry(); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + + Set warnedAbout = new HashSet<>(); + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + try { + resolveTypesIn(profile, validate, warnedAbout); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("In " + (schema != null ? schema + ", " : "") + profile, e); + } + } + } + + /** + * Resolves the types of all functions in the given profile + * + * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types + */ + private void resolveTypesIn(RankProfile profile, boolean validate, Set warnedAbout) { + MapEvaluationTypeContext context = profile.typeContext(queryProfiles); + for (Map.Entry function : profile.getFunctions().entrySet()) { + ExpressionFunction expressionFunction = function.getValue().function(); + if (hasUntypedArguments(expressionFunction)) continue; + + // Add any missing inputs for type resolution + for (String argument : expressionFunction.arguments()) { + Reference ref = Reference.fromIdentifier(argument); + if (context.getType(ref).equals(TensorType.empty)) { + context.setType(ref, expressionFunction.argumentTypes().get(argument)); + } + } + context.forgetResolvedTypes(); + + TensorType type = resolveType(expressionFunction.getBody(), "function '" + function.getKey() + "'", context); + function.getValue().setReturnType(type); + } + + if (validate) { + profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); + ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); + ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); + if ( ( context.tensorsAreUsed() || profile.isStrict()) + && ! context.queryFeaturesNotDeclared().isEmpty() + && ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { + if (profile.isStrict()) + throw new IllegalArgumentException(profile + " is strict but is missing a query profile type " + + "declaration of features " + context.queryFeaturesNotDeclared()); + else + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + profile + + " are not declared in query profile " + + "types and will be interpreted as scalars, not tensors: " + + context.queryFeaturesNotDeclared()); + warnedAbout.addAll(context.queryFeaturesNotDeclared()); + } + } + } + + private boolean hasUntypedArguments(ExpressionFunction function) { + return function.arguments().size() > function.argumentTypes().size(); + } + + private TensorType resolveType(RankingExpression expression, String expressionDescription, TypeContext context) { + if (expression == null) return null; + return resolveType(expression.getRoot(), expressionDescription, context); + } + + private TensorType resolveType(ExpressionNode expression, String expressionDescription, TypeContext context) { + TensorType type; + try { + type = expression.type(context); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("The " + expressionDescription + " is invalid", e); + } + if (type == null) // Not expected to happen + throw new IllegalStateException("Could not determine the type produced by " + expressionDescription); + return type; + } + + private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) { + if (expression == null) return; + TensorType type = resolveType(expression, expressionDescription, context); + if ( ! type.equals(TensorType.empty)) + throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " + + "(a tensor with no dimensions), but produces " + type); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java new file mode 100644 index 00000000000..43e39b1e546 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Class that processes reference fields and removes attribute aspect of such fields from summary. + * + * A document summary for a reference field should always be fetched from the document instance in back-end + * as the attribute vector does not store the original document id string. + * + * @author geirst + */ +public class ReferenceFieldsProcessor extends Processor { + + public ReferenceFieldsProcessor(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + clearSummaryAttributeAspectForConcreteFields(); + clearSummaryAttributeAspectForExplicitSummaryFields(); + } + + private void clearSummaryAttributeAspectForExplicitSummaryFields() { + for (DocumentSummary docSum : schema.getSummaries().values()) { + docSum.getSummaryFields().values().stream() + .filter(summaryField -> summaryField.getDataType() instanceof NewDocumentReferenceDataType) + .forEach(summaryField -> summaryField.setTransform(SummaryTransform.NONE)); + } + } + + private void clearSummaryAttributeAspectForConcreteFields() { + for (SDField field : schema.allConcreteFields()) { + if (field.getDataType() instanceof NewDocumentReferenceDataType) { + removeFromAttributePrefetchSummaryClass(field); + clearSummaryTransformOnSummaryFields(field); + } + } + } + + private void removeFromAttributePrefetchSummaryClass(SDField field) { + DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); + if (summary != null) { + summary.remove(field.getName()); + } + } + + private void clearSummaryTransformOnSummaryFields(SDField field) { + schema.getSummaryFields(field).forEach(summaryField -> summaryField.setTransform(SummaryTransform.NONE)); + } + +} + diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/schema/processing/ReservedDocumentNames.java new file mode 100644 index 00000000000..7eaf690d899 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReservedDocumentNames.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author Simon Thoresen Hult + */ +public class ReservedDocumentNames extends Processor { + + private static final Set RESERVED_NAMES = new HashSet<>(); + + static { + for (SDDocumentType dataType : SDDocumentType.VESPA_DOCUMENT.getTypes()) { + RESERVED_NAMES.add(dataType.getName()); + } + } + + public ReservedDocumentNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + String docName = schema.getDocument().getName(); + if (RESERVED_NAMES.contains(docName)) + throw new IllegalArgumentException("For " + schema + ": Document name '" + docName + "' is reserved."); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java new file mode 100644 index 00000000000..1ec4d5b58f2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Set; +import java.util.logging.Level; + +/** + * Issues a warning if some function has a reserved name. This is not necessarily + * an error, as a rank profile function can shadow a built-in function. + * + * @author lesters + */ +public class ReservedFunctionNames extends Processor { + + private static Set reservedNames = getReservedNames(); + + public ReservedFunctionNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + if (documentsOnly) return; + + for (RankProfile rp : rankProfileRegistry.all()) { + for (String functionName : rp.getFunctions().keySet()) { + if (reservedNames.contains(functionName)) { + deployLogger.logApplicationPackage(Level.WARNING, "Function '" + functionName + "' " + + "in rank profile '" + rp.name() + "' " + + "has a reserved name. This might mean that the function shadows " + + "the built-in function with the same name." + ); + } + } + } + } + + private static ImmutableSet getReservedNames() { + ImmutableSet.Builder names = ImmutableSet.builder(); + for (String token : RankingExpressionParserConstants.tokenImage) { + String tokenWithoutQuotes = token.substring(1, token.length()-1); + names.add(tokenWithoutQuotes); + } + return names.build(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/SearchMustHaveDocument.java new file mode 100644 index 00000000000..b90a5fdec98 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SearchMustHaveDocument.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * A search must have a document definition of the same name inside of it, otherwise crashes may occur as late as + * during feeding + * + * @author Vegard Havdal + */ +public class SearchMustHaveDocument extends Processor { + + public SearchMustHaveDocument(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + if (schema.getDocument() == null) + throw new IllegalArgumentException("For " + schema + + ": A search specification must have an equally named document inside of it."); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/schema/processing/SetRankTypeEmptyOnFilters.java new file mode 100644 index 00000000000..f84d6f19145 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SetRankTypeEmptyOnFilters.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * All rank: filter fields should have rank type empty. + * + * @author bratseth + */ +public class SetRankTypeEmptyOnFilters extends Processor { + + public SetRankTypeEmptyOnFilters(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.getRanking().isFilter()) { + field.setRankType(RankType.EMPTY); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/schema/processing/SortingSettings.java new file mode 100644 index 00000000000..e0dfbab9780 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SortingSettings.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Sorting; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Validate conflicting settings for sorting + * + * @author Vegard Havdal + */ +public class SortingSettings extends Processor { + + public SortingSettings(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (SDField field : schema.allConcreteFields()) { + for (Attribute attribute : field.getAttributes().values()) { + Sorting sorting = attribute.getSorting(); + if (sorting.getFunction() != Sorting.Function.UCA) { + if (sorting.getStrength()!=null && sorting.getStrength() != Sorting.Strength.PRIMARY) { + warn(schema, field, "Sort strength only works for sort function 'uca'."); + } + if (sorting.getLocale() != null && ! "".equals(sorting.getLocale())) { + warn(schema, field, "Sort locale only works for sort function 'uca'."); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/schema/processing/StringSettingsOnNonStringFields.java new file mode 100644 index 00000000000..8ca0b595907 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/StringSettingsOnNonStringFields.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +public class StringSettingsOnNonStringFields extends Processor { + + public StringSettingsOnNonStringFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (SDField field : schema.allConcreteFields()) { + if ( ! doCheck(field)) continue; + if (field.getMatching().isTypeUserSet()) { + warn(schema, field, "Matching type " + field.getMatching().getType() + + " is only allowed for string fields."); + } + if (field.getRanking().isLiteral()) { + warn(schema, field, "Rank type literal only applies to string fields"); + } + } + } + + private boolean doCheck(SDField field) { + if (field.getDataType() instanceof NumericDataType) return true; + if (field.getDataType() instanceof CollectionDataType) { + if (((CollectionDataType)field.getDataType()).getNestedType() instanceof NumericDataType) { + return true; + } + } + return false; + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java new file mode 100644 index 00000000000..4fb45c3c68f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -0,0 +1,131 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + +/** + * Ensure that summary field transforms for fields having the same name + * are consistent across summary classes + * + * @author bratseth + */ +public class SummaryConsistency extends Processor { + + public SummaryConsistency(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (DocumentSummary summary : schema.getSummaries().values()) { + if (summary.getName().equals("default")) continue; + + for (SummaryField summaryField : summary.getSummaryFields().values()) { + assertConsistency(summaryField, schema, validate); + makeAttributeTransformIfAppropriate(summaryField, schema); + makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + } + } + } + + private void assertConsistency(SummaryField summaryField, Schema schema, boolean validate) { + // Compare to default: + SummaryField existingDefault = schema.getSummariesInThis().get("default").getSummaryField(summaryField.getName()); + if (existingDefault != null) { + if (validate) + assertConsistentTypes(existingDefault, summaryField); + makeConsistentWithDefaultOrThrow(existingDefault, summaryField); + } + else { + // If no default, compare to whichever definition of the field + SummaryField existing = schema.getExplicitSummaryField(summaryField.getName()); + if (existing == null) return; + if (validate) + assertConsistentTypes(existing, summaryField); + makeConsistentOrThrow(existing, summaryField, schema); + } + } + + /** If the source is an attribute, make this use the attribute transform */ + private void makeAttributeTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() != SummaryTransform.NONE) return; + Attribute attribute = schema.getAttribute(summaryField.getSingleSource()); + if (attribute == null) return; + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + + /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ + private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String source_field_name = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(source_field_name); + if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { + summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + } + } + + private void assertConsistentTypes(SummaryField existing, SummaryField seen) { + if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && + ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) + return; // Disregard create-if-nonexistent and create-if-zero distinction + if ( ! compatibleTypes(seen.getDataType(), existing.getDataType())) + throw new IllegalArgumentException(existing.toLocateString() + " is inconsistent with " + + seen.toLocateString() + ": All declarations of the same summary field must have the same type"); + } + + private boolean compatibleTypes(DataType summaryType, DataType existingType) { + if (summaryType instanceof TensorDataType && existingType instanceof TensorDataType) { + return summaryType.isAssignableFrom(existingType); // TODO: Just do this for all types + } + return summaryType.equals(existingType); + } + + private void makeConsistentOrThrow(SummaryField field1, SummaryField field2, Schema schema) { + if (field2.getTransform() == SummaryTransform.ATTRIBUTE && field1.getTransform() == SummaryTransform.NONE) { + Attribute attribute = schema.getAttribute(field1.getName()); + if (attribute != null) { + field1.setTransform(SummaryTransform.ATTRIBUTE); + } + } + + if (field2.getTransform().equals(SummaryTransform.NONE)) { + field2.setTransform(field1.getTransform()); + } + else { // New field sets an explicit transform - must be the same + assertEqualTransform(field1,field2); + } + } + + private void makeConsistentWithDefaultOrThrow(SummaryField defaultField, SummaryField newField) { + if (newField.getTransform().equals(SummaryTransform.NONE)) { + newField.setTransform(defaultField.getTransform()); + } + else { // New field sets an explicit transform - must be the same + assertEqualTransform(defaultField,newField); + } + } + + private void assertEqualTransform(SummaryField field1, SummaryField field2) { + if ( ! field2.getTransform().equals(field1.getTransform())) { + throw new IllegalArgumentException("Conflicting summary transforms. " + field2 + " is already defined as " + + field1 + ". A field with the same name " + + "can not have different transforms in different summary classes"); + } + } + + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryDiskAccessValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDiskAccessValidator.java new file mode 100644 index 00000000000..40c38a350b0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDiskAccessValidator.java @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Optional; +import java.util.logging.Level; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + +/** + * Emits a warning for summaries which accesses disk. + * + * @author bratseth + */ +public class SummaryDiskAccessValidator extends Processor { + + public SummaryDiskAccessValidator(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + if (documentsOnly) return; + + for (DocumentSummary summary : schema.getSummaries().values()) { + for (SummaryField summaryField : summary.getSummaryFields().values()) { + for (SummaryField.Source source : summaryField.getSources()) { + ImmutableSDField field = schema.getField(source.getName()); + if (field == null) + field = findFieldProducingSummaryField(source.getName(), schema).orElse(null); + if (field == null && ! source.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) + throw new IllegalArgumentException(summaryField + " in " + summary + " references " + + source + ", but this field does not exist"); + if ( ! isInMemory(field, summaryField) && ! summary.isFromDisk()) { + deployLogger.logApplicationPackage(Level.WARNING, summaryField + " in " + summary + " references " + + source + ", which is not an attribute: Using this " + + "summary will cause disk accesses. " + + "Set 'from-disk' on this summary class to silence this warning."); + } + } + } + } + } + + private boolean isInMemory(ImmutableSDField field, SummaryField summaryField) { + if (field == null) return false; // For DOCUMENT_ID_FIELD, which may be implicit, but is then not in memory + if (isComplexFieldWithOnlyStructFieldAttributes(field) && + (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER || + summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER)) { + return true; + } + return field.doesAttributing(); + } + + private Optional findFieldProducingSummaryField(String name, Schema schema) { + return schema.allFields().filter(field -> field.getSummaryFields().get(name) != null).findAny(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java new file mode 100644 index 00000000000..ed1f47611eb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Fail if: + * An SD field explicitly says summary:dynamic , but the field is wset, array or struct. + * If there is an explicitly defined summary class, saying dynamic in one of its summary + * fields is always legal. + * + * @author Vegard Havdal + */ +public class SummaryDynamicStructsArrays extends Processor { + + public SummaryDynamicStructsArrays(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (SDField field : schema.allConcreteFields()) { + DataType type = field.getDataType(); + if (type instanceof ArrayDataType || type instanceof WeightedSetDataType || type instanceof StructDataType) { + for (SummaryField sField : field.getSummaryFields().values()) { + if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) { + throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " + + "for fields of type struct, array or weighted set. Use an " + + "explicit summary class with explicit summary fields sourcing" + + " from the array/struct/weighted set."); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSource.java new file mode 100644 index 00000000000..c8f201e2915 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSource.java @@ -0,0 +1,81 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Verifies that the source fields actually refers to a valid field. + * + * @author baldersheim + */ +public class SummaryFieldsMustHaveValidSource extends Processor { + + SummaryFieldsMustHaveValidSource(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (DocumentSummary summary : schema.getSummaries().values()) { + for (SummaryField summaryField : summary.getSummaryFields().values()) { + if (summaryField.getSources().isEmpty()) { + if ((summaryField.getTransform() != SummaryTransform.RANKFEATURES) && + (summaryField.getTransform() != SummaryTransform.SUMMARYFEATURES)) + { + verifySource(summaryField.getName(), summaryField, summary); + } + } else if (summaryField.getSourceCount() == 1) { + verifySource(summaryField.getSingleSource(), summaryField, summary); + } else { + for (SummaryField.Source source : summaryField.getSources()) { + if ( ! source.getName().equals(summaryField.getName()) ) { + verifySource(source.getName(), summaryField, summary); + } + } + } + } + } + + } + + private boolean isValid(String source, SummaryField summaryField, DocumentSummary summary) { + return isDocumentField(source) || + (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) || + (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) || + (SummaryClass.DOCUMENT_ID_FIELD.equals(source)); + } + + private void verifySource(String source, SummaryField summaryField, DocumentSummary summary) { + if ( ! isValid(source, summaryField, summary) ) { + throw new IllegalArgumentException("For " + schema + ", summary class '" + + summary.getName() + "'," + " summary field '" + summaryField.getName() + + "': there is no valid source '" + source + "'."); + } + } + + private static boolean isNotInThisSummaryClass(DocumentSummary summary, String name) { + return summary.getSummaryField(name) == null; + } + + private static boolean isInThisSummaryClass(DocumentSummary summary, String name) { + return summary.getSummaryField(name) != null; + } + + private boolean isDocumentField(String name) { + return schema.getField(name) != null; + } + + private boolean isSummaryField(String name) { + return schema.getSummaryField(name) != null; + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryNamesFieldCollisions.java new file mode 100644 index 00000000000..da5dfeb407b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryNamesFieldCollisions.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryField.Source; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Verifies that equally named summary fields in different summary classes don't use different fields for source. + * The summarymap config doesn't model this. + * + * @author Vegard Havdal + */ +public class SummaryNamesFieldCollisions extends Processor { + + public SummaryNamesFieldCollisions(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + Map> fieldToClassAndSource = new HashMap<>(); + for (DocumentSummary summary : schema.getSummaries().values()) { + if ("default".equals(summary.getName())) continue; + for (SummaryField summaryField : summary.getSummaryFields().values()) { + if (summaryField.isImplicit()) continue; + Pair prevClassAndSource = fieldToClassAndSource.get(summaryField.getName()); + for (Source source : summaryField.getSources()) { + if (prevClassAndSource!=null) { + String prevClass = prevClassAndSource.getFirst(); + String prevSource = prevClassAndSource.getSecond(); + if ( ! prevClass.equals(summary.getName())) { + if ( ! prevSource.equals(source.getName())) { + throw new IllegalArgumentException("For " + schema + + ", summary class '" + summary.getName() + "'," + + " summary field '" + summaryField.getName() + "':" + + " Can not use source '" + source.getName() + + "' for this summary field, an equally named field in summary class '" + + prevClass + "' uses a different source: '" + prevSource + "'."); + } + } + } else { + fieldToClassAndSource.put(summaryField.getName(), new Pair<>(summary.getName(), source.getName())); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TagType.java b/config-model/src/main/java/com/yahoo/schema/processing/TagType.java new file mode 100644 index 00000000000..f511d572bc6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TagType.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of the tag datatype + * + * @author bratseth + */ +public class TagType extends Processor { + + public TagType(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) + implementTagType(field); + } + } + + private void implementTagType(SDField field) { + field.setDataType(DataType.getWeightedSet(DataType.STRING, true, true)); + // Don't set matching and ranking if this field is not attribute nor index + if (!field.doesIndexing() && !field.doesAttributing()) return; + Matching m = field.getMatching(); + if ( ! m.isTypeUserSet()) + m.setType(MatchType.WORD); + if (field.getRankType() == null || field.getRankType() == RankType.DEFAULT) + field.setRankType((RankType.TAGS)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java new file mode 100644 index 00000000000..e0ce9917179 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java @@ -0,0 +1,118 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Class that processes and validates tensor fields. + * + * @author geirst + */ +public class TensorFieldProcessor extends Processor { + + public TensorFieldProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var field : schema.allConcreteFields()) { + if ( field.getDataType() instanceof TensorDataType ) { + if (validate) { + validateIndexingScripsForTensorField(field); + validateAttributeSettingForTensorField(field); + validateHnswIndexParametersRequiresIndexing(field); + } + processIndexSettingsForTensorField(field, validate); + } + else if (field.getDataType() instanceof CollectionDataType){ + if (validate) { + validateDataTypeForCollectionField(field); + } + } + } + } + + private void validateIndexingScripsForTensorField(SDField field) { + if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { + fail(schema, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that."); + } + } + + private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + if ((type.dimensions().size() == 1) && + type.dimensions().get(0).isIndexed()) { + return true; + } + return false; + } + + private boolean isTensorTypeThatSupportsDirectStore(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with at least one mapped/sparse dimensions can be "direct" + // (currenty triggered by fast-search flag) + for (var dim : type.dimensions()) { + if (dim.isMapped()) { + return true; + } + } + return false; + } + + private String tensorTypeToString(ImmutableSDField field) { + return ((TensorDataType)field.getDataType()).getTensorType().toString(); + } + + private void validateAttributeSettingForTensorField(SDField field) { + if (field.doesAttributing()) { + var attribute = field.getAttributes().get(field.getName()); + if (attribute != null && attribute.isFastSearch()) { + if (! isTensorTypeThatSupportsDirectStore(field)) { + fail(schema, field, "An attribute of type 'tensor' cannot be 'fast-search'."); + } + } + } + } + + private void validateHnswIndexParametersRequiresIndexing(SDField field) { + var index = field.getIndex(field.getName()); + if (index != null && index.getHnswIndexParams().isPresent() && !field.doesIndexing()) { + fail(schema, field, "A tensor that specifies hnsw index parameters must also specify 'index' in 'indexing'"); + } + } + + private void processIndexSettingsForTensorField(SDField field, boolean validate) { + if (!field.doesIndexing()) { + return; + } + if (isTensorTypeThatSupportsHnswIndex(field)) { + if (validate && !field.doesAttributing()) { + fail(schema, field, "A tensor that has an index must also be an attribute."); + } + var index = field.getIndex(field.getName()); + // TODO: Calculate default params based on tensor dimension size + var params = new HnswIndexParams(); + if (index != null) { + params = params.overrideFrom(index.getHnswIndexParams()); + } + field.getAttribute().setHnswIndexParams(params); + } + } + + private void validateDataTypeForCollectionField(SDField field) { + if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) + fail(schema, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java new file mode 100644 index 00000000000..1783a3c7c63 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -0,0 +1,127 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Set; +import java.util.TreeSet; + +/** + * @author Simon Thoresen Hult + */ +public class TextMatch extends Processor { + + public TextMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.getMatching().getType() != MatchType.TEXT) continue; + + ScriptExpression script = field.getIndexingScript(); + if (script == null) continue; + + DataType fieldType = field.getDataType(); + if (fieldType instanceof CollectionDataType) { + fieldType = ((CollectionDataType)fieldType).getNestedType(); + } + if (fieldType != DataType.STRING) continue; + + Set dynamicSummary = new TreeSet<>(); + Set staticSummary = new TreeSet<>(); + new IndexingOutputs(schema, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(schema, + field, + dynamicSummary, + staticSummary); + MyVisitor visitor = new MyVisitor(dynamicSummary); + visitor.visit(script); + if ( ! visitor.requiresTokenize) continue; + + ExpressionConverter converter = new MyStringTokenizer(schema, findAnnotatorConfig(schema, field)); + field.setIndexingScript((ScriptExpression)converter.convert(script)); + } + } + + private AnnotatorConfig findAnnotatorConfig(Schema schema, SDField field) { + AnnotatorConfig ret = new AnnotatorConfig(); + Stemming activeStemming = field.getStemming(); + if (activeStemming == null) { + activeStemming = schema.getStemming(); + } + ret.setStemMode(activeStemming.toStemMode()); + ret.setRemoveAccents(field.getNormalizing().doRemoveAccents()); + if ((field.getMatching() != null) && (field.getMatching().maxLength() != null)) { + ret.setMaxTokenLength(field.getMatching().maxLength()); + } + return ret; + } + + private static class MyVisitor extends ExpressionVisitor { + + final Set dynamicSummaryFields; + boolean requiresTokenize = false; + + MyVisitor(Set dynamicSummaryFields) { + this.dynamicSummaryFields = dynamicSummaryFields; + } + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof IndexExpression) { + requiresTokenize = true; + } + if (exp instanceof SummaryExpression && + dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName())) + { + requiresTokenize = true; + } + } + + } + + private static class MyStringTokenizer extends TypedTransformProvider { + + final AnnotatorConfig annotatorCfg; + + MyStringTokenizer(Schema schema, AnnotatorConfig annotatorCfg) { + super(TokenizeExpression.class, schema); + this.annotatorCfg = annotatorCfg; + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression newTransform(DataType fieldType) { + Expression exp = new TokenizeExpression(null, annotatorCfg); + if (fieldType instanceof CollectionDataType) { + exp = new ForEachExpression(exp); + } + return exp; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java new file mode 100644 index 00000000000..1836cd631ad --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; +import com.yahoo.vespa.indexinglanguage.expressions.*; + +/** + * @author Simon Thoresen Hult + */ +public abstract class TypedTransformProvider extends ValueTransformProvider { + + private final Schema schema; + private DataType fieldType; + + TypedTransformProvider(Class transformClass, Schema schema) { + super(transformClass); + this.schema = schema; + } + + @Override + protected final boolean requiresTransform(Expression exp) { + if (exp instanceof OutputExpression) { + String fieldName = ((OutputExpression)exp).getFieldName(); + if (exp instanceof AttributeExpression) { + Attribute attribute = schema.getAttribute(fieldName); + if (attribute == null) + throw new IllegalArgumentException("Attribute '" + fieldName + "' not found."); + fieldType = attribute.getDataType(); + } + else if (exp instanceof IndexExpression) { + Field field = schema.getConcreteField(fieldName); + if (field == null) + throw new IllegalArgumentException("Index field '" + fieldName + "' not found."); + fieldType = field.getDataType(); + } + else if (exp instanceof SummaryExpression) { + Field field = schema.getSummaryField(fieldName); + if (field == null) + throw new IllegalArgumentException("Summary field '" + fieldName + "' not found."); + fieldType = field.getDataType(); + } + else { + throw new UnsupportedOperationException(); + } + } + return requiresTransform(exp, fieldType); + } + + @Override + protected final Expression newTransform() { + return newTransform(fieldType); + } + + protected abstract boolean requiresTransform(Expression exp, DataType fieldType); + + protected abstract Expression newTransform(DataType fieldType); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java new file mode 100644 index 00000000000..a4773a42ed6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Arrays; +import java.util.List; + +/** + * @author baldersheim + */ +public class UriHack extends Processor { + + private static final List URL_SUFFIX = + Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); + + UriHack(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + if (field.doesIndexing()) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof CollectionDataType) { + fieldType = ((CollectionDataType)fieldType).getNestedType(); + } + if (fieldType == DataType.URI) { + processField(schema, field); + } + } + } + } + + private void processField(Schema schema, SDField uriField) { + String uriName = uriField.getName(); + uriField.setStemming(Stemming.NONE); + DataType generatedType = DataType.STRING; + if (uriField.getDataType() instanceof ArrayDataType) { + generatedType = new ArrayDataType(DataType.STRING); + } + else if (uriField.getDataType() instanceof WeightedSetDataType) { + WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType(); + generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero()); + } + + for (String suffix : URL_SUFFIX) { + String partName = uriName + "." + suffix; + // I wonder if this is explicit in qrs or implicit in backend? + // search.addFieldSetItem(uriName, partName); + SDField partField = new SDField(schema.getDocument(), partName, generatedType); + partField.setIndexStructureField(uriField.doesIndexing()); + partField.setRankType(uriField.getRankType()); + partField.setStemming(Stemming.NONE); + partField.getNormalizing().inferLowercase(); + if (uriField.getIndex(suffix) != null) { + partField.addIndex(uriField.getIndex(suffix)); + } + schema.addExtraField(partField); + schema.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, partField.getName()); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/UrlFieldValidator.java new file mode 100644 index 00000000000..63d4a342c72 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/UrlFieldValidator.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * @author bratseth + */ +public class UrlFieldValidator extends Processor { + + public UrlFieldValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + for (ImmutableSDField field : schema.allConcreteFields()) { + if ( ! field.getDataType().equals(DataType.URI)) continue; + + if (field.doesAttributing()) + throw new IllegalArgumentException("Error in " + field + " in " + schema + ": " + + "uri type fields cannot be attributes"); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java new file mode 100644 index 00000000000..2327cf4d9c9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashMap; +import java.util.Map; + +/** + * This Processor makes sure all fields with the same name have the same {@link DataType}. This check + * explicitly disregards whether a field is an index field, an attribute or a summary field. This is a requirement if we + * hope to move to a model where index fields, attributes and summary fields share a common field class. + * + * @author Simon Thoresen Hult + */ +public class ValidateFieldTypes extends Processor { + + public ValidateFieldTypes(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (!validate) return; + + String searchName = schema.getName(); + Map seenFields = new HashMap<>(); + verifySearchAndDocFields(searchName, seenFields); + verifySummaryFields(searchName, seenFields); + } + + final protected void verifySearchAndDocFields(String searchName, Map seenFields) { + schema.allFields().forEach(field -> { + checkFieldType(searchName, "index field", field.getName(), field.getDataType(), seenFields); + for (Map.Entry entry : field.getAttributes().entrySet()) { + checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), seenFields); + } + }); + + } + final protected void verifySummaryFields(String searchName, Map seenFields) { + for (DocumentSummary summary : schema.getSummaries().values()) { + for (SummaryField field : summary.getSummaryFields().values()) { + checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); + } + } + } + + private void checkFieldType(String searchName, String fieldDesc, String fieldName, DataType fieldType, + Map seenFields) { + DataType seenType = seenFields.get(fieldName); + if (seenType == null) { + seenFields.put(fieldName, fieldType); + } else if ( ! compatibleTypes(seenType, fieldType)) { + throw newProcessException(searchName, fieldName, "Incompatible types. Expected " + + seenType.getName() + " for " + fieldDesc + + " '" + fieldName + "', got " + fieldType.getName() + "."); + } + } + + private static boolean compatibleTypes(DataType seenType, DataType fieldType) { + // legacy tag field type compatibility; probably not needed any more (Oct 2016) + if ("tag".equals(seenType.getName())) { + return "tag".equals(fieldType.getName()) || "WeightedSet".equals(fieldType.getName()); + } + if ("tag".equals(fieldType.getName())) { + return "tag".equals(seenType.getName()) || "WeightedSet".equals(seenType.getName()); + } + if (seenType instanceof TensorDataType && fieldType instanceof TensorDataType) { + return fieldType.isAssignableFrom(seenType); // TODO: Just do this for all types + } + return seenType.equals(fieldType); + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypesDocumentsOnly.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypesDocumentsOnly.java new file mode 100644 index 00000000000..08771b40fe9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypesDocumentsOnly.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashMap; +import java.util.Map; + +public class ValidateFieldTypesDocumentsOnly extends ValidateFieldTypes { + public ValidateFieldTypesDocumentsOnly(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + String searchName = schema.getName(); + Map seenFields = new HashMap<>(); + verifySearchAndDocFields(searchName, seenFields); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldWithIndexSettingsCreatesIndex.java new file mode 100644 index 00000000000..5423defa74a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.Ranking; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Check that fields with index settings actually creates an index or attribute + * + * @author bratseth + */ +public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { + + public ValidateFieldWithIndexSettingsCreatesIndex(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + Matching defaultMatching = new Matching(); + Ranking defaultRanking = new Ranking(); + for (SDField field : schema.allConcreteFields()) { + if (field.doesIndexing()) continue; + if (field.doesAttributing()) continue; + + if ( ! field.getRanking().equals(defaultRanking)) + fail(schema, field, + "Fields which are not creating an index or attribute can not contain rank settings."); + if ( ! field.getMatching().equals(defaultMatching)) + fail(schema, field, + "Fields which are not creating an index or attribute can not contain match settings."); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateStructTypeInheritance.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateStructTypeInheritance.java new file mode 100644 index 00000000000..cad555a24b1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateStructTypeInheritance.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import com.yahoo.document.Field; +import com.yahoo.schema.document.SDDocumentType; + +import java.util.ArrayList; +import java.util.HashSet; + +/** + * @author arnej + */ +public class ValidateStructTypeInheritance extends Processor { + + public ValidateStructTypeInheritance(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (!validate) return; + verifyNoRedeclarations(schema.getDocument()); + } + + void fail(Field field, String message) { + throw newProcessException(schema, field, message); + } + + void verifyNoRedeclarations(SDDocumentType docType) { + for (SDDocumentType type : docType.allTypes().values()) { + if (type.isStruct()) { + var inheritedTypes = new ArrayList(type.getInheritedTypes()); + for (int i = 0; i < inheritedTypes.size(); i++) { + SDDocumentType inherit = inheritedTypes.get(i); + for (var extra : inherit.getInheritedTypes()) { + if (! inheritedTypes.contains(extra)) { + inheritedTypes.add(extra); + } + } + } + if (inheritedTypes.isEmpty()) continue; + var seenFieldNames = new HashSet<>(); + for (var field : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { + if (seenFieldNames.contains(field.getName())) { + // cannot happen? + fail(field, "struct "+type.getName()+" has multiple fields with same name: "+field.getName()); + } + seenFieldNames.add(field.getName()); + } + for (SDDocumentType inherit : inheritedTypes) { + if (inherit.isStruct()) { + for (var field : inherit.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { + if (seenFieldNames.contains(field.getName())) { + fail(field, "struct "+type.getName()+" cannot inherit from "+inherit.getName()+" and redeclare field "+field.getName()); + } + seenFieldNames.add(field.getName()); + } + } else { + fail(new Field("no field"), "struct cannot inherit from non-struct "+inherit.getName()+" class "+inherit.getClass()); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/WordMatch.java new file mode 100644 index 00000000000..1e312b71afd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/WordMatch.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of word matching - with word matching the field is assumed to contain a single "word" - some + * contiguous sequence of word and number characters - but without changing the data at the indexing side (as with text + * matching) to enforce this. Word matching is thus almost like exact matching on the indexing side (no action taken), + * and like text matching on the query side. This may be suitable for attributes, where people both expect the data to + * be left as in the input document, and trivially written queries to work by default. However, this may easily lead to + * data which cannot be matched at all as the indexing and query side does not agree. + * + * @author bratseth + */ +public class WordMatch extends Processor { + + public WordMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : schema.allConcreteFields()) { + processFieldRecursive(field); + } + } + + private void processFieldRecursive(SDField field) { + processField(field); + for (SDField structField : field.getStructFields()) { + processField(structField); + } + } + + private void processField(SDField field) { + if (!field.getMatching().getType().equals(MatchType.WORD)) { + return; + } + field.setStemming(Stemming.NONE); + field.getNormalizing().inferLowercase(); + field.addQueryCommand("word"); + } + + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java new file mode 100644 index 00000000000..565a377f2a9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; + +/** + * Resolver-class for harmonizing index-commands in multifield indexes + */ +public class IndexCommandResolver extends MultiFieldResolver { + + /** Commands which don't have to be harmonized between fields */ + private static List ignoredCommands = new ArrayList<>(); + + /** Commands which must be harmonized between fields */ + private static List harmonizedCommands = new ArrayList<>(); + + static { + String[] ignore = { "complete-boost", "literal-boost", "highlight" }; + ignoredCommands.addAll(Arrays.asList(ignore)); + String[] harmonize = { "stemming", "normalizing" }; + harmonizedCommands.addAll(Arrays.asList(harmonize)); + } + + public IndexCommandResolver(String indexName, List fields, Schema schema, DeployLogger logger) { + super(indexName, fields, schema, logger); + } + + /** + * Check index-commands for each field, report and attempt to fix any + * inconsistencies + */ + public void resolve() { + for (SDField field : fields) { + for (String command : field.getQueryCommands()) { + if (!ignoredCommands.contains(command)) + checkCommand(command); + } + } + } + + private void checkCommand(String command) { + for (SDField field : fields) { + if (!field.hasQueryCommand(command)) { + if (harmonizedCommands.contains(command)) { + deployLogger.logApplicationPackage(Level.WARNING, command + " must be added to all fields going to the same index (" + indexName + ")" + + ", adding to field " + field.getName()); + field.addQueryCommand(command); + } else { + deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() + + "\' doesn't contain command \'" + command+"\'"); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java new file mode 100644 index 00000000000..ed8ad61706b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; +import java.util.List; + +/** + * Abstract superclass of all multifield conflict resolvers + */ +public abstract class MultiFieldResolver { + + protected String indexName; + protected List fields; + protected Schema schema; + + protected DeployLogger deployLogger; + + public MultiFieldResolver(String indexName, List fields, Schema schema, DeployLogger logger) { + this.indexName = indexName; + this.fields = fields; + this.schema = schema; + this.deployLogger = logger; + } + + /** + * Checks the list of fields for specific conflicts, and reports and/or + * attempts to correct them + */ + public abstract void resolve(); + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java new file mode 100644 index 00000000000..3d79ac7d68a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -0,0 +1,105 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.processing.Processor; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Map; +import java.util.Optional; + +/** + * This processes a schema and adds input type settings on all rank profiles. + * + * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. + * + * @author geirst + */ +public class RankProfileTypeSettingsProcessor extends Processor { + + public RankProfileTypeSettingsProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + + processAttributeFields(); + processImportedFields(); + processQueryProfileTypes(); + } + + private void processAttributeFields() { + if (schema == null) return; // we're processing global profiles + for (ImmutableSDField field : schema.allConcreteFields()) { + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null && attribute.tensorType().isPresent()) { + addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString()); + } + } + } + + private void processImportedFields() { + if (schema == null) return; // we're processing global profiles + Optional importedFields = schema.importedFields(); + if (importedFields.isPresent()) { + importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field)); + } + } + + private void processImportedField(ImportedField field) { + ImmutableSDField targetField = field.targetField(); + Attribute attribute = targetField.getAttributes().get(targetField.getName()); + if (attribute != null && attribute.tensorType().isPresent()) { + addAttributeTypeToRankProfiles(field.fieldName(), attribute.tensorType().get().toString()); + } + } + + private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) { + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + profile.addAttributeType(attributeName, attributeType); + } + } + + private void processQueryProfileTypes() { + for (QueryProfileType queryProfileType : queryProfiles.getRegistry().getTypeRegistry().allComponents()) { + for (Map.Entry fieldDescEntry : queryProfileType.fields().entrySet()) { + processFieldDescription(fieldDescEntry.getValue()); + } + } + } + + private void processFieldDescription(FieldDescription fieldDescription) { + FieldType fieldType = fieldDescription.getType(); + if (fieldType instanceof TensorFieldType) { + TensorFieldType tensorFieldType = (TensorFieldType)fieldType; + Optional reference = Reference.simple(fieldDescription.getName()); + if (reference.isPresent() && FeatureNames.isQueryFeature(reference.get())) + addQueryFeatureTypeToRankProfiles(reference.get(), tensorFieldType.asTensorType()); + } + } + + private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) { + for (RankProfile profile : rankProfileRegistry.all()) { + if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence + profile.addInput(queryFeature, + new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty())); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java new file mode 100644 index 00000000000..6424fd8ba06 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; + +import java.util.List; +import java.util.logging.Level; + +/** + * Checks if fields have defined different rank types for the same + * index (typically in an index-to statement), and if they have + * output a warning and use the first ranktype. + * + * @author hmusum + */ +public class RankTypeResolver extends MultiFieldResolver { + + public RankTypeResolver(String indexName, List fields, Schema schema, DeployLogger logger) { + super(indexName, fields, schema, logger); + } + + public void resolve() { + RankType rankType = null; + if (fields.size() > 0) { + boolean first = true; + for (SDField field : fields) { + if (first) { + rankType = fields.get(0).getRankType(); + first = false; + } else if (!field.getRankType().equals(rankType)) { + deployLogger.logApplicationPackage(Level.WARNING, "In field '" + field.getName() + "' " + + field.getRankType() + " for index '" + indexName + + "' conflicts with " + rankType + + " defined for the same index in field '" + + field.getName() + "'. Using " + + rankType + "."); + field.setRankType(rankType); + } + } + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java new file mode 100644 index 00000000000..95d9a50a6ab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; + +import java.util.List; +import java.util.logging.Level; + +/** + * Class resolving conflicts when fields with different stemming-settings are + * combined into the same index + */ +public class StemmingResolver extends MultiFieldResolver { + + public StemmingResolver(String indexName, List fields, Schema schema, DeployLogger logger) { + super(indexName, fields, schema, logger); + } + + @Override + public void resolve() { + checkStemmingForIndexFields(indexName, fields); + } + + private void checkStemmingForIndexFields(String indexName, List fields) { + Stemming stemming = null; + SDField stemmingField = null; + for (SDField field : fields) { + if (stemming == null && stemmingField==null) { + stemming = field.getStemming(schema); + stemmingField = field; + } else if (stemming != field.getStemming(schema)) { + deployLogger.logApplicationPackage(Level.WARNING, "Field '" + field.getName() + "' has " + field.getStemming(schema) + + ", whereas field '" + stemmingField.getName() + "' has " + stemming + + ". All fields indexing to the index '" + indexName + "' must have the same stemming." + + " This should be corrected as it will make indexing fail in a few cases."); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/package-info.java b/config-model/src/main/java/com/yahoo/schema/processing/package-info.java new file mode 100644 index 00000000000..e81d50897ac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/package-info.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Classes in this package (processors) implements some search + * definition features by reducing them to simpler features. + * The processors are run after parsing of the search definition, + * before creating the derived model. + * + * For simplicity, features should always be implemented here + * rather than in the derived model if possible. + * + * New processors must be added to the list in Processing. + */ +@com.yahoo.api.annotations.PackageMarker +package com.yahoo.schema.processing; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java deleted file mode 100644 index 2dda670f07c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.derived.SearchOrderer; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.searchdefinition.processing.Processor; -import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * A collection of objects representing the content of an application package. - * This is created, then added to, and lastly validated when there is no more content to add. - * At that point it is ready to use for deriving configuration. - * - * @author bratseth - */ -public class Application { - - private final ApplicationPackage applicationPackage; - private final Map schemas; - private final DocumentModel documentModel; - - public Application(ApplicationPackage applicationPackage, - List schemas, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, - ModelContext.Properties properties, - boolean documentsOnly, - boolean validate, - Set> processorsToSkip, - DeployLogger logger) { - this.applicationPackage = applicationPackage; - - Map schemaMap = new LinkedHashMap<>(); - for (Schema schema : schemas) { - if (schemaMap.containsKey(schema.getName())) - throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); - schemaMap.put(schema.getName(), schema); - } - this.schemas = Collections.unmodifiableMap(schemaMap); - - schemas.forEach(schema -> schema.setOwner(this)); - if (validate) - schemas.forEach(schema -> schema.validate(logger)); - - new TemporarySDTypeResolver(schemas, logger).process(); - - List sdocs = new ArrayList<>(); - sdocs.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : schemas) { - if (schema.hasDocument()) { - sdocs.add(schema.getDocument()); - } - } - - var orderer = new SDDocumentTypeOrderer(sdocs, logger); - orderer.process(); - for (SDDocumentType sdoc : orderer.getOrdered()) { - new FieldOperationApplierForStructs().process(sdoc); - new FieldOperationApplier().process(sdoc); - } - - var resolver = new DocumentReferenceResolver(schemas); - sdocs.forEach(resolver::resolveReferences); - sdocs.forEach(resolver::resolveInheritedReferences); - var importedFieldsEnumerator = new ImportedFieldsEnumerator(schemas); - sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - - if (validate) - new DocumentGraphValidator().validateDocumentGraph(sdocs); - - List schemasSomewhatOrdered = new ArrayList<>(schemas); - for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? - new Processing(properties).process(schema, - logger, - rankProfileRegistry, - queryProfiles, - validate, - documentsOnly, - processorsToSkip); - } - - this.documentModel = new DocumentModelBuilder().build(schemasSomewhatOrdered); - } - - public ApplicationPackage applicationPackage() { return applicationPackage; } - - /** Returns an unmodifiable list of the schemas of this application */ - public Map schemas() { return schemas; } - - public DocumentModel documentModel() { return documentModel; } - - @Override - public String toString() { return "application " + applicationPackage.getApplicationId(); } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java deleted file mode 100644 index fe0044ecfa4..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java +++ /dev/null @@ -1,513 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.path.Path; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.parser.ConvertSchemaCollection; -import com.yahoo.searchdefinition.parser.IntermediateCollection; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.Processor; -import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Application builder. Usage: - * 1) Add all schemas, using the addXXX() methods, - * 2) provide the available rank types and rank expressions, using the setRankXXX() methods, - * 3) invoke the {@link #build} method - * - * @author bratseth - */ -public class ApplicationBuilder { - - private final IntermediateCollection mediator; - private final ApplicationPackage applicationPackage; - private final List schemas = new ArrayList<>(); - private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); - private final RankProfileRegistry rankProfileRegistry; - private final QueryProfileRegistry queryProfileRegistry; - private final FileRegistry fileRegistry; - private final DeployLogger deployLogger; - private final ModelContext.Properties properties; - /** True to build the document aspect only, skipping instantiation of rank profiles */ - private final boolean documentsOnly; - - private Application application; - - private final Set> processorsToSkip = new HashSet<>(); - - /** For testing only */ - public ApplicationBuilder() { - this(new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** For testing only */ - public ApplicationBuilder(DeployLogger deployLogger) { - this(MockApplicationPackage.createEmpty(), deployLogger); - } - - /** For testing only */ - public ApplicationBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { - this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); - } - - /** Used for generating documents for typed access to document fields in Java */ - public ApplicationBuilder(boolean documentsOnly) { - this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); - } - - /** For testing only */ - public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger) { - this(app, new MockFileRegistry(), deployLogger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** For testing only */ - public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { - this(app, new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, new QueryProfileRegistry()); - } - - /** For testing only */ - public ApplicationBuilder(RankProfileRegistry rankProfileRegistry) { - this(rankProfileRegistry, new QueryProfileRegistry()); - } - - /** For testing only */ - public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { - this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); - } - - /** For testing only */ - public ApplicationBuilder(ModelContext.Properties properties) { - this(new RankProfileRegistry(), new QueryProfileRegistry(), properties); - } - - /** For testing only */ - public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { - this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); - } - - /** normal constructor */ - public ApplicationBuilder(ApplicationPackage app, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - this(app, fileRegistry, deployLogger, properties, rankProfileRegistry, queryProfileRegistry, false); - } - - private ApplicationBuilder(ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry, - boolean documentsOnly) { - this.mediator = new IntermediateCollection(deployLogger, properties); - this.applicationPackage = applicationPackage; - this.rankProfileRegistry = rankProfileRegistry; - this.queryProfileRegistry = queryProfileRegistry; - this.fileRegistry = fileRegistry; - this.deployLogger = deployLogger; - this.properties = properties; - this.documentsOnly = documentsOnly; - for (NamedReader reader : applicationPackage.getSchemas()) - addSchema(reader); - } - - /** - * Adds a schema to this application. - * - * @param fileName the name of the file to import - * @throws IOException thrown if the file can not be read for some reason - * @throws ParseException thrown if the file does not contain a valid search definition - */ - public void addSchemaFile(String fileName) throws IOException, ParseException { - var parsedName = mediator.addSchemaFromFile(fileName); - addRankProfileFiles(parsedName); - } - - /** - * Reads and parses the schema string provided by the given reader. Once all schemas have been - * imported, call {@link #build}. - * - * @param reader the reader whose content to import - */ - public void addSchema(NamedReader reader) { - try { - var parsedName = mediator.addSchemaFromReader(reader); - addRankProfileFiles(parsedName); - } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); - } - } - - /** - * Adds a schema to this - * - * @param schemaString the content of the schema - */ - public void addSchema(String schemaString) throws ParseException { - var parsed = mediator.addSchemaFromString(schemaString); - addRankProfileFiles(parsed.name()); - } - - /** - * Registers the given schema to the application to be built during {@link #build}. A - * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most - * programmatically constructed schemas used in unit tests. - * - * @param schema the object to import - * @throws IllegalArgumentException if the given search object has already been processed - */ - public Schema add(Schema schema) { - if (schema.getName() == null) - throw new IllegalArgumentException("Schema has no name"); - schemas.add(schema); - return schema; - } - - private void addRankProfileFiles(String schemaName) throws ParseException { - if (applicationPackage == null) return; - - Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schemaName); - for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) { - mediator.addRankProfileFile(schemaName, reader); - } - - Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schemaName); - for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile", true)) { - System.out.println("Got " + reader); - mediator.addRankProfileFile(schemaName, reader); - } - } - - /** - * Processes and finalizes the schemas of this. - * - * @throws IllegalStateException thrown if this method has already been called - */ - public Application build(boolean validate) { - if (application != null) throw new IllegalStateException("Application already built"); - var converter = new ConvertSchemaCollection(mediator, - documentTypeManager, - applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - documentsOnly); - for (var schema : converter.convertToSchemas()) - add(schema); - application = new Application(applicationPackage, - schemas, - rankProfileRegistry, - new QueryProfiles(queryProfileRegistry, deployLogger), - properties, - documentsOnly, - validate, - processorsToSkip, - deployLogger); - return application; - } - - /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ - public Set> processorsToSkip() { return processorsToSkip; } - - /** - * Convenience method to call {@link #getSchema(String)} when there is only a single {@link Schema} object - * built. This method will never return null. - * - * @return the built object - * @throws IllegalStateException if there is not exactly one search. - */ - public Schema getSchema() { - if (application == null) throw new IllegalStateException("Application not built"); - if (application.schemas().size() != 1) - throw new IllegalStateException("This call only works if we have 1 schema. Schemas: " + - application.schemas().values()); - - return application.schemas().values().stream().findAny().get(); - } - - public DocumentModel getModel() { return application.documentModel(); } - - /** - * Returns the built {@link Schema} object that has the given name. If the name is unknown, this method will simply - * return null. - * - * @param name the name of the schema to return, - * or null to return the only one or throw an exception if there are multiple to choose from - * @return the built object, or null if none with this name - * @throws IllegalStateException if {@link #build} has not been called. - */ - public Schema getSchema(String name) { - if (application == null) throw new IllegalStateException("Application not built"); - if (name == null) return getSchema(); - return application.schemas().get(name); - } - - public Application application() { return application; } - - /** - * Convenience method to return a list of all built {@link Schema} objects. - * - * @return the list of built searches - */ - public List getSchemaList() { - return new ArrayList<>(application.schemas().values()); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a string. - * - * @param sd the string to build from - * @return the built {@link ApplicationBuilder} object - * @throws ParseException thrown if there is a problem parsing the string - */ - public static ApplicationBuilder createFromString(String sd) throws ParseException { - return createFromString(sd, new BaseDeployLogger()); - } - - public static ApplicationBuilder createFromString(String sd, DeployLogger logger) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(logger); - builder.addSchema(sd); - builder.build(true); - return builder; - } - - public static ApplicationBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(logger); - for (var schema : schemas) - builder.addSchema(schema); - builder.build(true); - return builder; - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. - * - * @param fileName the file to build from - * @return the built {@link ApplicationBuilder} object - * @throws IOException if there was a problem reading the file. - * @throws ParseException if there was a problem parsing the file content. - */ - public static ApplicationBuilder createFromFile(String fileName) throws IOException, ParseException { - return createFromFile(fileName, new BaseDeployLogger()); - } - - /** - * Convenience factory methods to create a SearchBuilder from multiple SD files. Only for testing. - */ - public static ApplicationBuilder createFromFiles(Collection fileNames) throws IOException, ParseException { - return createFromFiles(fileNames, new BaseDeployLogger()); - } - - public static ApplicationBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { - return createFromFile(fileName, logger, new RankProfileRegistry(), new QueryProfileRegistry()); - } - - private static ApplicationBuilder createFromFiles(Collection fileNames, DeployLogger logger) throws IOException, ParseException { - return createFromFiles(fileNames, new MockFileRegistry(), logger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. - * - * @param fileName the file to build from. - * @param deployLogger logger for deploy messages. - * @param rankProfileRegistry registry for rank profiles. - * @return the built {@link ApplicationBuilder} object. - * @throws IOException if there was a problem reading the file. - * @throws ParseException if there was a problem parsing the file content. - */ - private static ApplicationBuilder createFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) - throws IOException, ParseException { - return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), - rankProfileRegistry, queryprofileRegistry); - } - - /** - * Convenience factory methdd to create a SearchBuilder from multiple SD files.. - */ - private static ApplicationBuilder createFromFiles(Collection fileNames, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) - throws IOException, ParseException { - ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryprofileRegistry); - for (String fileName : fileNames) { - builder.addSchemaFile(fileName); - } - builder.build(true); - return builder; - } - - - public static ApplicationBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { - return createFromDirectory(dir, fileRegistry, logger, properties, new RankProfileRegistry()); - } - public static ApplicationBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return createFromDirectory(dir, fileRegistry, logger, properties, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); - } - private static ApplicationBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - return createFromDirectory(dir, MockApplicationPackage.fromSearchDefinitionAndRootDirectory(dir), fileRegistry, logger, properties, - rankProfileRegistry, queryProfileRegistry); - } - - private static ApplicationBuilder createFromDirectory(String dir, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - ApplicationBuilder builder = new ApplicationBuilder(applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryProfileRegistry); - for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.addSchemaFile(i.next().toString()); - } - builder.build(true); - return builder; - } - - private static QueryProfileRegistry createQueryProfileRegistryFromDirectory(String dir) { - File queryProfilesDir = new File(dir, "query-profiles"); - if ( ! queryProfilesDir.exists()) return new QueryProfileRegistry(); - return new QueryProfileXMLReader().read(queryProfilesDir.toString()); - } - - // TODO: The build methods below just call the create methods above - remove - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. - * - * @param fileName the file to build from - * @return the built {@link Schema} object - * @throws IOException thrown if there was a problem reading the file - * @throws ParseException thrown if there was a problem parsing the file content - */ - public static Schema buildFromFile(String fileName) throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. - * - * @param fileName the file to build from - * @param rankProfileRegistry registry for rank profiles - * @return the built {@link Schema} object - * @throws IOException thrown if there was a problem reading the file - * @throws ParseException thrown if there was a problem parsing the file content - */ - public static Schema buildFromFile(String fileName, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) - throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry); - } - - /** - * Convenience factory method to import and build a {@link Schema} from a file. - * - * @param fileName the file to build from - * @param deployLogger logger for deploy messages - * @param rankProfileRegistry registry for rank profiles - * @return the built {@link Schema} object - * @throws IOException thrown if there was a problem reading the file - * @throws ParseException thrown if there was a problem parsing the file content - */ - public static Schema buildFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) - throws IOException, ParseException { - return createFromFile(fileName, deployLogger, rankProfileRegistry, queryProfileRegistry).getSchema(); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a raw object. - * - * @param rawSchema the raw object to build from - * @return the built {@link ApplicationBuilder} object - * @see #add(Schema) - */ - public static ApplicationBuilder createFromRawSchema(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry); - builder.add(rawSchema); - builder.build(true); - return builder; - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a raw object. - * - * @param rawSchema the raw object to build from - * @return the built {@link Schema} object - * @see #add(Schema) - */ - public static Schema buildFromRawSchema(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - return createFromRawSchema(rawSchema, rankProfileRegistry, queryProfileRegistry).getSchema(); - } - - public RankProfileRegistry getRankProfileRegistry() { - return rankProfileRegistry; - } - - public QueryProfileRegistry getQueryProfileRegistry() { - return queryProfileRegistry; - } - - public ModelContext.Properties getProperties() { return properties; } - - public DeployLogger getDeployLogger() { return deployLogger; } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java deleted file mode 100644 index 69850b21224..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.derived.FileDistributedConstants; -import com.yahoo.searchdefinition.document.ImmutableSDField; - -import java.util.LinkedHashSet; -import java.util.Set; - -/** - * The rank profile containing default settings. This is derived from the fields - * whenever this is accessed. - * - * @author bratseth - */ -public class DefaultRankProfile extends RankProfile { - - /** - * Creates a new rank profile - * - * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} - * to use for storing and looking up rank profiles - */ - public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { - super("default", schema, rankProfileRegistry); - } - - /** Ignore self inheriting of default as some applications may use that for historical reasons. */ - public void inherit(String inheritedName) { - if (inheritedName.equals("default")) return; - super.inherit(inheritedName); - } - - @Override - public RankSetting getRankSetting(String fieldOrIndex, RankSetting.Type type) { - RankSetting setting = super.getRankSetting(fieldOrIndex, type); - if (setting != null) return setting; - - ImmutableSDField field = schema().getConcreteField(fieldOrIndex); - if (field != null) { - setting = toRankSetting(field, type); - if (setting != null) - return setting; - } - - Index index = schema().getIndex(fieldOrIndex); - if (index != null) { - setting = toRankSetting(index, type); - if (setting != null) - return setting; - } - - return null; - } - - private RankSetting toRankSetting(ImmutableSDField field, RankSetting.Type type) { - if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight() > 0 && field.getWeight() != 100) - return new RankSetting(field.getName(), type, field.getWeight()); - if (type.equals(RankSetting.Type.RANKTYPE)) - return new RankSetting(field.getName(), type, field.getRankType()); - if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost() > 0) - return new RankSetting(field.getName(), type, field.getLiteralBoost()); - - // Index level setting really - if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { - return new RankSetting(field.getName(), type, true); - } - - return null; - } - - private RankSetting toRankSetting(Index index, RankSetting.Type type) { - /* TODO: Add support for indexes by adding a ranking object to the index - if (type.equals(RankSetting.Type.PREFERBITVECTOR) && index.isPreferBitVector()) { - return new RankSetting(index.getName(), type, new Boolean(true)); - } - */ - return null; - } - - /** - * Returns the names of the fields which have a rank boost setting - * explicitly in this profile or in fields - */ - @Override - public Set rankSettings() { - Set settings = new LinkedHashSet<>(20); - settings.addAll(this.rankSettings); - for (ImmutableSDField field : schema().allConcreteFields() ) { - addSetting(field, RankSetting.Type.WEIGHT, settings); - addSetting(field, RankSetting.Type.RANKTYPE, settings); - addSetting(field, RankSetting.Type.LITERALBOOST, settings); - addSetting(field, RankSetting.Type.PREFERBITVECTOR, settings); - } - - // For settings that really pertains to indexes do the explicit indexes too - for (Index index : schema().getExplicitIndices()) { - addSetting(index, RankSetting.Type.PREFERBITVECTOR, settings); - } - return settings; - } - - private void addSetting(ImmutableSDField field, RankSetting.Type type, Set settings) { - if (type.isIndexLevel()) { - addIndexSettings(field, type, settings); - } - else { - RankSetting setting = toRankSetting(field, type); - if (setting == null) return; - settings.add(setting); - } - } - - private void addIndexSettings(ImmutableSDField field, RankSetting.Type type, Set settings) { - String indexName = field.getName(); - - // TODO: Make a ranking object in the index override the field level ranking object - if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { - settings.add(new RankSetting(indexName, type, true)); - } - } - - private void addSetting(Index index, RankSetting.Type type, Set settings) { - RankSetting setting = toRankSetting(index, type); - if (setting == null) return; - settings.add(setting); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java deleted file mode 100644 index be2aa500b2b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.FileReference; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.path.Path; - -import java.nio.ByteBuffer; -import java.util.Objects; - -public class DistributableResource implements Comparable { - - public enum PathType { FILE, URI, BLOB } - - /** The search definition-unique name of this constant */ - private final String name; - // TODO: Make path/pathType final - private PathType pathType; - private String path; - private FileReference fileReference = new FileReference(""); - - public PathType getPathType() { - return pathType; - } - - public DistributableResource(String name) { - this(name, null, PathType.FILE); - } - public DistributableResource(String name, String path) { - this(name, path, PathType.FILE); - } - public DistributableResource(String name, String path, PathType type) { - this.name = name; - this.path = path; - this.pathType = type; - } - - // TODO: Remove and make path/pathType final - public void setFileName(String fileName) { - Objects.requireNonNull(fileName, "Filename cannot be null"); - this.path = fileName; - this.pathType = PathType.FILE; - } - - // TODO: Remove and make path/pathType final - public void setUri(String uri) { - Objects.requireNonNull(uri, "uri cannot be null"); - this.path = uri; - this.pathType = PathType.URI; - } - - public String getName() { return name; } - public String getFileName() { return path; } - public Path getFilePath() { return Path.fromString(path); } - public String getUri() { return path; } - public String getFileReference() { return fileReference.value(); } - - public void validate() { - switch (pathType) { - case FILE: - case URI: - if (path == null || path.isEmpty()) - throw new IllegalArgumentException("Distributable URI/FILE resource must have a file or uri."); - break; - } - } - - public void register(FileRegistry fileRegistry) { - switch (pathType) { - case FILE: - fileReference = fileRegistry.addFile(path); - break; - case URI: - fileReference = fileRegistry.addUri(path); - break; - default: - throw new IllegalArgumentException("Unknown path type " + pathType); - } - } - - protected void register(FileRegistry fileRegistry, ByteBuffer blob) { - fileReference = fileRegistry.addBlob(path, blob); - } - - @Override - public String toString() { - return "resource '" + name + " of type '" + pathType + "' with ref '" + fileReference + "'"; - } - - @Override - public int compareTo(DistributableResource o) { - return name.compareTo(o.getName()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentGraphValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentGraphValidator.java deleted file mode 100644 index 33f202a55fa..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentGraphValidator.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDDocumentType; - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.List; - -import static java.util.stream.Collectors.joining; - -/** - * Validates that there are no cycles between document types (exception: self-reference is allowed). - * Example: if document B inherits A, then A cannot have a document reference to B. - * - * @author bjorncs - */ -public class DocumentGraphValidator { - - public void validateDocumentGraph(List documents) { - for (SDDocumentType document : documents) { - validateRoot(document); - } - } - - private static void validateRoot(SDDocumentType root) { - validateChildren(root, root); - } - - private static void validateChildren(SDDocumentType root, SDDocumentType currentDocument) { - try { - currentDocument.getDocumentReferences().get() - .forEach(entry -> { - SDDocumentType referencedDocument = entry.getValue().targetSearch().getDocument(); - validateDocument(root, referencedDocument); - }); - currentDocument.getInheritedTypes() - .forEach(inheritedDocument -> { - if (!isRootDocument(inheritedDocument)) { - validateDocument(root, inheritedDocument); - } - }); - } catch (DocumentGraphException e) { - e.addParentDocument(currentDocument); - throw e; - } - } - - private static void validateDocument(SDDocumentType root, SDDocumentType currentDocument) { - if (root.equals(currentDocument)) { - throw new DocumentGraphException(currentDocument); - } - validateChildren(root, currentDocument); - } - - private static boolean isRootDocument(SDDocumentType doc) { - return doc.getName().equals("document"); - } - - public static class DocumentGraphException extends IllegalArgumentException { - private final Deque deque = new ArrayDeque<>(); - - public DocumentGraphException(SDDocumentType document) { - deque.addLast(document); - } - - public void addParentDocument(SDDocumentType document) { - deque.addFirst(document); - } - - @Override - public String getMessage() { - return deque.stream() - .map(SDDocumentType::getName) - .collect(joining("->", "Document dependency cycle detected: ", ".")); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java deleted file mode 100644 index bfe0db6f65d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ /dev/null @@ -1,641 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; -import com.yahoo.document.StructDataType; -import com.yahoo.document.StructuredDataType; -import com.yahoo.document.TemporaryStructuredDataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.documentmodel.DataTypeCollection; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.documentmodel.OwnedStructDataType; -import com.yahoo.documentmodel.OwnedTemporaryType; -import com.yahoo.documentmodel.TemporaryUnknownType; -import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedFields; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; -import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.documentmodel.FieldView; -import com.yahoo.vespa.documentmodel.SearchDef; -import com.yahoo.vespa.documentmodel.SearchField; - -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @author baldersheim - */ -public class DocumentModelBuilder { - - private final DocumentModel model; - - public DocumentModelBuilder() { - this.model = new DocumentModel(); - this.model.getDocumentManager().add(VespaDocumentType.INSTANCE); - } - - public DocumentModel build(Collection schemaList) { - List docList = new LinkedList<>(); - for (Schema schema : schemaList) { - docList.add(schema.getDocument()); - } - docList = sortDocumentTypes(docList); - addDocumentTypes(docList); - for (Collection toAdd = tryAdd(schemaList); - ! toAdd.isEmpty() && (toAdd.size() < schemaList.size()); - toAdd = tryAdd(schemaList)) { - schemaList = toAdd; - } - return model; - } - - private List sortDocumentTypes(List docList) { - Set doneNames = new HashSet<>(); - doneNames.add(SDDocumentType.VESPA_DOCUMENT.getName()); - List doneList = new LinkedList<>(); - List prevList = null; - List nextList = docList; - while (prevList == null || nextList.size() < prevList.size()) { - prevList = nextList; - nextList = new LinkedList<>(); - for (SDDocumentType doc : prevList) { - boolean isDone = true; - for (SDDocumentType inherited : doc.getInheritedTypes()) { - if (!doneNames.contains(inherited.getName())) { - isDone = false; - break; - } - } - if (isDone) { - doneNames.add(doc.getName()); - doneList.add(doc); - } else { - nextList.add(doc); - } - } - } - if (!nextList.isEmpty()) { - throw new IllegalArgumentException("Could not resolve inheritance of document types " + - toString(prevList) + "."); - } - return doneList; - } - - private static String toString(List lst) { - StringBuilder out = new StringBuilder(); - for (int i = 0, len = lst.size(); i < len; ++i) { - out.append("'").append(lst.get(i).getName()).append("'"); - if (i < len - 2) { - out.append(", "); - } else if (i < len - 1) { - out.append(" and "); - } - } - return out.toString(); - } - - private Collection tryAdd(Collection schemaList) { - Collection left = new ArrayList<>(); - for (Schema schema : schemaList) { - try { - addToModel(schema); - } catch (RetryLaterException e) { - left.add(schema); - } - } - return left; - } - - private void addToModel(Schema schema) { - // Then we add the search specific stuff - SearchDef searchDef = new SearchDef(schema.getName()); - addSearchFields(schema.extraFieldList(), searchDef); - for (Field f : schema.getDocument().fieldSet()) { - addSearchField((SDField) f, searchDef); - } - for (SDField field : schema.allConcreteFields()) { - for (Attribute attribute : field.getAttributes().values()) { - if ( ! searchDef.getFields().containsKey(attribute.getName())) { - searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true)); - } - } - } - - for (Field f : schema.getDocument().fieldSet()) { - addAlias((SDField) f, searchDef); - } - model.getSearchManager().add(searchDef); - } - - private static void addSearchFields(Collection fields, SearchDef searchDef) { - for (SDField field : fields) { - addSearchField(field, searchDef); - } - } - - private static void addSearchField(SDField field, SearchDef searchDef) { - SearchField searchField = - new SearchField(field, - field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), - field.getAttributes().containsKey(field.getName())); - searchDef.add(searchField); - - // Add field to views - addToView(field.getIndices().keySet(), searchField, searchDef); - } - - private static void addAlias(SDField field, SearchDef searchDef) { - for (Map.Entry entry : field.getAliasToName().entrySet()) { - searchDef.addAlias(entry.getKey(), entry.getValue()); - } - } - - private static void addToView(Collection views, Field field, SearchDef searchDef) { - for (String viewName : views) { - addToView(viewName, field, searchDef); - } - } - - private static void addToView(String viewName, Field field, SearchDef searchDef) { - if (searchDef.getViews().containsKey(viewName)) { - searchDef.getViews().get(viewName).add(field); - } else { - if (!searchDef.getFields().containsKey(viewName)) { - FieldView view = new FieldView(viewName); - view.add(field); - searchDef.add(view); - } - } - } - - private static String descT(DataType type) { - if (type == null) { return ""; } - return "'" + type.getName() + "' [" + type.getId() + "] {"+type.getClass() + "}"; - } - - private void addDocumentTypes(List docList) { - LinkedList lst = new LinkedList<>(); - for (SDDocumentType doc : docList) { - lst.add(convert(doc)); - model.getDocumentManager().add(lst.getLast()); - } - Map replacements = new IdentityHashMap<>(); - for(NewDocumentType doc : lst) { - resolveTemporaries(doc.getAllTypes(), lst, replacements); - resolveTemporariesRecurse(doc.getContentStruct(), doc.getAllTypes(), lst, replacements); - } - for(NewDocumentType doc : lst) { - for (var entry : replacements.entrySet()) { - var old = entry.getKey(); - if (doc.getDataType(old.getId()) == old) { - doc.replace(entry.getValue()); - } - } - } - } - - private static void resolveTemporaries(DataTypeCollection dtc, - Collection docs, - Map replacements) { - for (DataType type : dtc.getTypes()) { - resolveTemporariesRecurse(type, dtc, docs, replacements); - } - } - - @SuppressWarnings("deprecation") - private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, - Collection docs, - Map replacements) { - if (replacements.containsKey(type)) { - return replacements.get(type); - } - DataType original = type; - if (type instanceof TemporaryStructuredDataType) { - throw new IllegalArgumentException("Cannot handle temporary: " + type); - } - if (type instanceof TemporaryUnknownType) { - // must be a known struct or document type - DataType other = repo.getDataType(type.getId()); - if (other == null || other == type) { - // maybe it is the name of a document type: - other = getDocumentType(docs, type.getName()); - } - if (other == null) { - throw new IllegalArgumentException("No replacement found for temporary type: " + type); - } - type = other; - } else if (type instanceof OwnedTemporaryType) { - // must be replaced with the real struct type - DataType other = repo.getDataType(type.getId()); - if (other == null || other == type) { - throw new IllegalArgumentException("No replacement found for temporary type: " + type); - } - if (other instanceof OwnedStructDataType) { - var owned = (OwnedTemporaryType) type; - String ownedBy = owned.getOwnerName(); - var otherOwned = (OwnedStructDataType) other; - String otherOwnedBy = otherOwned.getOwnerName(); - if (! ownedBy.equals(otherOwnedBy)) { - throw new IllegalArgumentException("Wrong document for type: " + otherOwnedBy + " but expected " + ownedBy); - } - } else { - throw new IllegalArgumentException("Found wrong sort of type: " + other + " [" + other.getClass() + "]"); - } - type = other; - } else if (type instanceof DocumentType) { - DataType other = getDocumentType(docs, type.getName()); - if (other != null) { - type = other; - } else if (type != DataType.DOCUMENT) { - throw new IllegalArgumentException - ("Can not handle nested document definitions. Undefined document type: " + type.toString()); - } - } else if (type instanceof NewDocumentType) { - DataType other = getDocumentType(docs, type.getName()); - if (other != null) { - type = other; - } - } else if (type instanceof StructDataType) { - // trick avoids infinite recursion: - var old = replacements.put(original, type); - assert(old == null); - StructDataType dt = (StructDataType) type; - for (com.yahoo.document.Field field : dt.getFields()) { - var ft = field.getDataType(); - var newft = resolveTemporariesRecurse(ft, repo, docs, replacements); - if (ft != newft) { - // XXX deprecated: - field.setDataType(newft); - } - } - old = replacements.remove(original); - assert(old == type); - } - else if (type instanceof MapDataType) { - MapDataType t = (MapDataType) type; - var old_kt = t.getKeyType(); - var old_vt = t.getValueType(); - var kt = resolveTemporariesRecurse(old_kt, repo, docs, replacements); - var vt = resolveTemporariesRecurse(old_vt, repo, docs, replacements); - if (kt != old_kt || vt != old_vt) { - type = new MapDataType(kt, vt, t.getId()); - } - } - else if (type instanceof ArrayDataType) { - ArrayDataType t = (ArrayDataType) type; - var old_nt = t.getNestedType(); - var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); - if (nt != old_nt) { - type = new ArrayDataType(nt, t.getId()); - } - } - else if (type instanceof WeightedSetDataType) { - WeightedSetDataType t = (WeightedSetDataType) type; - var old_nt = t.getNestedType(); - var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); - if (nt != old_nt) { - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - type = new WeightedSetDataType(nt, c, r, t.getId()); - } - } - else if (type instanceof NewDocumentReferenceDataType) { - var t = (NewDocumentReferenceDataType) type; - var doc = getDocumentType(docs, t.getTargetTypeName()); - type = doc.getReferenceDataType(); - } - if (type != original) { - replacements.put(original, type); - } - return type; - } - - private static NewDocumentType getDocumentType(Collection docs, String name) { - for (NewDocumentType doc : docs) { - if (doc.getName().equals(name)) { - return doc; - } - } - return null; - } - - private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { - if (sa.getInherits() != null) { - AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); - SDAnnotationType inherited = (SDAnnotationType) tmp; - return ((inherited.getSdDocType() != null) || anyParentsHavePayLoad(inherited, sdoc)); - } - return false; - } - - private NewDocumentType convert(SDDocumentType sdoc) { - NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), - sdoc.getDocumentType().contentStruct(), - sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences()), - convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); - for (SDDocumentType n : sdoc.getInheritedTypes()) { - NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); - NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); - if (inherited != null) { - dt.inherit(inherited); - } - } - var extractor = new TypeExtractor(dt); - extractor.extract(sdoc); - return dt; - } - - static class TypeExtractor { - private final NewDocumentType targetDt; - Map annotationInheritance = new LinkedHashMap<>(); - Map structInheritance = new LinkedHashMap<>(); - private final Map inProgress = new IdentityHashMap<>(); - TypeExtractor(NewDocumentType target) { - this.targetDt = target; - } - - void extract(SDDocumentType sdoc) { - for (SDDocumentType type : sdoc.getTypes()) { - if (type.isStruct()) { - handleStruct(type); - } else { - throw new IllegalArgumentException("Data type '" + type.getName() + "' is not a struct => tostring='" + type.toString() + "'."); - } - } - for (SDDocumentType type : sdoc.getTypes()) { - for (SDDocumentType proxy : type.getInheritedTypes()) { - var inherited = (StructDataType) targetDt.getDataTypeRecursive(proxy.getName()); - var converted = (StructDataType) targetDt.getDataType(type.getName()); - assert(converted instanceof OwnedStructDataType); - assert(inherited instanceof OwnedStructDataType); - if (! converted.inherits(inherited)) { - converted.inherit(inherited); - } - } - } - for (AnnotationType annotation : sdoc.getAnnotations().values()) { - targetDt.add(annotation); - } - for (AnnotationType annotation : sdoc.getAnnotations().values()) { - SDAnnotationType sa = (SDAnnotationType) annotation; - if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { - annotationInheritance.put(annotation, sa.getInherits()); - } - if (annotation.getDataType() == null) { - if (sa.getSdDocType() != null) { - StructDataType s = handleStruct(sa.getSdDocType()); - annotation.setDataType(s); - if ((sa.getInherits() != null)) { - structInheritance.put(s, "annotation." + sa.getInherits()); - } - } else if (sa.getInherits() != null) { - StructDataType s = new OwnedStructDataType("annotation." + annotation.getName(), sdoc.getName()); - if (anyParentsHavePayLoad(sa, sdoc)) { - annotation.setDataType(s); - addType(s); - } - structInheritance.put(s, "annotation." + sa.getInherits()); - } - } else { - var dt = annotation.getDataType(); - if (dt instanceof StructDataType) { - handleStruct((StructDataType) dt); - } - } - } - for (Map.Entry e : annotationInheritance.entrySet()) { - e.getKey().inherit(targetDt.getAnnotationType(e.getValue())); - } - for (Map.Entry e : structInheritance.entrySet()) { - StructDataType s = (StructDataType)targetDt.getDataType(e.getValue()); - if (s != null) { - e.getKey().inherit(s); - } - } - handleStruct(sdoc.getDocumentType().contentStruct()); - extractDataTypesFromFields(sdoc.fieldSet()); - } - - private void extractDataTypesFromFields(Collection fields) { - for (Field f : fields) { - DataType type = f.getDataType(); - if (testAddType(type)) { - extractNestedTypes(type); - addType(type); - } - } - } - - private void extractNestedTypes(DataType type) { - if (inProgress.containsKey(type)) { - return; - } - inProgress.put(type, this); - if (type instanceof StructDataType) { - StructDataType tmp = (StructDataType) type; - extractDataTypesFromFields(tmp.getFieldsThisTypeOnly()); - } else if (type instanceof CollectionDataType) { - CollectionDataType tmp = (CollectionDataType) type; - extractNestedTypes(tmp.getNestedType()); - addType(tmp.getNestedType()); - } else if (type instanceof MapDataType) { - MapDataType tmp = (MapDataType) type; - extractNestedTypes(tmp.getKeyType()); - extractNestedTypes(tmp.getValueType()); - addType(tmp.getKeyType()); - addType(tmp.getValueType()); - } else if (type instanceof TemporaryAnnotationReferenceDataType) { - throw new IllegalArgumentException(type.toString()); - } - } - - private boolean testAddType(DataType type) { return internalAddType(type, true); } - - private boolean addType(DataType type) { return internalAddType(type, false); } - - private boolean internalAddType(DataType type, boolean dryRun) { - DataType oldType = targetDt.getDataTypeRecursive(type.getId()); - if (oldType == null) { - if ( ! dryRun) { - targetDt.add(type); - } - return true; - } - if (oldType == type) { - return false; - } - if (targetDt.getDataType(type.getId()) == null) { - if ((oldType instanceof OwnedStructDataType) - && (type instanceof OwnedStructDataType)) - { - var oldOwned = (OwnedStructDataType) oldType; - var newOwned = (OwnedStructDataType) type; - if (newOwned.getOwnerName().equals(targetDt.getName()) && - ! oldOwned.getOwnerName().equals(targetDt.getName())) - { - if ( ! dryRun) { - targetDt.add(type); - } - return true; - } - } - } - if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { - StructDataType s = (StructDataType) type; - StructDataType os = (StructDataType) oldType; - if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { - if ( ! dryRun) { - targetDt.replace(type); - } - return true; - } - } - return false; - } - - - @SuppressWarnings("deprecation") - private void specialHandleAnnotationReference(Field field) { - DataType fieldType = specialHandleAnnotationReferenceRecurse(field.getName(), field.getDataType()); - if (fieldType == null) { - return; - } - field.setDataType(fieldType); // XXX deprecated - } - - private DataType specialHandleAnnotationReferenceRecurse(String fieldName, - DataType dataType) { - if (dataType instanceof TemporaryAnnotationReferenceDataType) { - TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; - if (refType.getId() != 0) { - return null; - } - AnnotationType target = targetDt.getAnnotationType(refType.getTarget()); - if (target == null) { - throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + - "' does not exist."); - } - dataType = new AnnotationReferenceDataType(target); - addType(dataType); - return dataType; - } - else if (dataType instanceof MapDataType) { - MapDataType t = (MapDataType)dataType; - DataType valueType = specialHandleAnnotationReferenceRecurse(fieldName, t.getValueType()); - if (valueType == null) { - return null; - } - var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); - addType(mapType); - return mapType; - } - else if (dataType instanceof ArrayDataType) { - ArrayDataType t = (ArrayDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); - if (nestedType == null) { - return null; - } - var lstType = new ArrayDataType(nestedType, t.getId()); - addType(lstType); - return lstType; - } - else if (dataType instanceof WeightedSetDataType) { - WeightedSetDataType t = (WeightedSetDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); - if (nestedType == null) { - return null; - } - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); - addType(lstType); - return lstType; - } - return null; - } - - @SuppressWarnings("deprecation") - private StructDataType handleStruct(SDDocumentType type) { - if (type.isStruct()) { - var st = type.getStruct(); - if (st.getName().equals(type.getName()) && - (st instanceof StructDataType) && - (! (st instanceof TemporaryUnknownType)) && - (! (st instanceof OwnedTemporaryType))) - { - return handleStruct((StructDataType) st); - } - } - StructDataType s = new OwnedStructDataType(type.getName(), targetDt.getName()); - for (Field f : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { - specialHandleAnnotationReference(f); - s.addField(f); - } - for (StructDataType inherited : type.getDocumentType().contentStruct().getInheritedTypes()) { - s.inherit(inherited); - } - extractNestedTypes(s); - addType(s); - return s; - } - - private StructDataType handleStruct(StructDataType s) { - for (Field f : s.getFieldsThisTypeOnly()) { - specialHandleAnnotationReference(f); - } - extractNestedTypes(s); - addType(s); - return s; - } - - } - - private static Set convertDocumentReferencesToNames(Optional documentReferences) { - if (!documentReferences.isPresent()) { - return Set.of(); - } - return documentReferences.get().referenceMap().values().stream() - .map(documentReference -> documentReference.targetSearch().getDocument()) - .map(documentType -> new NewDocumentType.Name(documentType.getName())) - .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); - } - - private static Set convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { - if (importedFields == null) { - return Set.of(); - } - return Collections.unmodifiableSet(importedFields.fields().keySet()); - } - - public static class RetryLaterException extends IllegalArgumentException { - public RetryLaterException(String message) { - super(message); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java deleted file mode 100644 index 1d71a9f1494..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.document.SDDocumentType; - -/** - * A search that was derived from an sd file containing no search element(s), only - * document specifications, so the name of this is decided by parsing and adding the document instance. - * - * @author vegardh - */ -public class DocumentOnlySchema extends Schema { - - public DocumentOnlySchema(ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties) { - super(applicationPackage, fileRegistry, deployLogger, properties); - } - - @Override - public void addDocument(SDDocumentType docType) { - if (getName() == null) { - setName(docType.getName()); - } - super.addDocument(docType); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReference.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReference.java deleted file mode 100644 index 145effcdc9d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReference.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.Field; - -/** - * Represents a document reference. Contains the document reference field and the search instance of the referred document. - * - * @author bjorncs - */ -public class DocumentReference { - - private final Field referenceField; - private final Schema targetSchema; - - public DocumentReference(Field referenceField, Schema targetSchema) { - this.referenceField = referenceField; - this.targetSchema = targetSchema; - } - - public Field referenceField() { - return referenceField; - } - - public Schema targetSearch() { - return targetSchema; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java deleted file mode 100644 index 2e8362f1839..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.Field; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; - -import java.util.Collection; -import java.util.Map; -import java.util.stream.Stream; - -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; - -/** - * Resolves all document references in the search definitions - * - * Iterates through all document fields having a {@link NewDocumentReferenceDataType} and uses {@link NewDocumentReferenceDataType#getTargetTypeName()} - * to determine the referenced document. This information is aggregated into a {@link DocumentReferences} object. - * - * @author bjorncs - */ -public class DocumentReferenceResolver { - - private final Map schemaMapping; - - public DocumentReferenceResolver(Collection schemas) { - this.schemaMapping = createDocumentNameToSearchMapping(schemas); - } - - public void resolveReferences(SDDocumentType documentType) { - var references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType)); - documentType.setDocumentReferences(references); - } - - public void resolveInheritedReferences(SDDocumentType documentType) { - resolveInheritedReferencesRecursive(documentType, documentType.getInheritedTypes()); - } - - private void resolveInheritedReferencesRecursive(SDDocumentType documentType, - Collection inheritedTypes) { - for (var inheritedType : inheritedTypes) { - documentType.getDocumentReferences().get().mergeFrom(inheritedType.getDocumentReferences().get()); - } - for (var inheritedType : inheritedTypes) { - resolveInheritedReferencesRecursive(documentType, inheritedType.getInheritedTypes()); - } - } - - private Map createFieldToDocumentReferenceMapping(SDDocumentType documentType) { - return fieldStream(documentType) - .filter(field -> field.getDataType() instanceof NewDocumentReferenceDataType) - .collect(toMap(Field::getName, this::createDocumentReference)); - } - - private DocumentReference createDocumentReference(Field field) { - if (!isAttribute(field)) { - throw new IllegalArgumentException( - String.format( - "The field '%s' is an invalid document reference. The field must be an attribute.", - field.getName())); - } - NewDocumentReferenceDataType reference = (NewDocumentReferenceDataType) field.getDataType(); - String targetDocumentName = getTargetDocumentName(reference); - Schema schema = schemaMapping.get(targetDocumentName); - if (schema == null) { - throw new IllegalArgumentException( - String.format("Invalid document reference '%s': " + - "Could not find document type '%s'", field.getName(), targetDocumentName)); - } - return new DocumentReference(field, schema); - } - - private static boolean isAttribute(Field field) { - SDField sdField = (SDField) field; // Ugly, but SDDocumentType only expose the fields as the super class Field - return sdField.doesAttributing(); - } - - private static Map createDocumentNameToSearchMapping(Collection schemaDefintions) { - return schemaDefintions.stream() - .filter(search -> search.getDocument() != null) - .collect(toMap(search -> search.getDocument().getName(), identity())); - } - - private static Stream fieldStream(SDDocumentType documentType) { - return documentType.getDocumentType().getFields().stream(); - } - - private static String getTargetDocumentName(NewDocumentReferenceDataType reference) { - return reference.getTargetTypeName(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java deleted file mode 100644 index d0ac967ce7e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import java.util.Collections; -import java.util.Iterator; -import java.util.Map; -import java.util.stream.Stream; - -/** - * Contains all document references for a document mapped by field name - * - * @author bjorncs - */ -public class DocumentReferences implements Iterable> { - private final Map references; - - public DocumentReferences(Map references) { - this.references = references; - } - - public void mergeFrom(DocumentReferences other) { - references.putAll(other.references); - } - - @Override - public Iterator> iterator() { - return Collections.unmodifiableSet(references.entrySet()).iterator(); - } - - public Map referenceMap() { - return Collections.unmodifiableMap(references); - } - - public Stream> stream() { - return references.entrySet().stream(); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java deleted file mode 100644 index caaef63fd73..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import java.util.List; - -/** - * A rank profile which ignores all calls made to it which may fail in a document only setting. - * This is used by the search definition parser when it is requested to parse documents only, - * to avoid having to check for this in every method which adds to the rank profile. - * (And why do we ever want to parse documents only? Because it is used when generating Java classes - * from documents, where the full application package may not be available.) - * - * @author bratseth - */ -public class DocumentsOnlyRankProfile extends RankProfile { - - public DocumentsOnlyRankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { - super(name, schema, rankProfileRegistry); - } - - @Override - public void setFirstPhaseRanking(String expression) { - // Ignore - } - - @Override - public void setSecondPhaseRanking(String expression) { - // Ignore - } - - @Override - public void addFunction(String name, List arguments, String expression, boolean inline) { - // Ignore - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java deleted file mode 100644 index a226fa4812f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchlib.rankingexpression.Reference; - -import java.util.Optional; - -/** - * Utility methods for query, document and constant rank feature names - * - * @author bratseth - */ -public class FeatureNames { - - public static Reference asConstantFeature(String constantName) { - return Reference.simple("constant", quoteIfNecessary(constantName)); - } - - public static Reference asAttributeFeature(String attributeName) { - return Reference.simple("attribute", attributeName); - } - - public static Reference asQueryFeature(String propertyName) { - return Reference.simple("query", quoteIfNecessary(propertyName)); - } - - /** Returns true if the given reference is an attribute, constant or query feature */ - public static boolean isSimpleFeature(Reference reference) { - if ( ! reference.isSimple()) return false; - String name = reference.name(); - return name.equals("attribute") || name.equals("constant") || name.equals("query"); - } - - /** Returns true if this is a constant */ - public static boolean isConstantFeature(Reference reference) { - if ( ! isSimpleFeature(reference)) return false; - return reference.name().equals("constant"); - } - - /** Returns true if this is a query feature */ - public static boolean isQueryFeature(Reference reference) { - if ( ! isSimpleFeature(reference)) return false; - return reference.name().equals("query"); - } - - /** Returns true if this is an attribute feature */ - public static boolean isAttributeFeature(Reference reference) { - if ( ! isSimpleFeature(reference)) return false; - return reference.name().equals("attribute"); - } - - /** - * Returns the single argument of the given feature name, without any quotes, - * or empty if it is not a valid query, attribute or constant feature name - */ - public static Optional argumentOf(String feature) { - Optional reference = Reference.simple(feature); - if ( reference.isEmpty()) return Optional.empty(); - if ( ! ( reference.get().name().equals("attribute") || - reference.get().name().equals("constant") || - reference.get().name().equals("query"))) - return Optional.empty(); - - return Optional.of(reference.get().arguments().expressions().get(0).toString()); - } - - private static String quoteIfNecessary(String s) { - if (notNeedQuotes(s)) - return s; - else - return "\"" + s + "\""; - } - - static boolean notNeedQuotes(String s) { - // Faster version of the regexp [A-Za-z0-9_][A-Za-z0-9_-]* - if (s.isEmpty()) return false; - if ( ! isValidFirst(s.charAt(0))) return false; - for (int i = 1; i < s.length(); i++) { - if (!isValidAny(s.charAt(i))) return false; - } - return true; - } - private static boolean isValidFirst(char c) { - // [A-Za-z0-9_] - return (c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || ((c >= '0') && (c <= '9')); - } - private static boolean isValidAny(char c) { - // [A-Za-z0-9_-]* - return c == '-' || isValidFirst(c); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java deleted file mode 100644 index 7fa009bed67..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class FieldOperationApplier { - - public void process(SDDocumentType sdoc) { - if (!sdoc.isStruct()) { - apply(sdoc); - } - } - - protected void apply(SDDocumentType type) { - for (Field field : type.fieldSet()) { - apply(field); - } - } - - protected void apply(Field field) { - if (field instanceof SDField) { - SDField sdField = (SDField) field; - sdField.applyOperations(); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java deleted file mode 100644 index 52110e95ac7..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.SDDocumentType; - -/** - * @author Einar M R Rosenvinge - */ -public class FieldOperationApplierForSearch extends FieldOperationApplier { - - @Override - public void process(SDDocumentType sdoc) { - //Do nothing - } - - public void process(Schema schema) { - for (Field field : schema.extraFieldList()) { - apply(field); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java deleted file mode 100644 index 4a5a858f828..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -/** - * @author Einar M R Rosenvinge - */ -public class FieldOperationApplierForStructs extends FieldOperationApplier { - - @Override - public void process(SDDocumentType sdoc) { - for (SDDocumentType type : sdoc.getAllTypes()) { - if (type.isStruct()) { - apply(type); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java deleted file mode 100644 index 065ade9b094..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import com.yahoo.searchdefinition.document.FieldSet; - -/** - * The field sets owned by a {@link Schema} - * Both built in and user defined. - * - * @author vegardh - */ -public class FieldSets { - - private final Optional owner; - private final Map userFieldSets; - private final Map builtInFieldSets; - - public FieldSets(Optional owner) { - this.owner = owner; - userFieldSets = new LinkedHashMap<>(); - builtInFieldSets = new LinkedHashMap<>(); - } - - /** - * Adds an entry to user field sets, creating entries as needed - * - * @param setName name of a field set - * @param field field to add to field set - */ - public void addUserFieldSetItem(String setName, String field) { - if (userFieldSets.get(setName) == null) { - // First entry in this set - userFieldSets.put(setName, new FieldSet(setName)); - } - userFieldSets.get(setName).addFieldName(field); - } - - /** - * Adds an entry to built in field sets, creating entries as needed - * - * @param setName name of a field set - * @param field field to add to field set - */ - public void addBuiltInFieldSetItem(String setName, String field) { - if (builtInFieldSets.get(setName) == null) { - // First entry in this set - builtInFieldSets.put(setName, new FieldSet(setName)); - } - builtInFieldSets.get(setName).addFieldName(field); - } - - /** Returns the built in field sets, unmodifiable */ - public Map builtInFieldSets() { - if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(builtInFieldSets); - if (builtInFieldSets.isEmpty()) return owner.get().inherited().get().fieldSets().builtInFieldSets(); - - var fieldSets = new LinkedHashMap<>(owner.get().inherited().get().fieldSets().builtInFieldSets()); - fieldSets.putAll(builtInFieldSets); - return Collections.unmodifiableMap(fieldSets); - } - - /** Returns the user defined field sets, unmodifiable */ - public Map userFieldSets() { - if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(userFieldSets); - if (userFieldSets.isEmpty()) return owner.get().inherited().get().fieldSets().userFieldSets(); - - var fieldSets = new LinkedHashMap<>(owner.get().inherited().get().fieldSets().userFieldSets()); - fieldSets.putAll(userFieldSets); - return Collections.unmodifiableMap(fieldSets); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java deleted file mode 100644 index 3719313179f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.derived.FileDistributedOnnxModels; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.vespa.documentmodel.SummaryField; - -import java.io.Reader; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Stream; - -/** - * An interface containing the non-mutating methods of {@link Schema}. - * For description of the methods see {@link Schema}. - * - * @author bjorncs - */ -public interface ImmutableSchema { - - String getName(); - Optional inherited(); - Index getIndex(String name); - ImmutableSDField getConcreteField(String name); - //TODO split in mutating/immutable by returning List - List allConcreteFields(); - List getExplicitIndices(); - Reader getRankingExpression(String fileName); - ApplicationPackage applicationPackage(); - DeployLogger getDeployLogger(); - ModelContext.Properties getDeployProperties(); - Map constants(); - LargeRankExpressions rankExpressionFiles(); - Map onnxModels(); - Stream allImportedFields(); - SDDocumentType getDocument(); - ImmutableSDField getField(String name); - - default Stream allFields() { - return allFieldsList().stream(); - } - List allFieldsList(); - - List getSummaryFields(ImmutableSDField field); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java deleted file mode 100644 index 512cae5021a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDDocumentType; - -import java.util.Collection; - -/** - * Enumerates and emplaces a set of all imported fields into a SDDocumentType from - * its corresponding Search instance. - */ -public class ImportedFieldsEnumerator { - - private final Collection schemas; - - public ImportedFieldsEnumerator(Collection schemas) { - this.schemas = schemas; - } - - public void enumerateImportedFields(SDDocumentType documentType) { - var search = this.schemas.stream() - .filter(s -> s.getDocument() != null) - .filter(s -> s.getDocument().getName().equals(documentType.getName())) - .findFirst(); - if (search.isEmpty()) { - return; // No imported fields present. - } - search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java deleted file mode 100644 index 14f07f224ab..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.Stemming; - -import java.io.Serializable; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * An index definition in a search definition. - * Two indices are equal if they have the same name and the same settings, except - * alias settings (which are excluded). - * - * @author bratseth - */ -public class Index implements Cloneable, Serializable { - - public enum Type { - - VESPA("vespa"); - private final String name; - Type(String name) { this.name = name; } - public String getName() { return name; } - - } - - // Please see hashCode, equals and copy when adding attributes to this - - /** The search definition-unique name of this index */ - private String name; - - /** The rank type of this index */ - private RankType rankType = null; - - /** Whether this index supports prefix search */ - private boolean prefix; - - /** The list of aliases (Strings) to this index name */ - private Set aliases = new java.util.LinkedHashSet<>(1); - - /** - * The stemming setting of this field, or null to use the default. - * Default is determined by the owning search definition. - */ - private Stemming stemming = null; - - private Type type = Type.VESPA; - - /** The boolean index definition, if set */ - private BooleanIndexDefinition boolIndex; - - private Optional hnswIndexParams = Optional.empty(); - - /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ - private boolean interleavedFeatures = false; - - public Index(String name) { - this(name, false); - } - - public Index(String name, boolean prefix) { - this.name = name; - this.prefix = prefix; - } - - public void setName(String name) { this.name = name; } - - public String getName() { return name; } - - /** Sets the rank type of this field */ - public void setRankType(RankType rankType) { this.rankType = rankType; } - - /** Returns the rank type of this field, or null if nothing is set */ - public RankType getRankType() { return rankType; } - - /** Return the stemming setting of this index, may be null */ - public Stemming getStemming() { return stemming; } - - /** - * Whether this field should be stemmed in this search definition, - * this is never null - */ - public Stemming getStemming(Schema schema) { - if (stemming != null) - return stemming; - else - return schema.getStemming(); - } - - /** - * Sets how this field should be stemmed, or set to null to use the default. - */ - public void setStemming(Stemming stemming) { this.stemming = stemming; } - - /** Returns whether this index supports prefix search, default is false */ - public boolean isPrefix() { return prefix; } - - /** Sets whether this index supports prefix search */ - public void setPrefix(boolean prefix) { this.prefix=prefix; } - - /** Adds an alias to this index name */ - public void addAlias(String alias) { - aliases.add(alias); - } - - /** Returns a read-only iterator of the aliases (Strings) to this index name */ - public Iterator aliasIterator() { - return Collections.unmodifiableSet(aliases).iterator(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Index index = (Index) o; - return prefix == index.prefix && - interleavedFeatures == index.interleavedFeatures && - Objects.equals(name, index.name) && - rankType == index.rankType && - Objects.equals(aliases, index.aliases) && - stemming == index.stemming && - type == index.type && - Objects.equals(boolIndex, index.boolIndex) && - Objects.equals(hnswIndexParams, index.hnswIndexParams); - } - - @Override - public int hashCode() { - return Objects.hash(name, rankType, prefix, aliases, stemming, type, boolIndex, hnswIndexParams, interleavedFeatures); - } - - public String toString() { - String rankTypeName = rankType == null ? "(none)" : rankType.name(); - return "index '" + name + - "' [ranktype: " + rankTypeName + - ", prefix: " + prefix + "]"; - } - - /** Makes a deep copy of this index */ - @Override - public Object clone() { - try { - Index copy = (Index)super.clone(); - copy.aliases = new LinkedHashSet<>(this.aliases); - return copy; - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error",e); - } - } - - public Index copy() { - return (Index)clone(); - } - - /** Returns the index engine type */ - public Type getType() { - return type; - } - - /** Sets the index engine type */ - public void setType(Type type) { - this.type = type; - } - - /** Returns the boolean index definition */ - public BooleanIndexDefinition getBooleanIndexDefiniton() { - return boolIndex; - } - - /** Sets the boolean index definition */ - public void setBooleanIndexDefiniton(BooleanIndexDefinition def) { - boolIndex = def; - } - - public Optional getHnswIndexParams() { - return hnswIndexParams; - } - - public void setHnswIndexParams(HnswIndexParams params) { - hnswIndexParams = Optional.of(params); - } - - public void setInterleavedFeatures(boolean value) { - interleavedFeatures = value; - } - - public boolean useInterleavedFeatures() { - return interleavedFeatures; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java deleted file mode 100644 index 5e3ef39f360..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.FileRegistry; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -public class LargeRankExpressions { - private final Map expressions = new ConcurrentHashMap<>(); - private final FileRegistry fileRegistry; - private final int limit; - - public LargeRankExpressions(FileRegistry fileRegistry) { - this(fileRegistry, 8192); - } - public LargeRankExpressions(FileRegistry fileRegistry, int limit) { - this.fileRegistry = fileRegistry; - this.limit = limit; - } - - public void add(RankExpressionBody expression) { - String name = expression.getName(); - RankExpressionBody prev = expressions.putIfAbsent(name, expression); - if (prev == null) { - expression.validate(); - expression.register(fileRegistry); - } else { - if ( ! prev.getBlob().equals(expression.getBlob())) { - throw new IllegalArgumentException("Rank expression '" + name + - "' defined twice. Previous blob with " + prev.getBlob().remaining() + - " bytes, while current has " + expression.getBlob().remaining() + " bytes"); - } - } - } - public int limit() { return limit; } - - /** Returns a read-only list of ranking constants ordered by name */ - public Collection expressions() { - return expressions.values().stream().sorted().collect(Collectors.toUnmodifiableList()); - } - - // Note: Use by integration tests in internal repo - /** Returns a read-only map of the ranking constants in this indexed by name */ - public Map asMap() { - return Collections.unmodifiableMap(expressions); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java deleted file mode 100644 index 23538deed66..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; -import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; -import com.yahoo.searchlib.rankingexpression.rule.NameNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; - -import java.util.ArrayDeque; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.stream.Collectors; - -/** - * A context which only contains type information. - * This returns empty tensor types (double) for unknown features which are not - * query, attribute or constant features, as we do not have information about which such - * features exist (but we know those that exist are doubles). - * - * This is not multithread safe. - * - * @author bratseth - */ -public class MapEvaluationTypeContext extends FunctionReferenceContext implements TypeContext { - - private final Optional parent; - - private final Map featureTypes = new HashMap<>(); - - private final Map resolvedTypes = new HashMap<>(); - - /** To avoid re-resolving diamond-shaped dependencies */ - private final Map globallyResolvedTypes; - - /** For invocation loop detection */ - private final Deque currentResolutionCallStack; - - private final SortedSet queryFeaturesNotDeclared; - private boolean tensorsAreUsed; - - MapEvaluationTypeContext(ImmutableMap functions, Map featureTypes) { - super(functions); - this.parent = Optional.empty(); - this.featureTypes.putAll(featureTypes); - this.currentResolutionCallStack = new ArrayDeque<>(); - this.queryFeaturesNotDeclared = new TreeSet<>(); - tensorsAreUsed = false; - globallyResolvedTypes = new HashMap<>(); - } - - private MapEvaluationTypeContext(Map functions, - Map bindings, - Optional parent, - Map featureTypes, - Deque currentResolutionCallStack, - SortedSet queryFeaturesNotDeclared, - boolean tensorsAreUsed, - Map globallyResolvedTypes) { - super(functions, bindings); - this.parent = parent; - this.featureTypes.putAll(featureTypes); - this.currentResolutionCallStack = currentResolutionCallStack; - this.queryFeaturesNotDeclared = queryFeaturesNotDeclared; - this.tensorsAreUsed = tensorsAreUsed; - this.globallyResolvedTypes = globallyResolvedTypes; - } - - public void setType(Reference reference, TensorType type) { - featureTypes.put(reference, type); - queryFeaturesNotDeclared.remove(reference); - } - - public Map featureTypes() { return Collections.unmodifiableMap(featureTypes); } - - @Override - public TensorType getType(String reference) { - throw new UnsupportedOperationException("Not able to parse general references from string form"); - } - - public void forgetResolvedTypes() { - resolvedTypes.clear(); - } - - private boolean referenceCanBeResolvedGlobally(Reference reference) { - Optional function = functionInvocation(reference); - return function.isPresent() && function.get().arguments().size() == 0; - // are there other cases we would like to resolve globally? - } - - @Override - public TensorType getType(Reference reference) { - // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: - boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); - - TensorType resolvedType = resolvedTypes.get(reference); - if (resolvedType == null && canBeResolvedGlobally) { - resolvedType = globallyResolvedTypes.get(reference); - } - if (resolvedType != null) { - return resolvedType; - } - - resolvedType = resolveType(reference); - if (resolvedType == null) - return defaultTypeOf(reference); // Don't store fallback to default as we may know more later - resolvedTypes.put(reference, resolvedType); - if (resolvedType.rank() > 0) - tensorsAreUsed = true; - - if (canBeResolvedGlobally) { - globallyResolvedTypes.put(reference, resolvedType); - } - - return resolvedType; - } - - MapEvaluationTypeContext getParent(String forArgument, String boundTo) { - return parent.orElseThrow( - () -> new IllegalArgumentException("argument "+forArgument+" is bound to "+boundTo+" but there is no parent context")); - } - - String resolveBinding(String argument) { - String bound = getBinding(argument); - if (bound == null) { - return argument; - } - return getParent(argument, bound).resolveBinding(bound); - } - - private TensorType resolveType(Reference reference) { - if (currentResolutionCallStack.contains(reference)) - throw new IllegalArgumentException("Invocation loop: " + - currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) + - " -> " + reference); - - // Bound to a function argument? - Optional binding = boundIdentifier(reference); - if (binding.isPresent()) { - try { - // This is not pretty, but changing to bind expressions rather - // than their string values requires deeper changes - var expr = new RankingExpression(binding.get()); - var type = expr.type(getParent(reference.name(), binding.get())); - return type; - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - try { - currentResolutionCallStack.addLast(reference); - - // A reference to an attribute, query or constant feature? - if (FeatureNames.isSimpleFeature(reference)) { - // The argument may be a local identifier bound to the actual value - String argument = reference.simpleArgument().get(); - String argumentBinding = resolveBinding(argument); - reference = Reference.simple(reference.name(), argumentBinding); - return featureTypes.get(reference); - } - - // A reference to a function? - Optional function = functionInvocation(reference); - if (function.isPresent()) { - var body = function.get().getBody(); - var child = this.withBindings(bind(function.get().arguments(), reference.arguments())); - var type = body.type(child); - return type; - } - - // A reference to an ONNX model? - Optional onnxFeatureType = onnxFeatureType(reference); - if (onnxFeatureType.isPresent()) { - return onnxFeatureType.get(); - } - - // A reference to a feature for transformer token input? - Optional transformerTokensFeatureType = transformerTokensFeatureType(reference); - if (transformerTokensFeatureType.isPresent()) { - return transformerTokensFeatureType.get(); - } - - // A reference to a feature which returns a tensor? - Optional featureTensorType = tensorFeatureType(reference); - if (featureTensorType.isPresent()) { - return featureTensorType.get(); - } - - // A directly injected identifier? (Useful for stateless model evaluation) - if (reference.isIdentifier() && featureTypes.containsKey(reference)) { - return featureTypes.get(reference); - } - - // the name of a constant feature? - if (reference.isIdentifier()) { - Reference asConst = FeatureNames.asConstantFeature(reference.name()); - if (featureTypes.containsKey(asConst)) { - return featureTypes.get(asConst); - } - } - - // We do not know what this is - since we do not have complete knowledge about the match features - // in Java we must assume this is a match feature and return the double type - which is the type of - // all match features - return TensorType.empty; - } - finally { - currentResolutionCallStack.removeLast(); - } - } - - /** - * Returns the default type for this simple feature, or null if it does not have a default - */ - public TensorType defaultTypeOf(Reference reference) { - if ( ! FeatureNames.isSimpleFeature(reference)) - throw new IllegalArgumentException("This can only be called for simple references, not " + reference); - if (reference.name().equals("query")) { // we do not require all query features to be declared, only non-doubles - queryFeaturesNotDeclared.add(reference); - return TensorType.empty; - } - return null; - } - - /** - * Returns the binding if this reference is a simple identifier which is bound in this context. - * Returns empty otherwise. - */ - private Optional boundIdentifier(Reference reference) { - if ( ! reference.arguments().isEmpty()) return Optional.empty(); - if ( reference.output() != null) return Optional.empty(); - return Optional.ofNullable(getBinding(reference.name())); - } - - private Optional functionInvocation(Reference reference) { - if (reference.output() != null) return Optional.empty(); - ExpressionFunction function = getFunctions().get(reference.name()); - if (function == null) return Optional.empty(); - if (function.arguments().size() != reference.arguments().size()) return Optional.empty(); - return Optional.of(function); - } - - private Optional onnxFeatureType(Reference reference) { - if ( ! reference.name().equals("onnxModel") && ! reference.name().equals("onnx")) - return Optional.empty(); - - if ( ! featureTypes.containsKey(reference)) { - String configOrFileName = reference.arguments().expressions().get(0).toString(); - - // Look up standardized format as added in RankProfile - String modelConfigName = OnnxModelTransformer.getModelConfigName(reference); - String modelOutput = OnnxModelTransformer.getModelOutput(reference, null); - - reference = new Reference("onnxModel", new Arguments(new ReferenceNode(modelConfigName)), modelOutput); - if ( ! featureTypes.containsKey(reference)) { - throw new IllegalArgumentException("Missing onnx-model config for '" + configOrFileName + "'"); - } - } - - return Optional.of(featureTypes.get(reference)); - } - - private Optional transformerTokensFeatureType(Reference reference) { - if ( ! reference.name().equals("tokenTypeIds") && - ! reference.name().equals("tokenInputIds") && - ! reference.name().equals("tokenAttentionMask")) - return Optional.empty(); - - if ( ! (reference.arguments().size() > 1)) - throw new IllegalArgumentException(reference.name() + " must have at least 2 arguments"); - - ExpressionNode size = reference.arguments().expressions().get(0); - return Optional.of(TokenTransformer.createTensorType(reference.name(), size)); - } - - /** - * There are two features which returns the (non-empty) tensor type: tensorFromLabels and tensorFromWeightedSet. - * This returns the type of those features if this is a reference to either of them, or empty otherwise. - */ - private Optional tensorFeatureType(Reference reference) { - if ( ! reference.name().equals("tensorFromLabels") && ! reference.name().equals("tensorFromWeightedSet")) - return Optional.empty(); - - if (reference.arguments().size() != 1 && reference.arguments().size() != 2) - throw new IllegalArgumentException(reference.name() + " must have one or two arguments"); - - ExpressionNode arg0 = reference.arguments().expressions().get(0); - if ( ! ( arg0 instanceof ReferenceNode) || ! FeatureNames.isSimpleFeature(((ReferenceNode)arg0).reference())) - throw new IllegalArgumentException("The first argument of " + reference.name() + - " must be a simple feature, not " + arg0); - - String dimension; - if (reference.arguments().size() > 1) { - ExpressionNode arg1 = reference.arguments().expressions().get(1); - if ( ( ! (arg1 instanceof ReferenceNode) || ! (((ReferenceNode)arg1).reference().isIdentifier())) - && - ( ! (arg1 instanceof NameNode))) - throw new IllegalArgumentException("The second argument of " + reference.name() + - " must be a dimension name, not " + arg1); - dimension = reference.arguments().expressions().get(1).toString(); - } - else { // default - dimension = ((ReferenceNode)arg0).reference().arguments().expressions().get(0).toString(); - } - - // TODO: Determine the type of the weighted set/vector and use that as value type - return Optional.of(new TensorType.Builder().mapped(dimension).build()); - } - - /** Binds the given list of formal arguments to their actual values */ - private Map bind(List formalArguments, - Arguments invocationArguments) { - Map bindings = new HashMap<>(formalArguments.size()); - for (int i = 0; i < formalArguments.size(); i++) { - String identifier = invocationArguments.expressions().get(i).toString(); - bindings.put(formalArguments.get(i), identifier); - } - return bindings; - } - - /** - * Returns an unmodifiable view of the query features which was requested but for which we have no type info - * (such that they default to TensorType.empty), shared between all instances of this - * involved in resolving a particular rank profile. - */ - public SortedSet queryFeaturesNotDeclared() { - return Collections.unmodifiableSortedSet(queryFeaturesNotDeclared); - } - - /** Returns true if any feature across all instances involved in resolving this rank profile resolves to a tensor */ - public boolean tensorsAreUsed() { return tensorsAreUsed; } - - @Override - public MapEvaluationTypeContext withBindings(Map bindings) { - return new MapEvaluationTypeContext(getFunctions(), - bindings, - Optional.of(this), - featureTypes, - currentResolutionCallStack, - queryFeaturesNotDeclared, - tensorsAreUsed, - globallyResolvedTypes); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java deleted file mode 100644 index 4b849af9662..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.model.ml.OnnxModelInfo; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -/** - * A global ONNX model distributed using file distribution, similar to ranking constants. - * - * @author lesters - */ -public class OnnxModel extends DistributableResource { - - private OnnxModelInfo modelInfo = null; - private final Map inputMap = new HashMap<>(); - private final Map outputMap = new HashMap<>(); - - private String statelessExecutionMode = null; - private Integer statelessInterOpThreads = null; - private Integer statelessIntraOpThreads = null; - - public OnnxModel(String name) { - super(name); - } - - public OnnxModel(String name, String fileName) { - super(name, fileName); - validate(); - } - - @Override - public void setUri(String uri) { - throw new IllegalArgumentException("URI for ONNX models are not currently supported"); - } - - public void addInputNameMapping(String onnxName, String vespaName) { - addInputNameMapping(onnxName, vespaName, true); - } - - public void addInputNameMapping(String onnxName, String vespaName, boolean overwrite) { - Objects.requireNonNull(onnxName, "Onnx name cannot be null"); - Objects.requireNonNull(vespaName, "Vespa name cannot be null"); - if (overwrite || ! inputMap.containsKey(onnxName)) { - inputMap.put(onnxName, vespaName); - } - } - - public void addOutputNameMapping(String onnxName, String vespaName) { - addOutputNameMapping(onnxName, vespaName, true); - } - - public void addOutputNameMapping(String onnxName, String vespaName, boolean overwrite) { - Objects.requireNonNull(onnxName, "Onnx name cannot be null"); - Objects.requireNonNull(vespaName, "Vespa name cannot be null"); - if (overwrite || ! outputMap.containsKey(onnxName)) { - outputMap.put(onnxName, vespaName); - } - } - - public void setModelInfo(OnnxModelInfo modelInfo) { - Objects.requireNonNull(modelInfo, "Onnx model info cannot be null"); - for (String onnxName : modelInfo.getInputs()) { - addInputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false); - } - for (String onnxName : modelInfo.getOutputs()) { - addOutputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false); - } - this.modelInfo = modelInfo; - } - - public Map getInputMap() { return Collections.unmodifiableMap(inputMap); } - public Map getOutputMap() { return Collections.unmodifiableMap(outputMap); } - - public String getDefaultOutput() { - return modelInfo != null ? modelInfo.getDefaultOutput() : ""; - } - - TensorType getTensorType(String onnxName, Map inputTypes) { - return modelInfo != null ? modelInfo.getTensorType(onnxName, inputTypes) : TensorType.empty; - } - - public void setStatelessExecutionMode(String executionMode) { - if ("parallel".equalsIgnoreCase(executionMode)) { - this.statelessExecutionMode = "parallel"; - } else if ("sequential".equalsIgnoreCase(executionMode)) { - this.statelessExecutionMode = "sequential"; - } - } - - public Optional getStatelessExecutionMode() { - return Optional.ofNullable(statelessExecutionMode); - } - - public void setStatelessInterOpThreads(int interOpThreads) { - if (interOpThreads >= 0) { - this.statelessInterOpThreads = interOpThreads; - } - } - - public Optional getStatelessInterOpThreads() { - return Optional.ofNullable(statelessInterOpThreads); - } - - public void setStatelessIntraOpThreads(int intraOpThreads) { - if (intraOpThreads >= 0) { - this.statelessIntraOpThreads = intraOpThreads; - } - } - - public Optional getStatelessIntraOpThreads() { - return Optional.ofNullable(statelessIntraOpThreads); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java deleted file mode 100644 index 90109026dce..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.FileRegistry; - -import java.nio.ByteBuffer; - -import static java.util.Objects.requireNonNull; - -public class RankExpressionBody extends DistributableResource { - - private final ByteBuffer blob; - - public RankExpressionBody(String name, ByteBuffer blob) { - super(name, name + ".lz4", PathType.BLOB); - this.blob = requireNonNull(blob, "Blob cannot be null"); - } - - public ByteBuffer getBlob() { return blob; } - - public void validate() { - // Remove once pathType is final - if (getPathType() != PathType.BLOB) { - throw new IllegalArgumentException("PathType must be BLOB."); - } - } - - public void register(FileRegistry fileRegistry) { - register(fileRegistry, blob); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java deleted file mode 100644 index ec560484513..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ /dev/null @@ -1,1526 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.google.common.collect.ImmutableMap; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.path.Path; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.expressiontransforms.ExpressionTransforms; -import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.FeatureList; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.io.IOException; -import java.io.Reader; -import java.io.Serializable; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Represents a rank profile - a named set of ranking settings - * - * @author bratseth - */ -public class RankProfile implements Cloneable { - - public final static String FIRST_PHASE = "firstphase"; - public final static String SECOND_PHASE = "secondphase"; - - /** The schema-unique name of this rank profile */ - private final String name; - - /** The schema owning this profile, or null if global (owned by a model) */ - private final ImmutableSchema schema; - - private final List inheritedNames = new ArrayList<>(); - - /** The resolved inherited profiles, or null when not resolved. */ - private List inherited; - - private MatchPhaseSettings matchPhaseSettings = null; - - protected Set rankSettings = new java.util.LinkedHashSet<>(); - - /** The ranking expression to be used for first phase */ - private RankingExpressionFunction firstPhaseRanking = null; - - /** The ranking expression to be used for second phase */ - private RankingExpressionFunction secondPhaseRanking = null; - - /** Number of hits to be reranked in second phase, -1 means use default */ - private int rerankCount = -1; - - /** Mysterious attribute */ - private int keepRankCount = -1; - - private int numThreadsPerSearch = -1; - private int minHitsPerThread = -1; - private int numSearchPartitions = -1; - - private Double termwiseLimit = null; - private Double postFilterThreshold = null; - private Double approximateThreshold = null; - - /** The drop limit used to drop hits with rank score less than or equal to this value */ - private double rankScoreDropLimit = -Double.MAX_VALUE; - - private Set summaryFeatures; - private String inheritedSummaryFeaturesProfileName; - - private Set matchFeatures; - private String inheritedMatchFeaturesProfileName; - - private Set rankFeatures; - - /** The properties of this - a multimap */ - private Map> rankProperties = new LinkedHashMap<>(); - - private Boolean ignoreDefaultRankFeatures = null; - - private Map functions = new LinkedHashMap<>(); - // This cache must be invalidated every time modifications are done to 'functions'. - private CachedFunctions allFunctionsCached = null; - - private Map inputs = new LinkedHashMap<>(); - - private Map constants = new LinkedHashMap<>(); - - private Map onnxModels = new LinkedHashMap<>(); - - private Set filterFields = new HashSet<>(); - - private final RankProfileRegistry rankProfileRegistry; - - private final TypeSettings attributeTypes = new TypeSettings(); - - private List allFieldsList; - - private Boolean strict; - - private final ApplicationPackage applicationPackage; - private final DeployLogger deployLogger; - - /** - * Creates a new rank profile for a particular schema - * - * @param name the name of the new profile - * @param schema the schema owning this profile - * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing - * and looking up rank profiles. - */ - public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { - this.name = Objects.requireNonNull(name, "name cannot be null"); - this.schema = Objects.requireNonNull(schema, "schema cannot be null"); - this.rankProfileRegistry = rankProfileRegistry; - this.applicationPackage = schema.applicationPackage(); - this.deployLogger = schema.getDeployLogger(); - } - - /** - * Creates a global rank profile - * - * @param name the name of the new profile - */ - public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry) { - this.name = Objects.requireNonNull(name, "name cannot be null"); - this.schema = null; - this.rankProfileRegistry = rankProfileRegistry; - this.applicationPackage = applicationPackage; - this.deployLogger = deployLogger; - } - - public String name() { return name; } - - /** Returns the search definition owning this, or null if it is global */ - public ImmutableSchema schema() { return schema; } - - /** Returns the application this is part of */ - public ApplicationPackage applicationPackage() { - return applicationPackage; - } - - private Stream allFields() { - if (schema == null) return Stream.empty(); - if (allFieldsList == null) { - allFieldsList = schema.allFieldsList(); - } - return allFieldsList.stream(); - } - - private Stream allImportedFields() { - return schema != null ? schema.allImportedFields() : Stream.empty(); - } - - /** - * Returns whether type checking should fail if this profile accesses query features that are - * not defined in query profile types. - * - * Default is false. - */ - public boolean isStrict() { - Boolean declaredStrict = declaredStrict(); - if (declaredStrict != null) return declaredStrict; - return false; - } - - /** Returns the strict value declared in this or any parent profile. */ - public Boolean declaredStrict() { - if (strict != null) return strict; - return uniquelyInherited(p -> p.declaredStrict(), "strict").orElse(null); - } - - public void setStrict(Boolean strict) { - this.strict = strict; - } - - /** - * Adds a profile to those inherited by this. - * The profile must belong to this schema (directly or by inheritance). - */ - public void inherit(String inheritedName) { - inherited = null; - inheritedNames.add(inheritedName); - } - - /** Returns the names of the profiles this inherits, if any. */ - public List inheritedNames() { return Collections.unmodifiableList(inheritedNames); } - - /** Returns the rank profiles inherited by this. */ - private List inherited() { - if (inheritedNames.isEmpty()) return List.of(); - if (inherited != null) return inherited; - - inherited = resolveInheritedProfiles(schema); - List children = new ArrayList<>(); - children.add(createFullyQualifiedName()); - inherited.forEach(profile -> verifyNoInheritanceCycle(children, profile)); - return inherited; - } - - private String createFullyQualifiedName() { - return (schema != null) - ? (schema.getName() + "." + name()) - : name(); - } - - private void verifyNoInheritanceCycle(List children, RankProfile parent) { - children.add(parent.createFullyQualifiedName()); - String root = children.get(0); - if (root.equals(parent.createFullyQualifiedName())) - throw new IllegalArgumentException("There is a cycle in the inheritance for rank-profile '" + root + "' = " + children); - for (RankProfile parentInherited : parent.inherited()) - verifyNoInheritanceCycle(children, parentInherited); - } - - private List resolveInheritedProfiles(ImmutableSchema schema) { - List inherited = new ArrayList<>(); - for (String inheritedName : inheritedNames) { - RankProfile inheritedProfile = schema == null - ? rankProfileRegistry.getGlobal(inheritedName) - : resolveInheritedProfile(schema, inheritedName); - if (inheritedProfile == null) - throw new IllegalArgumentException("rank-profile '" + name() + "' inherits '" + inheritedName + - "', but this is not found in " + - ((schema() != null) ? schema() : " global rank profiles")); - inherited.add(inheritedProfile); - } - return inherited; - } - - private RankProfile resolveInheritedProfile(ImmutableSchema schema, String inheritedName) { - SDDocumentType documentType = schema.getDocument(); - if (documentType != null) { - if (name.equals(inheritedName)) { - // If you seemingly inherit yourself, you are actually referencing a rank-profile in one of your inherited schemas - for (SDDocumentType baseType : documentType.getInheritedTypes()) { - RankProfile resolvedFromBase = rankProfileRegistry.resolve(baseType, inheritedName); - if (resolvedFromBase != null) return resolvedFromBase; - } - } - return rankProfileRegistry.resolve(documentType, inheritedName); - } - return rankProfileRegistry.get(schema.getName(), inheritedName); - } - - /** Returns whether this profile inherits (directly or indirectly) the given profile name. */ - public boolean inherits(String name) { - for (RankProfile inheritedProfile : inherited()) { - if (inheritedProfile.name().equals(name)) return true; - if (inheritedProfile.inherits(name)) return true; - } - return false; - } - - public void setMatchPhaseSettings(MatchPhaseSettings settings) { - settings.checkValid(); - this.matchPhaseSettings = settings; - } - - public MatchPhaseSettings getMatchPhaseSettings() { - if (matchPhaseSettings != null) return matchPhaseSettings; - return uniquelyInherited(p -> p.getMatchPhaseSettings(), "match phase settings").orElse(null); - } - - /** Returns the uniquely determined property, where non-empty is defined as non-null */ - private Optional uniquelyInherited(Function propertyRetriever, - String propertyDescription) { - return uniquelyInherited(propertyRetriever, p -> p != null, propertyDescription); - } - - /** - * Returns the property retrieved by the given function, if it is only present in a single unique variant - * among all profiled inherited by this, or empty if not present. - * Note that for properties that don't implement a values-based equals this reverts to the stricter condition that - * only one inherited profile can define a non-empty value at all. - * - * @throws IllegalArgumentException if the inherited profiles defines multiple different values of the property - */ - private Optional uniquelyInherited(Function propertyRetriever, - Predicate nonEmptyValueFilter, - String propertyDescription) { - Set uniqueProperties = inherited().stream() - .map(p -> propertyRetriever.apply(p)) - .filter(p -> nonEmptyValueFilter.test(p)) - .collect(Collectors.toSet()); - if (uniqueProperties.isEmpty()) return Optional.empty(); - if (uniqueProperties.size() == 1) return Optional.of(uniqueProperties.stream().findAny().get()); - throw new IllegalArgumentException("Only one of the profiles inherited by " + this + " can contain " + - propertyDescription + ", but it is present in multiple"); - } - - public void addRankSetting(RankSetting rankSetting) { - rankSettings.add(rankSetting); - } - - public void addRankSetting(String fieldName, RankSetting.Type type, Object value) { - addRankSetting(new RankSetting(fieldName, type, value)); - } - - /** - * Returns the a rank setting of a field, or null if there is no such rank setting in this profile - * - * @param field the field whose settings to return - * @param type the type that the field is required to be - * @return the rank setting found, or null - */ - RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { - for (Iterator i = declaredRankSettingIterator(); i.hasNext(); ) { - RankSetting setting = i.next(); - if (setting.getFieldName().equals(field) && setting.getType() == type) { - return setting; - } - } - return null; - } - - /** - * Returns a rank setting of field or index, or null if there is no such rank setting in this profile or one it - * inherits - * - * @param field the field whose settings to return - * @param type the type that the field is required to be - * @return the rank setting found, or null - */ - public RankSetting getRankSetting(String field, RankSetting.Type type) { - RankSetting rankSetting = getDeclaredRankSetting(field, type); - if (rankSetting != null) return rankSetting; - - return uniquelyInherited(p -> p.getRankSetting(field, type), "rank setting " + type + " on " + field).orElse(null); - } - - /** - * Returns the rank settings in this rank profile - * - * @return an iterator for the declared rank setting - */ - public Iterator declaredRankSettingIterator() { - return Collections.unmodifiableSet(rankSettings).iterator(); - } - - /** - * Returns all settings in this profile or any profile it inherits - * - * @return an iterator for all rank settings of this - */ - public Iterator rankSettingIterator() { - return rankSettings().iterator(); - } - - /** - * Returns a snapshot of the rank settings of this and everything it inherits. - * Changes to the returned set will not be reflected in this rank profile. - */ - public Set rankSettings() { - Set settings = new LinkedHashSet<>(); - for (RankProfile inheritedProfile : inherited()) { - for (RankSetting setting : inheritedProfile.rankSettings()) { - if (settings.contains(setting)) - throw new IllegalArgumentException(setting + " is present in " + inheritedProfile + " inherited by " + - this + ", but is also present in another profile inherited by it"); - settings.add(setting); - } - } - - // TODO: Here we do things in the wrong order to not break tests. Reverse this. - Set finalSettings = new LinkedHashSet<>(rankSettings); - finalSettings.addAll(settings); - return finalSettings; - } - - public void add(Constant constant) { - constants.put(constant.name(), constant); - } - - /** Returns an unmodifiable view of the constants declared in this */ - public Map declaredConstants() { return Collections.unmodifiableMap(constants); } - - /** Returns an unmodifiable view of the constants available in this */ - public Map constants() { - Map allConstants = new HashMap<>(); - for (var inheritedProfile : inherited()) { - for (var constant : inheritedProfile.constants().values()) { - if (allConstants.containsKey(constant.name())) - throw new IllegalArgumentException(constant + "' is present in " + - inheritedProfile + " inherited by " + - this + ", but is also present in another profile inherited by it"); - allConstants.put(constant.name(), constant); - } - } - - if (schema != null) - allConstants.putAll(schema.constants()); - allConstants.putAll(constants); - return allConstants; - } - - public void add(OnnxModel model) { - onnxModels.put(model.getName(), model); - } - - /** Returns an unmodifiable map of the onnx models declared in this. */ - public Map declaredOnnxModels() { return onnxModels; } - - /** Returns an unmodifiable map of the onnx models available in this. */ - public Map onnxModels() { - Map allModels = new HashMap<>(); - for (var inheritedProfile : inherited()) { - for (var model : inheritedProfile.onnxModels().values()) { - if (allModels.containsKey(model.getName())) - throw new IllegalArgumentException(model + "' is present in " + - inheritedProfile + " inherited by " + - this + ", but is also present in another profile inherited by it"); - allModels.put(model.getName(), model); - } - } - - if (schema != null) - allModels.putAll(schema.onnxModels()); - allModels.putAll(onnxModels); - return allModels; - } - - public void addAttributeType(String attributeName, String attributeType) { - attributeTypes.addType(attributeName, attributeType); - } - - public Map getAttributeTypes() { - return attributeTypes.getTypes(); - } - - /** - * Returns the ranking expression to use by this. This expression must not be edited. - * Returns null if no expression is set. - */ - public RankingExpression getFirstPhaseRanking() { - RankingExpressionFunction function = getFirstPhase(); - if (function == null) return null; - return function.function.getBody(); - } - - public RankingExpressionFunction getFirstPhase() { - if (firstPhaseRanking != null) return firstPhaseRanking; - return uniquelyInherited(p -> p.getFirstPhase(), "first-phase expression").orElse(null); - } - - void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); - } - - public void setFirstPhaseRanking(String expression) { - try { - firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); - } catch (ParseException e) { - throw new IllegalArgumentException("Illegal first phase ranking function", e); - } - } - - /** - * Returns the ranking expression to use by this. This expression must not be edited. - * Returns null if no expression is set. - */ - public RankingExpression getSecondPhaseRanking() { - RankingExpressionFunction function = getSecondPhase(); - if (function == null) return null; - return function.function().getBody(); - } - - public RankingExpressionFunction getSecondPhase() { - if (secondPhaseRanking != null) return secondPhaseRanking; - return uniquelyInherited(p -> p.getSecondPhase(), "second-phase expression").orElse(null); - } - - public void setSecondPhaseRanking(String expression) { - try { - secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); - } - catch (ParseException e) { - throw new IllegalArgumentException("Illegal second phase ranking function", e); - } - } - - // TODO: Below we have duplicate methods for summary and match features: Encapsulate this in a single parametrized - // class instead (and probably make rank features work the same). - - /** - * Sets the name this should inherit the summary features of. - * Without setting this, this will either have the summary features of the single parent setting them, - * or if summary features are set in this, only have the summary features in this. - * With this set the resulting summary features of this will be the superset of those defined in this and - * the final (with inheritance included) summary features of the given parent. - * The profile must be one which is directly inherited by this. - */ - public void setInheritedSummaryFeatures(String parentProfile) { - if ( ! inheritedNames().contains(parentProfile)) - throw new IllegalArgumentException("This can only inherit the summary features of a directly inherited profile, '" + - ", but attempting to inherit '" + parentProfile); - this.inheritedSummaryFeaturesProfileName = parentProfile; - } - - /** - * Sets the name of a profile this should inherit the match features of. - * Without setting this, this will either have the match features of the single parent setting them, - * or if match features are set in this, only have the match features in this. - * With this set the resulting match features of this will be the superset of those defined in this and - * the final (with inheritance included) match features of the given parent. - * The profile must be one which which is directly inherited by this. - * - */ - public void setInheritedMatchFeatures(String parentProfile) { - if ( ! inheritedNames().contains(parentProfile)) - throw new IllegalArgumentException("This can only inherit the match features of a directly inherited profile, '" + - ", but attempting to inherit '" + parentProfile); - this.inheritedMatchFeaturesProfileName = parentProfile; - } - - /** Returns a read-only view of the summary features to use in this profile. This is never null */ - public Set getSummaryFeatures() { - if (inheritedSummaryFeaturesProfileName != null && summaryFeatures != null) { - Set combined = new HashSet<>(); - RankProfile inherited = inherited().stream() - .filter(p -> p.name().equals(inheritedSummaryFeaturesProfileName)) - .findAny() - .orElseThrow(); - combined.addAll(inherited.getSummaryFeatures()); - combined.addAll(summaryFeatures); - return Collections.unmodifiableSet(combined); - } - if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - return uniquelyInherited(p -> p.getSummaryFeatures(), f -> ! f.isEmpty(), "summary features") - .orElse(Set.of()); - } - - /** Returns a read-only view of the match features to use in this profile. This is never null */ - public Set getMatchFeatures() { - if (inheritedMatchFeaturesProfileName != null && matchFeatures != null) { - Set combined = new HashSet<>(); - RankProfile inherited = inherited().stream() - .filter(p -> p.name().equals(inheritedMatchFeaturesProfileName)) - .findAny() - .orElseThrow(); - combined.addAll(inherited.getMatchFeatures()); - combined.addAll(matchFeatures); - return Collections.unmodifiableSet(combined); - } - if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - return uniquelyInherited(p -> p.getMatchFeatures(), f -> ! f.isEmpty(), "match features") - .orElse(Set.of()); - } - - private void addSummaryFeature(ReferenceNode feature) { - if (summaryFeatures == null) - summaryFeatures = new LinkedHashSet<>(); - summaryFeatures.add(feature); - } - - private void addMatchFeature(ReferenceNode feature) { - if (matchFeatures == null) - matchFeatures = new LinkedHashSet<>(); - matchFeatures.add(feature); - } - - /** Adds the content of the given feature list to the internal list of summary features. */ - public void addSummaryFeatures(FeatureList features) { - for (ReferenceNode feature : features) { - addSummaryFeature(feature); - } - } - - /** Adds the content of the given feature list to the internal list of match features. */ - public void addMatchFeatures(FeatureList features) { - for (ReferenceNode feature : features) { - addMatchFeature(feature); - } - } - - /** Returns a read-only view of the rank features to use in this profile. This is never null */ - public Set getRankFeatures() { - if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - return uniquelyInherited(p -> p.getRankFeatures(), f -> ! f.isEmpty(), "summary-features") - .orElse(Set.of()); - } - - private void addRankFeature(ReferenceNode feature) { - if (rankFeatures == null) - rankFeatures = new LinkedHashSet<>(); - rankFeatures.add(feature); - } - - /** - * Adds the content of the given feature list to the internal list of rank features. - * - * @param features The features to add. - */ - public void addRankFeatures(FeatureList features) { - for (ReferenceNode feature : features) { - addRankFeature(feature); - } - } - - /** Returns a read only flattened list view of the rank properties to use in this profile. This is never null. */ - public List getRankProperties() { - List properties = new ArrayList<>(); - for (List propertyList : getRankPropertyMap().values()) { - properties.addAll(propertyList); - } - return Collections.unmodifiableList(properties); - } - - /** Returns a read only map view of the rank properties to use in this profile. This is never null. */ - public Map> getRankPropertyMap() { - if (rankProperties.size() == 0 && inherited().isEmpty()) return Map.of(); - if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); - - var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") - .orElse(Map.of()); - if (rankProperties.isEmpty()) return inheritedProperties; - - // Neither is null - Map> combined = new LinkedHashMap<>(inheritedProperties); - combined.putAll(rankProperties); // Don't combine values across inherited properties - return Collections.unmodifiableMap(combined); - } - - public void addRankProperty(String name, String parameter) { - addRankProperty(new RankProperty(name, parameter)); - } - - private void addRankProperty(RankProperty rankProperty) { - // Just the usual multimap semantics here - rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty); - } - - public void setRerankCount(int rerankCount) { this.rerankCount = rerankCount; } - - public int getRerankCount() { - if (rerankCount >= 0) return rerankCount; - return uniquelyInherited(p -> p.getRerankCount(), c -> c >= 0, "rerank-count").orElse(-1); - } - - public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } - - public int getNumThreadsPerSearch() { - if (numThreadsPerSearch >= 0) return numThreadsPerSearch; - return uniquelyInherited(p -> p.getNumThreadsPerSearch(), n -> n >= 0, "num-threads-per-search") - .orElse(-1); - } - - public void setMinHitsPerThread(int minHits) { this.minHitsPerThread = minHits; } - - public int getMinHitsPerThread() { - if (minHitsPerThread >= 0) return minHitsPerThread; - return uniquelyInherited(p -> p.getMinHitsPerThread(), n -> n >= 0, "min-hits-per-search").orElse(-1); - } - - public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } - - public int getNumSearchPartitions() { - if (numSearchPartitions >= 0) return numSearchPartitions; - return uniquelyInherited(p -> p.getNumSearchPartitions(), n -> n >= 0, "num-search-partitions").orElse(-1); - } - - public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } - public void setPostFilterThreshold(double threshold) { this.postFilterThreshold = threshold; } - public void setApproximateThreshold(double threshold) { this.approximateThreshold = threshold; } - - public OptionalDouble getTermwiseLimit() { - if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); - return uniquelyInherited(p -> p.getTermwiseLimit(), l -> l.isPresent(), "termwise-limit") - .orElse(OptionalDouble.empty()); - } - - public OptionalDouble getPostFilterThreshold() { - if (postFilterThreshold != null) { - return OptionalDouble.of(postFilterThreshold); - } - return uniquelyInherited(p -> p.getPostFilterThreshold(), l -> l.isPresent(), "post-filter-threshold").orElse(OptionalDouble.empty()); - } - - public OptionalDouble getApproximateThreshold() { - if (approximateThreshold != null) { - return OptionalDouble.of(approximateThreshold); - } - return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); - } - - /** Whether we should ignore the default rank features. Set to null to use inherited */ - public void setIgnoreDefaultRankFeatures(Boolean ignoreDefaultRankFeatures) { - this.ignoreDefaultRankFeatures = ignoreDefaultRankFeatures; - } - - public Boolean getIgnoreDefaultRankFeatures() { - if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return uniquelyInherited(p -> p.getIgnoreDefaultRankFeatures(), "ignore-default-rank-features").orElse(false); - } - - public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } - - public int getKeepRankCount() { - if (keepRankCount >= 0) return keepRankCount; - return uniquelyInherited(p -> p.getKeepRankCount(), c -> c >= 0, "keep-rank-count").orElse(-1); - } - - public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } - - public double getRankScoreDropLimit() { - if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; - return uniquelyInherited(p -> p.getRankScoreDropLimit(), c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") - .orElse(rankScoreDropLimit); - } - - public void addFunction(String name, List arguments, String expression, boolean inline) { - try { - addFunction(parseRankingExpression(name, arguments, expression), inline); - } - catch (ParseException e) { - throw new IllegalArgumentException("Could not parse function '" + name + "'", e); - } - } - - /** Adds a function and returns it */ - public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { - RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); - if (functions.containsKey(function.getName())) { - deployLogger.log(Level.WARNING, "Function '" + function.getName() + "' is defined twice " + - "in rank profile '" + this.name + "'"); - } - functions.put(function.getName(), rankingExpressionFunction); - allFunctionsCached = null; - return rankingExpressionFunction; - } - - /** - * Adds the type of an input feature consumed by this profile. - * All inputs must either be declared through this or in query profile types, - * otherwise they are assumes to be scalars. - */ - public void addInput(Reference reference, Input input) { - if (inputs.containsKey(reference)) { - Input existing = inputs().get(reference); - if (! input.equals(existing)) - throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); - } - inputs.put(reference, input); - } - - /** Returns the inputs of this, which also includes all inputs of the parents of this. */ - // This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles - // because inputs are tied closer to functions than the profile itself. - public Map inputs() { - if (inputs.isEmpty() && inherited().isEmpty()) return Map.of(); - if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs); - - // Combine - Map allInputs = new LinkedHashMap<>(); - for (var inheritedProfile : inherited()) { - for (var input : inheritedProfile.inputs().entrySet()) { - Input existing = allInputs.get(input.getKey()); - if (existing != null && ! existing.equals(input.getValue())) - throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + - input.getValue() + ", but this input is already defined as " + - existing + " in another profile this inherits"); - allInputs.put(input.getKey(), input.getValue()); - } - } - allInputs.putAll(inputs); - return Collections.unmodifiableMap(allInputs); - } - - public static class MutateOperation { - public enum Phase { on_match, on_first_phase, on_second_phase, on_summary} - final Phase phase; - final String attribute; - final String operation; - public MutateOperation(Phase phase, String attribute, String operation) { - this.phase = phase; - this.attribute = attribute; - this.operation = operation; - } - } - private final List mutateOperations = new ArrayList<>(); - - public void addMutateOperation(MutateOperation op) { - mutateOperations.add(op); - String prefix = "vespa.mutate." + op.phase.toString(); - addRankProperty(prefix + ".attribute", op.attribute); - addRankProperty(prefix + ".operation", op.operation); - } - public void addMutateOperation(MutateOperation.Phase phase, String attribute, String operation) { - addMutateOperation(new MutateOperation(phase, attribute, operation)); - } - public List getMutateOperations() { return mutateOperations; } - - public RankingExpressionFunction findFunction(String name) { - RankingExpressionFunction function = functions.get(name); - if (function != null) return function; - return uniquelyInherited(p -> p.findFunction(name), "function '" + name + "'").orElse(null); - } - - /** Returns an unmodifiable snapshot of the functions in this */ - public Map getFunctions() { - updateCachedFunctions(); - return allFunctionsCached.allRankingExpressionFunctions; - } - private ImmutableMap getExpressionFunctions() { - updateCachedFunctions(); - return allFunctionsCached.allExpressionFunctions; - } - private void updateCachedFunctions() { - if (needToUpdateFunctionCache()) { - allFunctionsCached = new CachedFunctions(gatherAllFunctions()); - } - } - - private Map gatherAllFunctions() { - if (functions.isEmpty() && inherited().isEmpty()) return Map.of(); - if (inherited().isEmpty()) return Collections.unmodifiableMap(new LinkedHashMap<>(functions)); - - // Combine - Map allFunctions = new LinkedHashMap<>(); - for (var inheritedProfile : inherited()) { - for (var function : inheritedProfile.getFunctions().entrySet()) { - if (allFunctions.containsKey(function.getKey())) - throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + - function.getValue() + ", but this function is already " + - "defined in another profile this inherits"); - allFunctions.put(function.getKey(), function.getValue()); - } - } - allFunctions.putAll(functions); - return Collections.unmodifiableMap(allFunctions); - } - - private boolean needToUpdateFunctionCache() { - if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; - return allFunctionsCached == null; - } - - public Set filterFields() { return filterFields; } - - /** Returns all filter fields in this profile and any profile it inherits. */ - public Set allFilterFields() { - Set inheritedFilterFields = uniquelyInherited(p -> p.allFilterFields(), fields -> ! fields.isEmpty(), - "filter fields").orElse(Set.of()); - - if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); - - Set combined = new LinkedHashSet<>(inheritedFilterFields); - combined.addAll(filterFields()); - return combined; - } - - private ExpressionFunction parseRankingExpression(String name, List arguments, String expression) throws ParseException { - if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); - - try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { - return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); - } - catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { - ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + name() + ", " + name + "."); - throw (ParseException)exception.initCause(e); - } - catch (IOException e) { - throw new RuntimeException("IOException parsing ranking expression '" + name + "'", e); - } - } - - private static String extractFileName(String expression) { - String fileName = expression.substring("file:".length()).trim(); - if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX)) - fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX; - - return fileName; - } - - private Reader openRankingExpressionReader(String expName, String expression) { - if (!expression.startsWith("file:")) return new StringReader(expression); - - String fileName = extractFileName(expression); - Path.fromString(fileName); // No ".." - if (fileName.contains("/")) // See ticket 4102122 - throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + - fileName + "' in a different directory, which is not supported."); - - return schema.getRankingExpression(fileName); - } - - /** Shallow clones this */ - @Override - public RankProfile clone() { - try { - RankProfile clone = (RankProfile)super.clone(); - clone.rankSettings = new LinkedHashSet<>(this.rankSettings); - clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? - clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; - clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null; - clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; - clone.rankProperties = new LinkedHashMap<>(this.rankProperties); - clone.inputs = new LinkedHashMap<>(this.inputs); - clone.functions = new LinkedHashMap<>(this.functions); - clone.allFunctionsCached = null; - clone.filterFields = new HashSet<>(this.filterFields); - clone.constants = new HashMap<>(this.constants); - return clone; - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Won't happen", e); - } - } - - /** - * Returns a copy of this where the content is optimized for execution. - * Compiled profiles should never be modified. - */ - public RankProfile compile(QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { - try { - RankProfile compiled = this.clone(); - compiled.compileThis(queryProfiles, importedModels); - return compiled; - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Rank profile '" + name() + "' is invalid", e); - } - } - - private void compileThis(QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { - checkNameCollisions(getFunctions(), constants()); - ExpressionTransforms expressionTransforms = new ExpressionTransforms(); - - Map featureTypes = featureTypes(); - // Function compiling first pass: compile inline functions without resolving other functions - Map inlineFunctions = - compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); - - firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); - - // Function compiling second pass: compile all functions and insert previously compiled inline functions - // TODO: This merges all functions from inherited profiles too and erases inheritance information. Not good. - functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - allFunctionsCached = null; - } - - private void checkNameCollisions(Map functions, Map constants) { - for (var functionEntry : functions.entrySet()) { - if (constants.containsKey(FeatureNames.asConstantFeature(functionEntry.getKey()))) - throw new IllegalArgumentException("Cannot have both a constant and function named '" + - functionEntry.getKey() + "'"); - } - } - - private Map getInlineFunctions() { - return getFunctions().entrySet().stream().filter(x -> x.getValue().inline()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private Map compileFunctions(Supplier> functions, - QueryProfileRegistry queryProfiles, - Map featureTypes, - ImportedMlModels importedModels, - Map inlineFunctions, - ExpressionTransforms expressionTransforms) { - Map compiledFunctions = new LinkedHashMap<>(); - Map.Entry entry; - // Compile all functions. Why iterate in such a complicated way? - // Because some functions (imported models adding generated functions) may add other functions during compiling. - // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException - while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { - RankingExpressionFunction rankingExpressionFunction = entry.getValue(); - RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, - importedModels, constants(), inlineFunctions, - expressionTransforms); - compiledFunctions.put(entry.getKey(), compiled); - } - return compiledFunctions; - } - - private static Map.Entry findUncompiledFunction(Map functions, - Set compiledFunctionNames) { - for (Map.Entry entry : functions.entrySet()) { - if ( ! compiledFunctionNames.contains(entry.getKey())) - return entry; - } - return null; - } - - private RankingExpressionFunction compile(RankingExpressionFunction function, - QueryProfileRegistry queryProfiles, - Map featureTypes, - ImportedMlModels importedModels, - Map constants, - Map inlineFunctions, - ExpressionTransforms expressionTransforms) { - if (function == null) return null; - - RankProfileTransformContext context = new RankProfileTransformContext(this, - queryProfiles, - featureTypes, - importedModels, - constants, - inlineFunctions); - RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context); - for (Map.Entry rankProperty : context.rankProperties().entrySet()) { - addRankProperty(rankProperty.getKey(), rankProperty.getValue()); - } - return function.withExpression(expression); - } - - /** - * Creates a context containing the type information of all constants, attributes and query profiles - * referable from this rank profile. - */ - public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles) { - return typeContext(queryProfiles, featureTypes()); - } - - public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); } - - private Map featureTypes() { - Map featureTypes = inputs().values().stream() - .collect(Collectors.toMap(input -> input.name(), input -> input.type())); - allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); - allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); - return featureTypes; - } - - public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, - Map featureTypes) { - MapEvaluationTypeContext context = new MapEvaluationTypeContext(getExpressionFunctions(), featureTypes); - - constants().forEach((k, v) -> context.setType(k, v.type())); - - // Add query features from all rank profile types - for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { - for (FieldDescription field : queryProfileType.declaredFields().values()) { - TensorType type = field.getType().asTensorType(); - Optional feature = Reference.simple(field.getName()); - if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; - if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides - - TensorType existingType = context.getType(feature.get()); - if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) - type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> - new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + - " with type " + field.getType().asTensorType() + - ", but this is already defined in another query profile with type " + - context.getType(feature.get()))); - context.setType(feature.get(), type); - } - } - - // Add output types for ONNX models - for (var model : onnxModels().values()) { - Arguments args = new Arguments(new ReferenceNode(model.getName())); - Map inputTypes = resolveOnnxInputTypes(model, context); - - TensorType defaultOutputType = model.getTensorType(model.getDefaultOutput(), inputTypes); - context.setType(new Reference("onnxModel", args, null), defaultOutputType); - - for (Map.Entry mapping : model.getOutputMap().entrySet()) { - TensorType type = model.getTensorType(mapping.getKey(), inputTypes); - context.setType(new Reference("onnxModel", args, mapping.getValue()), type); - } - } - return context; - } - - private Map resolveOnnxInputTypes(OnnxModel model, MapEvaluationTypeContext context) { - Map inputTypes = new HashMap<>(); - for (String onnxInputName : model.getInputMap().keySet()) { - resolveOnnxInputType(onnxInputName, model, context).ifPresent(type -> inputTypes.put(onnxInputName, type)); - } - return inputTypes; - } - - private Optional resolveOnnxInputType(String onnxInputName, OnnxModel model, MapEvaluationTypeContext context) { - String source = model.getInputMap().get(onnxInputName); - if (source != null) { - // Source is either a simple reference (query/attribute/constant/rankingExpression)... - Optional reference = Reference.simple(source); - if (reference.isPresent()) { - if (reference.get().name().equals("rankingExpression") && reference.get().simpleArgument().isPresent()) { - source = reference.get().simpleArgument().get(); // look up function below - } else { - return Optional.of(context.getType(reference.get())); - } - } - // ... or a function - ExpressionFunction func = context.getFunction(source); - if (func != null) { - return Optional.of(func.getBody().type(context)); - } - } - return Optional.empty(); // if this context does not contain this input - } - - private void addAttributeFeatureTypes(ImmutableSDField field, Map featureTypes) { - Attribute attribute = field.getAttribute(); - field.getAttributes().forEach((k, a) -> { - String name = k; - if (attribute == a) // this attribute should take the fields name - name = field.getName(); // switch to that - it is separate for imported fields - featureTypes.put(FeatureNames.asAttributeFeature(name), - a.tensorType().orElse(TensorType.empty)); - }); - } - - @Override - public String toString() { - return "rank profile '" + name() + "'"; - } - - /** - * A rank setting. The identity of a rank setting is its field name and type (not value). - * A rank setting is immutable. - */ - public static class RankSetting implements Serializable { - - private final String fieldName; - - private final Type type; - - /** The rank value */ - private final Object value; - - public enum Type { - - RANKTYPE("rank-type"), - LITERALBOOST("literal-boost"), - WEIGHT("weight"), - PREFERBITVECTOR("preferbitvector",true); - - private final String name; - - /** True if this setting really pertains to an index, not a field within an index */ - private final boolean isIndexLevel; - - Type(String name) { - this(name,false); - } - - Type(String name,boolean isIndexLevel) { - this.name = name; - this.isIndexLevel=isIndexLevel; - } - - /** True if this setting really pertains to an index, not a field within an index */ - public boolean isIndexLevel() { return isIndexLevel; } - - /** Returns the name of this type */ - public String getName() { - return name; - } - - @Override - public String toString() { - return "type " + name; - } - - } - - public RankSetting(String fieldName, RankSetting.Type type, Object value) { - this.fieldName = fieldName; - this.type = type; - this.value = value; - } - - public String getFieldName() { return fieldName; } - - public Type getType() { return type; } - - public Object getValue() { return value; } - - /** Returns the value as an int, or a negative value if it is not an integer */ - public int getIntValue() { - if (value instanceof Integer) { - return ((Integer)value); - } - else { - return -1; - } - } - - @Override - public int hashCode() { - return fieldName.hashCode() + 17 * type.hashCode(); - } - - @Override - public boolean equals(Object object) { - if (!(object instanceof RankSetting)) { - return false; - } - RankSetting other = (RankSetting)object; - return - fieldName.equals(other.fieldName) && - type.equals(other.type); - } - - @Override - public String toString() { - return type + " setting " + fieldName + ": " + value; - } - - } - - /** A rank property. Rank properties are Value Objects */ - public static class RankProperty implements Serializable { - - private final String name; - private final String value; - - public RankProperty(String name, String value) { - this.name = name; - this.value = value; - } - - public String getName() { return name; } - - public String getValue() { return value; } - - @Override - public int hashCode() { - return name.hashCode() + 17 * value.hashCode(); - } - - @Override - public boolean equals(Object object) { - if (! (object instanceof RankProperty)) return false; - RankProperty other=(RankProperty)object; - return (other.name.equals(this.name) && other.value.equals(this.value)); - } - - @Override - public String toString() { - return name + " = " + value; - } - - } - - /** A function in a rank profile */ - public static class RankingExpressionFunction { - - private ExpressionFunction function; - - /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ - private final boolean inline; - - RankingExpressionFunction(ExpressionFunction function, boolean inline) { - this.function = function; - this.inline = inline; - } - - public void setReturnType(TensorType type) { - this.function = function.withReturnType(type); - } - - public ExpressionFunction function() { return function; } - - public boolean inline() { - return inline && function.arguments().isEmpty(); // only inline no-arg functions; - } - - RankingExpressionFunction withExpression(RankingExpression expression) { - return new RankingExpressionFunction(function.withBody(expression), inline); - } - - @Override - public String toString() { - return function.toString(); - } - - } - - public static final class DiversitySettings { - - private String attribute = null; - private int minGroups = 0; - private double cutoffFactor = 10; - private Diversity.CutoffStrategy cutoffStrategy = Diversity.CutoffStrategy.loose; - - public void setAttribute(String value) { attribute = value; } - public void setMinGroups(int value) { minGroups = value; } - public void setCutoffFactor(double value) { cutoffFactor = value; } - public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; } - public String getAttribute() { return attribute; } - public int getMinGroups() { return minGroups; } - public double getCutoffFactor() { return cutoffFactor; } - public Diversity.CutoffStrategy getCutoffStrategy() { return cutoffStrategy; } - - void checkValid() { - if (attribute == null || attribute.isEmpty()) { - throw new IllegalArgumentException("'diversity' did not set non-empty diversity attribute name."); - } - if (minGroups <= 0) { - throw new IllegalArgumentException("'diversity' did not set min-groups > 0"); - } - if (cutoffFactor < 1.0) { - throw new IllegalArgumentException("diversity.cutoff.factor must be larger or equal to 1.0."); - } - } - } - - public static class MatchPhaseSettings { - - private String attribute = null; - private boolean ascending = false; - private int maxHits = 0; // try to get this many hits before degrading the match phase - private double maxFilterCoverage = 0.2; // Max coverage of original corpus that will trigger the filter. - private DiversitySettings diversity = null; - private double evaluationPoint = 0.20; - private double prePostFilterTippingPoint = 1.0; - - public void setDiversity(DiversitySettings value) { - value.checkValid(); - diversity = value; - } - - public void setAscending(boolean value) { ascending = value; } - public void setAttribute(String value) { attribute = value; } - public void setMaxHits(int value) { maxHits = value; } - public void setMaxFilterCoverage(double value) { maxFilterCoverage = value; } - public void setEvaluationPoint(double evaluationPoint) { this.evaluationPoint = evaluationPoint; } - public void setPrePostFilterTippingPoint(double prePostFilterTippingPoint) { this.prePostFilterTippingPoint = prePostFilterTippingPoint; } - - public boolean getAscending() { return ascending; } - public String getAttribute() { return attribute; } - public int getMaxHits() { return maxHits; } - public double getMaxFilterCoverage() { return maxFilterCoverage; } - public DiversitySettings getDiversity() { return diversity; } - public double getEvaluationPoint() { return evaluationPoint; } - public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } - - public void checkValid() { - if (attribute == null) { - throw new IllegalArgumentException("match-phase did not set any attribute"); - } - if (! (maxHits > 0)) { - throw new IllegalArgumentException("match-phase did not set max-hits > 0"); - } - } - - } - - public static class TypeSettings { - - private final Map types = new HashMap<>(); - - void addType(String name, String type) { - types.put(name, type); - } - - public Map getTypes() { - return Collections.unmodifiableMap(types); - } - - } - - public static final class Input { - - private final Reference name; - private final TensorType type; - private final Optional defaultValue; - - public Input(Reference name, TensorType type, Optional defaultValue) { - this.name = name; - this.type = type; - this.defaultValue = defaultValue; - } - - public Reference name() { return name; } - public TensorType type() { return type; } - public Optional defaultValue() { return defaultValue; } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof Input)) return false; - Input other = (Input)o; - if ( ! other.name().equals(this.name())) return false; - if ( ! other.type().equals(this.type())) return false; - if ( ! other.defaultValue().equals(this.defaultValue())) return false; - return true; - } - - @Override - public int hashCode() { - return Objects.hash(name, type, defaultValue); - } - - @Override - public String toString() { - return "input '" + name + "' " + type + - (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : ""); - } - - } - - public static final class Constant { - - private final Reference name; - private final TensorType type; - - // One of these are non-empty - private final Optional value; - private final Optional valuePath; - - // Always set only if valuePath is set - private final Optional pathType; - - public Constant(Reference name, Tensor value) { - this(name, value.type(), Optional.of(value), Optional.empty(), Optional.empty()); - } - - public Constant(Reference name, TensorType type, String valuePath) { - this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(DistributableResource.PathType.FILE)); - } - - public Constant(Reference name, TensorType type, String valuePath, DistributableResource.PathType pathType) { - this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(pathType)); - } - - private Constant(Reference name, TensorType type, Optional value, - Optional valuePath, Optional pathType) { - this.name = Objects.requireNonNull(name); - this.type = Objects.requireNonNull(type); - this.value = Objects.requireNonNull(value); - this.valuePath = Objects.requireNonNull(valuePath); - this.pathType = Objects.requireNonNull(pathType); - - if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) - throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type + - ": Dense tensor dimensions must have a size"); - } - - public Reference name() { return name; } - public TensorType type() { return type; } - - /** Returns the value of this, if its path is empty. */ - public Optional value() { return value; } - - /** Returns the path to the value of this, if its value is empty. */ - public Optional valuePath() { return valuePath; } - - /** Returns the path type, if valuePath is set. */ - public Optional pathType() { return pathType; } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof Constant)) return false; - Constant other = (Constant)o; - if ( ! other.name().equals(this.name())) return false; - if ( ! other.type().equals(this.type())) return false; - if ( ! other.value().equals(this.value())) return false; - if ( ! other.valuePath().equals(this.valuePath())) return false; - if ( ! other.pathType().equals(this.pathType())) return false; - return true; - } - - @Override - public int hashCode() { - return Objects.hash(name, type, value, valuePath, pathType); - } - - @Override - public String toString() { - return "constant '" + name + "' " + type + ":" + - (value().isPresent() ? value.get().toAbbreviatedString() : " file:" + valuePath.get()); - } - - } - - private static class CachedFunctions { - - private final Map allRankingExpressionFunctions; - - private final ImmutableMap allExpressionFunctions; - - CachedFunctions(Map functions) { - allRankingExpressionFunctions = functions; - ImmutableMap.Builder mapBuilder = new ImmutableMap.Builder<>(); - for (var entry : functions.entrySet()) { - ExpressionFunction function = entry.getValue().function(); - mapBuilder.put(function.getName(), function); - } - allExpressionFunctions = mapBuilder.build(); - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java deleted file mode 100644 index 2e815b7b503..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDDocumentType; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Mapping from name to {@link RankProfile} as well as a reverse mapping of {@link RankProfile} to {@link Schema}. - * Having both of these mappings consolidated here make it easier to remove dependencies on these mappings at - * run time, since it is essentially only used when building rank profile config at deployment time. - * - * Global rank profiles are represented by the Search key null. - * - * @author Ulf Lilleengen - */ -public class RankProfileRegistry { - - private final Map> rankProfiles = new LinkedHashMap<>(); - private static final String globalRankProfilesKey = "[global]"; - - /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ - static final Set overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); - - public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Schema schema) { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); - return rankProfileRegistry; - } - - private String extractName(ImmutableSchema search) { - return search != null ? search.getName() : globalRankProfilesKey; - } - - /** Adds a rank profile to this registry */ - public void add(RankProfile rankProfile) { - String schemaName = extractName(rankProfile.schema()); - if ( ! rankProfiles.containsKey(schemaName)) { - rankProfiles.put(schemaName, new LinkedHashMap<>()); - } - checkForDuplicate(rankProfile); - rankProfiles.get(schemaName).put(rankProfile.name(), rankProfile); - } - - private void checkForDuplicate(RankProfile rankProfile) { - String rankProfileName = rankProfile.name(); - RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.schema())).get(rankProfileName); - if (existingRankProfileWithSameName == null) return; - - if ( ! overridableRankProfileNames.contains(rankProfileName)) { - throw new IllegalArgumentException("Duplicate rank profile '" + rankProfileName + "' in " + - rankProfile.schema()); - } - } - - /** - * Returns a named rank profile, null if the search definition doesn't have one with the given name - * - * @param schema the {@link Schema} that owns the rank profile - * @param name the name of the rank profile - * @return the RankProfile to return. - */ - public RankProfile get(String schema, String name) { - Map profiles = rankProfiles.get(schema); - if (profiles == null) return null; - return profiles.get(name); - } - - public RankProfile get(ImmutableSchema schema, String name) { - var profile = get(schema.getName(), name); - if (profile != null) return profile; - if (schema.inherited().isPresent()) return get(schema.inherited().get(), name); - return null; - } - - public RankProfile getGlobal(String name) { - Map profiles = rankProfiles.get(globalRankProfilesKey); - if (profiles == null) return null; - return profiles.get(name); - } - - public RankProfile resolve(SDDocumentType docType, String name) { - RankProfile rankProfile = get(docType.getName(), name); - if (rankProfile != null) return rankProfile; - for (var parent : docType.getInheritedTypes()) { - RankProfile parentProfile = resolve(parent, name); - if (parentProfile != null) return parentProfile; - } - return get(globalRankProfilesKey, name); - } - - /** - * Rank profiles that are collected across clusters. - * - * @return a set of global {@link RankProfile} instances - */ - public Collection all() { - List all = new ArrayList<>(); - for (var entry : rankProfiles.values()) { - all.addAll(entry.values()); - } - return all; - } - - /** - * Retrieve all rank profiles for a schema - * - * @param schema the schema to fetch rank profiles for, or null for the global ones - * @return a collection of {@link RankProfile} instances - */ - public Collection rankProfilesOf(ImmutableSchema schema) { - String key = schema == null ? globalRankProfilesKey : schema.getName(); - - if ( ! rankProfiles.containsKey(key)) return List.of(); - - var profiles = new LinkedHashMap<>(rankProfiles.get(key)); - // Add all profiles in inherited schemas, unless they are already present (overridden) - while (schema != null && schema.inherited().isPresent()) { - schema = schema.inherited().get(); - var inheritedProfiles = rankProfiles.get(schema.getName()); - if (inheritedProfiles != null) { - for (Map.Entry inheritedProfile : inheritedProfiles.entrySet()) { - profiles.putIfAbsent(inheritedProfile.getKey(), inheritedProfile.getValue()); - } - } - } - return profiles.values(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java deleted file mode 100644 index ce32f323f22..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.*; -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.TemporarySDDocumentType; - -import java.util.*; -import java.util.logging.Level; - -/** - * @author Einar M R Rosenvinge - */ -public class SDDocumentTypeOrderer { - - private final Map createdSDTypes = new LinkedHashMap<>(); - private final Set seenTypes = Collections.newSetFromMap(new IdentityHashMap<>()); - List processingOrder = new LinkedList<>(); - private final DeployLogger deployLogger; - - public SDDocumentTypeOrderer(List sdTypes, DeployLogger deployLogger) { - this.deployLogger = deployLogger; - for (SDDocumentType type : sdTypes) { - createdSDTypes.put(type.getDocumentName(), type); - } - } - - List getOrdered() { return processingOrder; } - - public void process() { - for (SDDocumentType type : createdSDTypes.values()) { - process(type, type); - } - } - - private void process(SDDocumentType docOrStruct, SDDocumentType owningDocument) { - resolveAndProcessInheritedTemporaryTypes(docOrStruct, owningDocument); - if (seenTypes.contains(docOrStruct)) { - return; - } - seenTypes.add(docOrStruct); - for (Field field : docOrStruct.fieldSet()) { - var type = field.getDataType(); - String typeName = type.getName(); - if (!seenTypes.contains(type)) { - seenTypes.add(type); - //we haven't seen this before, do it - visit(type, owningDocument); - } - } - processingOrder.add(docOrStruct); - } - - private void resolveAndProcessInheritedTemporaryTypes(SDDocumentType type, SDDocumentType owningDocument) { - List toReplace = new ArrayList<>(); - for (SDDocumentType sdoc : type.getInheritedTypes()) { - if (sdoc instanceof TemporarySDDocumentType) { - toReplace.add(sdoc.getDocumentName()); - } - } - for (DataTypeName name : toReplace) { - SDDocumentType inherited; - if (type.isStruct()) { - inherited = owningDocument.allTypes().get(new NewDocumentType.Name(name.getName())); - if (inherited == null) throw new IllegalArgumentException("Struct '" + name + "' not found in " + owningDocument); - process(inherited, owningDocument); - } - else { - inherited = createdSDTypes.get(name); - if (inherited == null) { - throw new IllegalArgumentException("document " + type.getName() + - " inherits from unavailable document " + name); - } - process(inherited, inherited); - } - type.inherit(inherited); - } - } - - private SDDocumentType find(String name) { - SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); - if (sdDocType != null) { - return sdDocType; - } - for(SDDocumentType sdoc : createdSDTypes.values()) { - for (SDDocumentType stype : sdoc.getTypes()) { - if (stype.getName().equals(name)) { - return stype; - } - } - } - return null; - } - - private void visit(DataType type, SDDocumentType owningDocument) { - if (type instanceof StructuredDataType) { - StructuredDataType structType = (StructuredDataType) type; - SDDocumentType sdDocType = owningDocument.getType(structType.getName()); - if (sdDocType == null) { - sdDocType = find(structType.getName()); - } - if (sdDocType == null) { - throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'"); - } - process(sdDocType, owningDocument); - return; - } - - if (type instanceof MapDataType) { - MapDataType mType = (MapDataType) type; - visit(mType.getValueType(), owningDocument); - visit(mType.getKeyType(), owningDocument); - } else if (type instanceof WeightedSetDataType) { - WeightedSetDataType wType = (WeightedSetDataType) type; - visit(wType.getNestedType(), owningDocument); - } else if (type instanceof CollectionDataType) { - CollectionDataType cType = (CollectionDataType) type; - visit(cType.getNestedType(), owningDocument); - } else if (type instanceof AnnotationReferenceDataType) { - //do nothing - } else if (type instanceof PrimitiveDataType) { - //do nothing - } else if (type instanceof TensorDataType) { - //do nothing - } else if (type instanceof NewDocumentReferenceDataType) { - //do nothing - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unknown type : " + type); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java deleted file mode 100644 index 8139d46cc0a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ /dev/null @@ -1,756 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.document.DataTypeName; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.derived.FileDistributedOnnxModels; -import com.yahoo.searchdefinition.derived.SummaryClass; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.document.TemporaryImportedFields; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; - -import java.io.Reader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.stream.Stream; - -/** - * A schema contains a document type, additional fields, rank profiles and document summaries. - * - * @author bratseth - */ -// TODO: Make a class owned by this, for each of these responsibilities: -// Managing indexes, managing attributes, managing summary classes. -// Ensure that after the processing step, all implicit instances of the above types are explicitly represented -public class Schema implements ImmutableSchema { - - private static final String SD_DOC_FIELD_NAME = "sddocname"; - private static final List RESERVED_NAMES = List.of( - "index", "index_url", "summary", "attribute", "select_input", "host", SummaryClass.DOCUMENT_ID_FIELD, - "position", "split_foreach", "tokenize", "if", "else", "switch", "case", SD_DOC_FIELD_NAME, "relevancy"); - - /** The unique name of this schema */ - private String name; - - /** The application package this is constructed from */ - private final ApplicationPackage applicationPackage; - - /** The name of the schema this should inherit all the content of, if any */ - private final Optional inherited; - - /** True if this doesn't define a search, just a document type */ - private final boolean documentsOnly; - - private Boolean rawAsBase64 = null; - - /** The stemming setting of this schema. Default is BEST. */ - private Stemming stemming = null; - - private final FieldSets fieldSets = new FieldSets(Optional.of(this)); - - /** The document contained in this schema */ - private SDDocumentType documentType; - - /** The extra fields of this schema */ - private final Map fields = new LinkedHashMap<>(); - - private final Map indices = new LinkedHashMap<>(); - - /** The explicitly defined summaries of this schema. _Must_ preserve order. */ - private final Map summaries = new LinkedHashMap<>(); - - /** External rank expression files of this */ - private final LargeRankExpressions largeRankExpressions; - - /** Constants that will be available in all rank profiles. */ - // TODO: Remove on Vespa 9: Should always be in a rank profile - private final Map constants = new LinkedHashMap<>(); - - // TODO: Remove on Vespa 9: Should always be in a rank profile - private final Map onnxModels = new LinkedHashMap<>(); - - /** All imported fields of this (and parent schemas) */ - // TODO: Use empty, not optional - // TODO: Merge this and importedFields - private final Optional temporaryImportedFields = Optional.of(new TemporaryImportedFields(this)); - /** The resulting processed field */ - private Optional importedFields = Optional.empty(); - - private final DeployLogger deployLogger; - private final ModelContext.Properties properties; - - private Application owner; - - /** Testing only */ - public Schema(String name, ApplicationPackage applicationPackage) { - this(name, applicationPackage, Optional.empty(), null, new BaseDeployLogger(), new TestProperties()); - } - - public Schema(String name, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties) { - this(name, applicationPackage, Optional.empty(), fileRegistry, deployLogger, properties); - } - - /** - * Creates a schema - * - * @param name of the schema - * @param inherited the schema this inherits, if any - */ - public Schema(String name, - ApplicationPackage applicationPackage, - Optional inherited, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties) { - this(inherited, applicationPackage, fileRegistry, deployLogger, properties, false); - this.name = Objects.requireNonNull(name, "A schema must have a name"); - } - - protected Schema(ApplicationPackage applicationPackage, FileRegistry fileRegistry, - DeployLogger deployLogger, ModelContext.Properties properties) { - this(Optional.empty(), applicationPackage, fileRegistry, deployLogger, properties, true); - } - - private Schema(Optional inherited, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - boolean documentsOnly) { - this.inherited = inherited; - this.applicationPackage = applicationPackage; - this.deployLogger = deployLogger; - this.properties = properties; - this.documentsOnly = documentsOnly; - largeRankExpressions = new LargeRankExpressions(fileRegistry); - } - - /** - * Assigns the owner of this - * - * @throws IllegalStateException if an owner is already assigned - */ - public void setOwner(Application owner) { - if (this.owner != null) - throw new IllegalStateException("Cannot reassign the owner of " + this); - this.owner = owner; - } - - protected void setName(String name) { this.name = name; } - - @Override - public String getName() {return name; } - - /** Returns true if this only defines a document type, not a full schema */ - public boolean isDocumentsOnly() { - return documentsOnly; - } - - @Override - public Optional inherited() { - return inherited.map(name -> owner.schemas().get(name)); - } - - /** - * Returns true if 'raw' fields shall be presented as base64 in summary - * Note that this is temporary and will disappear on Vespa 8 as it will become default, and only option. - * - * @return true if raw shall be encoded as base64 in summary - */ - public boolean isRawAsBase64() { - if (rawAsBase64 != null) return rawAsBase64; - // TODO Vespa 8: flip default: - if (inherited.isEmpty()) return false; - return requireInherited().isRawAsBase64(); - } - - public void enableRawAsBase64(boolean value) { rawAsBase64 = value; } - - /** - * Sets the stemming default of fields. Default is ALL - * - * @param stemming set default stemming for this searchdefinition - * @throws NullPointerException if this is attempted set to null - */ - public void setStemming(Stemming stemming) { - this.stemming = Objects.requireNonNull(stemming, "Stemming cannot be null"); - } - - /** Returns whether fields should be stemmed by default or not. Default is BEST. This is never null. */ - public Stemming getStemming() { - if (stemming != null) return stemming; - if (inherited.isEmpty()) return Stemming.BEST; - return requireInherited().getStemming(); - } - - /** - * Adds a document type which is defined in this search definition - * - * @param document the document type to add - */ - public void addDocument(SDDocumentType document) { - if (documentType != null) { - throw new IllegalArgumentException("Schema cannot have more than one document"); - } - documentType = document; - } - - @Override - public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } - - public void add(RankProfile.Constant constant) { - constants.put(constant.name(), constant); - } - - /** Returns an unmodifiable map of the constants declared in this. */ - public Map declaredConstants() { return constants; } - - /** Returns an unmodifiable map of the constants available in this. */ - @Override - public Map constants() { - if (inherited().isEmpty()) return Collections.unmodifiableMap(constants); - if (constants.isEmpty()) return inherited().get().constants(); - - Map allConstants = new LinkedHashMap<>(inherited().get().constants()); - allConstants.putAll(constants); - return allConstants; - } - - public void add(OnnxModel model) { - onnxModels.put(model.getName(), model); - } - - /** Returns an unmodifiable map of the onnx models declared in this. */ - public Map declaredOnnxModels() { return onnxModels; } - - /** Returns an unmodifiable map of the onnx models available in this. */ - @Override - public Map onnxModels() { - if (inherited().isEmpty()) return Collections.unmodifiableMap(onnxModels); - if (onnxModels.isEmpty()) return inherited().get().onnxModels(); - - Map allModels = new LinkedHashMap<>(inherited().get().onnxModels()); - allModels.putAll(onnxModels); - return allModels; - } - - public Optional temporaryImportedFields() { - return temporaryImportedFields; - } - - public Optional importedFields() { - return importedFields; - } - - public void setImportedFields(ImportedFields importedFields) { - this.importedFields = Optional.of(importedFields); - } - - @Override - public Stream allImportedFields() { - return importedFields - .map(fields -> fields.fields().values().stream()) - .orElse(Stream.empty()) - .map(field -> field.asImmutableSDField()); - } - - @Override - public ImmutableSDField getField(String name) { - ImmutableSDField field = getConcreteField(name); - if (field != null) return field; - return allImportedFields() - .filter(f -> f.getName().equals(name)) - .findFirst() - .orElse(null); - } - - @Override - public List allFieldsList() { - List all = new ArrayList<>(); - all.addAll(extraFieldList()); - for (Field field : documentType.fieldSet()) { - all.add((ImmutableSDField) field); - } - if (importedFields.isPresent()) { - for (ImportedField imported : importedFields.get().fields().values()) { - all.add(imported.asImmutableSDField()); - } - } - return all; - } - - /** - * Gets a document from this search definition - * - * @param name the name of the document to return - * @return the contained or used document type, or null if there is no such document - */ - public SDDocumentType getDocument(String name) { - if (documentType != null && name.equals(documentType.getName())) { - return documentType; - } - return null; - } - - /** - * @return true if the document has been added. - */ - public boolean hasDocument() { - return documentType != null; - } - - /** - * @return The document in this search. - */ - @Override - public SDDocumentType getDocument() { - return documentType; - } - - /** - * Returns a list of all the fields of this search definition, that is all fields in all documents, in the documents - * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not - * impact this - */ - @Override - public List allConcreteFields() { - List allFields = new ArrayList<>(); - allFields.addAll(extraFieldList()); - for (Field field : documentType.fieldSet()) { - allFields.add((SDField)field); - } - return allFields; - } - - /** - * Returns the content of a ranking expression file - */ - @Override - public Reader getRankingExpression(String fileName) { - return applicationPackage.getRankingExpression(fileName); - } - - public Application application() { return owner; } - - @Override - public ApplicationPackage applicationPackage() { return applicationPackage; } - - @Override - public DeployLogger getDeployLogger() { return deployLogger; } - - @Override - public ModelContext.Properties getDeployProperties() { return properties; } - - /** - * Returns a field defined in this search definition or one if its documents. Fields in this search definition takes - * precedence over document fields having the same name - * - * @param name of the field - * @return the SDField representing the field - */ - @Override - public SDField getConcreteField(String name) { - SDField field = getExtraField(name); - if (field != null) return field; - - return (SDField) documentType.getField(name); - } - - /** - * Returns a field defined in one of the documents of this search definition. - * This does not include the extra fields defined outside the document - * (those accessible through the getExtraField() method). - * - * @param name the name of the field to return - * @return the named field, or null if not found - */ - public SDField getDocumentField(String name) { - return (SDField) documentType.getField(name); - } - - /** - * Adds an extra field of this search definition not contained in a document - * - * @param field to add to the schemas list of external fields - */ - public void addExtraField(SDField field) { - if (fields.containsKey(field.getName())) { - deployLogger.logApplicationPackage(Level.WARNING, "Duplicate field " + field.getName() + " in search definition " + getName()); - } else { - field.setIsExtraField(true); - fields.put(field.getName(), field); - } - } - - public Collection extraFieldList() { - if (inherited.isEmpty()) return fields.values(); - var fields = new HashSet<>(requireInherited().extraFieldList()); - fields.addAll(this.fields.values()); - return fields; - } - - public Collection allExtraFields() { - Map extraFields = new TreeMap<>(); - if (inherited.isPresent()) - requireInherited().allExtraFields().forEach(field -> extraFields.put(field.getName(), field)); - for (Field field : documentType.fieldSet()) { - SDField sdField = (SDField) field; - if (sdField.isExtraField()) { - extraFields.put(sdField.getName(), sdField); - } - } - for (SDField field : extraFieldList()) { - extraFields.put(field.getName(), field); - } - return extraFields.values(); - } - - /** - * Returns a field by name, or null if it is not present - * - * @param fieldName the name of the external field to get - * @return the SDField of this name - */ - public SDField getExtraField(String fieldName) { - SDField field = fields.get(fieldName); - if (field != null) return field; - if (inherited.isEmpty()) return null; - return requireInherited().getExtraField(fieldName); - } - - /** - * Adds an explicitly defined index to this search definition - * - * @param index the index to add - */ - public void addIndex(Index index) { - indices.put(index.getName(), index); - } - - /** - * Returns an index, or null if no index with this name has had some explicit settings applied. Even if - * this returns null, the index may be implicitly defined by an indexing statement. This will return the - * index whether it is defined on this schema or on one of its fields. - * - * @param name the name of the index to get - * @return the index requested - */ - @Override - public Index getIndex(String name) { - List sameIndices = new ArrayList<>(1); - - getSchemaIndex(name).ifPresent(sameIndices::add); - - for (ImmutableSDField field : allConcreteFields()) { - if (field.getIndex(name) != null) - sameIndices.add(field.getIndex(name)); - } - if (sameIndices.size() == 0) return null; - if (sameIndices.size() == 1) return sameIndices.get(0); - return consolidateIndices(sameIndices); - } - - /** Returns the schema level index of this name, in this or any inherited schema, if any */ - Optional getSchemaIndex(String name) { - if (indices.containsKey(name)) return Optional.of(indices.get(name)); - if (inherited.isPresent()) return requireInherited().getSchemaIndex(name); - return Optional.empty(); - } - - public boolean existsIndex(String name) { - if (indices.get(name) != null) - return true; - if (inherited.isPresent() && requireInherited().existsIndex(name)) - return true; - for (ImmutableSDField field : allConcreteFields()) { - if (field.existsIndex(name)) - return true; - } - return false; - } - - /** - * Consolidates a set of index settings for the same index into one - * - * @param indices the list of indexes to consolidate - * @return the consolidated index - */ - private Index consolidateIndices(List indices) { - Index first = indices.get(0); - Index consolidated = new Index(first.getName()); - consolidated.setRankType(first.getRankType()); - consolidated.setType(first.getType()); - for (Index current : indices) { - if (current.isPrefix()) { - consolidated.setPrefix(true); - } - if (current.useInterleavedFeatures()) { - consolidated.setInterleavedFeatures(true); - } - - if (consolidated.getRankType() == null) { - consolidated.setRankType(current.getRankType()); - } else { - if (current.getRankType() != null && consolidated.getRankType() != current.getRankType()) - deployLogger.logApplicationPackage(Level.WARNING, "Conflicting rank type settings for " + - first.getName() + " in " + this + ", using " + - consolidated.getRankType()); - } - - for (Iterator j = current.aliasIterator(); j.hasNext();) { - consolidated.addAlias(j.next()); - } - } - return consolidated; - } - - /** All explicitly defined indices, both on this schema itself (returned first) and all its fields */ - @Override - public List getExplicitIndices() { - List allIndices = new ArrayList<>(indices.values()); - - if (inherited.isPresent()) { - for (Index inheritedIndex : requireInherited().getExplicitIndices()) { - if ( ! indices.containsKey(inheritedIndex.getName())) // child redefinitions shadows parents - allIndices.add(inheritedIndex); - } - } - - for (ImmutableSDField field : allConcreteFields()) - allIndices.addAll(field.getIndices().values()); - - return Collections.unmodifiableList(allIndices); - } - - /** Adds an explicitly defined summary to this search definition */ - public void addSummary(DocumentSummary summary) { - summaries.put(summary.getName(), summary); - } - - /** - * Returns a summary class defined by this search definition, or null if no summary with this name is defined. - * The default summary, named "default" is always present. - */ - public DocumentSummary getSummary(String name) { - var summary = summaries.get(name); - if (summary != null) return summary; - if (inherited.isEmpty()) return null; - return requireInherited().getSummary(name); - } - - /** - * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly - * or explicitly) in any summary class. - */ - public SummaryField getSummaryField(String name) { - for (DocumentSummary summary : summaries.values()) { - SummaryField summaryField = summary.getSummaryField(name); - if (summaryField != null) { - return summaryField; - } - } - if (inherited.isEmpty()) return null; - return requireInherited().getSummaryField(name); - } - - /** - * Returns the first explicit instance found of a summary field with this name, or null if not present explicitly in - * any summary class - * - * @param name the name of the explicit summary field to get. - * @return the SummaryField found. - */ - public SummaryField getExplicitSummaryField(String name) { - for (DocumentSummary summary : summaries.values()) { - SummaryField summaryField = summary.getSummaryField(name); - if (summaryField != null && !summaryField.isImplicit()) - return summaryField; - } - if (inherited.isEmpty()) return null; - return requireInherited().getExplicitSummaryField(name); - } - - /** - * Summaries defined by fields of this search definition. The default summary, named "default", is always the first - * one in the returned iterator. - */ - public Map getSummaries() { - // Shortcuts - if (inherited.isEmpty()) return summaries; - if (summaries.isEmpty()) return requireInherited().getSummaries(); - - var allSummaries = new LinkedHashMap<>(requireInherited().getSummaries()); - allSummaries.putAll(summaries); - return allSummaries; - } - - /** Returns the summaries defines in this only, not any that are inherited. */ - public Map getSummariesInThis() { return Collections.unmodifiableMap(summaries); } - - /** - * Returns all summary fields, of all document summaries, which has the given field as source. - * The list becomes owned by the receiver. - * - * @param field the source field - * @return the list of summary fields found - */ - @Override - public List getSummaryFields(ImmutableSDField field) { - List summaryFields = inherited.isPresent() - ? requireInherited().getSummaryFields(field) - : new java.util.ArrayList<>(); - for (DocumentSummary documentSummary : summaries.values()) { - for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { - if (summaryField.hasSource(field.getName())) { - boolean wanted = true; - for (var already : summaryFields) { - if (summaryField == already) wanted = false; - } - if (wanted) { - summaryFields.add(summaryField); - } - } - } - } - return summaryFields; - } - - /** - * Returns one summary field for each summary field name. If there are multiple summary fields with the same - * name, the last one will be used. Multiple fields of the same name should all have the same content in a valid - * search definition, except from the destination set. So this method can be used for all summary handling except - * processing the destination set. The map becomes owned by the receiver. - */ - public Map getUniqueNamedSummaryFields() { - Map summaryFields = inherited.isPresent() ? requireInherited().getUniqueNamedSummaryFields() - : new java.util.LinkedHashMap<>(); - for (DocumentSummary documentSummary : summaries.values()) { - for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { - summaryFields.put(summaryField.getName(), summaryField); - } - } - return summaryFields; - } - - /** Returns the first occurrence of an attribute having this name, or null if none */ - public Attribute getAttribute(String name) { - for (ImmutableSDField field : allConcreteFields()) { - Attribute attribute = field.getAttributes().get(name); - if (attribute != null) { - return attribute; - } - } - return null; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Schema)) { - return false; - } - - Schema other = (Schema)o; - return getName().equals(other.getName()); - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public String toString() { - return "schema '" + getName() + "'"; - } - - public boolean isAccessingDiskSummary(SummaryField field) { - if (!field.getTransform().isInMemory()) return true; - if (field.getSources().size() == 0) return isAccessingDiskSummary(getName()); - for (SummaryField.Source source : field.getSources()) { - if (isAccessingDiskSummary(source.getName())) - return true; - } - return false; - } - - private boolean isAccessingDiskSummary(String source) { - SDField field = getConcreteField(source); - if (field == null) return false; - if (field.doesSummarying() && !field.doesAttributing()) return true; - return false; - } - - public FieldSets fieldSets() { return fieldSets; } - - /** Returns the schema inherited by this, or throws if none */ - private Schema requireInherited() { return owner.schemas().get(inherited.get()); } - - /** - * For adding structs defined in document scope - * - * @param dt the struct to add - * @return self, for chaining - */ - public Schema addType(SDDocumentType dt) { - documentType.addType(dt); // TODO This is a very very dirty thing. It must go - return this; - } - - public Schema addAnnotation(SDAnnotationType dt) { - documentType.addAnnotation(dt); - return this; - } - - public void validate(DeployLogger logger) { - if (inherited.isPresent()) { - if (! owner.schemas().containsKey(inherited.get())) - throw new IllegalArgumentException(this + " inherits '" + inherited.get() + - "', but this schema does not exist"); - - // Require schema and document type inheritance to be consistent to keep things simple - // And require it to be explicit so we have the option to support other possibilities later - var parentDocument = owner.schemas().get(inherited.get()).getDocument(); - if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) - throw new IllegalArgumentException(this + " inherits '" + inherited.get() + - "', but its document type does not inherit the parent's document type"); - } - for (var summary : summaries.values()) - summary.validate(logger); - } - - /** Returns true if the given field name is a reserved name */ - public static boolean isReservedName(String name) { - return RESERVED_NAMES.contains(name); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TemporarySDTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/TemporarySDTypeResolver.java deleted file mode 100644 index 2eaf0d5e5ba..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/TemporarySDTypeResolver.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.TemporarySDDocumentType; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.logging.Level; - -/** - * @author arnej - */ -public class TemporarySDTypeResolver { - - private final DeployLogger deployLogger; - private final Collection toProcess; - private final List docTypes = new LinkedList<>(); - - public TemporarySDTypeResolver(Collection schemas, DeployLogger deployLogger) { - this.deployLogger = deployLogger; - this.toProcess = schemas; - } - - private SDDocumentType findDocType(String name) { - assert(name != null); - for (var doc : docTypes) { - if (doc.getName().equals(name)) { - return doc; - } - } - deployLogger.logApplicationPackage(Level.WARNING, "No document type in application matching name: "+name); - return null; - } - - public void process() { - docTypes.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : toProcess) { - if (schema.hasDocument()) { - docTypes.add(schema.getDocument()); - } - } - // first, fix inheritance - for (SDDocumentType doc : docTypes) { - for (SDDocumentType inherited : doc.getInheritedTypes()) { - if (inherited instanceof TemporarySDDocumentType) { - var actual = findDocType(inherited.getName()); - if (actual != null) { - doc.inherit(actual); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for document "+doc.getName()); - } - } - } - } - // next, check owned types (structs only?) - for (SDDocumentType doc : docTypes) { - for (SDDocumentType owned : doc.getTypes()) { - if (owned instanceof TemporarySDDocumentType) { - deployLogger.logApplicationPackage(Level.WARNING, "Schema '"+doc.getName()+"' owned type '"+owned.getName()+"' is temporary, should not happen"); - continue; - } - for (SDDocumentType inherited : owned.getInheritedTypes()) { - if (inherited instanceof TemporarySDDocumentType) { - var actual = doc.getType(inherited.getName()); - if (actual != null) { - owned.inherit(actual); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "Unresolved inherit '"+inherited.getName() +"' for type '"+owned.getName()+"' in document "+doc.getName()); - } - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java deleted file mode 100644 index 0e4578878df..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.derived.FileDistributedConstants; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; - -/** - * A low-cost ranking profile to use for watcher queries etc. - * - * @author Vegard Havdal - */ -public class UnrankedRankProfile extends RankProfile { - - public UnrankedRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { - super("unranked", schema, rankProfileRegistry); - try { - RankingExpression exp = new RankingExpression("value(0)"); - this.setFirstPhaseRanking(exp); - } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse the ranking expression 'value(0)' when setting up " + - "the 'unranked' rank profile"); - } - this.setIgnoreDefaultRankFeatures(true); - this.setKeepRankCount(0); - this.setRerankCount(0); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java deleted file mode 100644 index cd10263b0ff..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Ranking; -import com.yahoo.searchdefinition.document.Sorting; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; - -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField; - -/** - * The set of all attribute fields defined by a search definition - * - * @author bratseth - */ -public class AttributeFields extends Derived implements AttributesConfig.Producer { - - public enum FieldSet {ALL, FAST_ACCESS} - - private Map attributes = new java.util.LinkedHashMap<>(); - private Map importedAttributes = new java.util.LinkedHashMap<>(); - - /** Whether this has any position attribute */ - private boolean hasPosition = false; - - public static final AttributeFields empty = new AttributeFields(null); - - public AttributeFields(Schema schema) { - if (schema != null) - derive(schema); - } - - /** Derives everything from a field */ - @Override - protected void derive(ImmutableSDField field, Schema schema) { - if (unsupportedFieldType(field)) { - return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) - } - if (isArrayOfSimpleStruct(field)) { - deriveArrayOfSimpleStruct(field); - } else if (isMapOfSimpleStruct(field)) { - deriveMapOfSimpleStruct(field); - } else if (isMapOfPrimitiveType(field)) { - deriveMapOfPrimitiveType(field); - } else { - deriveAttributes(field); - } - } - - private static boolean unsupportedFieldType(ImmutableSDField field) { - return (field.usesStructOrMap() && - !isSupportedComplexField(field) && - !GeoPos.isAnyPos(field)); - } - - /** Returns an attribute by name, or null if it doesn't exist */ - public Attribute getAttribute(String attributeName) { - return attributes.get(attributeName); - } - - public boolean containsAttribute(String attributeName) { - return getAttribute(attributeName) != null; - } - - /** Derives one attribute. TODO: Support non-default named attributes */ - private void deriveAttributes(ImmutableSDField field) { - if (field.isImportedField()) { - deriveImportedAttributes(field); - return; - } - for (Attribute fieldAttribute : field.getAttributes().values()) { - deriveAttribute(field, fieldAttribute); - } - - if (field.containsExpression(ToPositionExpression.class)) { - // TODO: Move this check to processing and remove this - if (hasPosition) { - throw new IllegalArgumentException("Can not specify more than one set of position attributes per field: " + field.getName()); - } - hasPosition = true; - } - } - - private void applyRanking(ImmutableSDField field, Attribute attribute) { - Ranking ranking = field.getRanking(); - if (ranking != null && ranking.isFilter()) { - attribute.setEnableBitVectors(true); - attribute.setEnableOnlyBitVector(true); - } - } - - private void deriveAttribute(ImmutableSDField field, Attribute fieldAttribute) { - Attribute attribute = getAttribute(fieldAttribute.getName()); - if (attribute == null) { - attributes.put(fieldAttribute.getName(), fieldAttribute); - attribute = getAttribute(fieldAttribute.getName()); - } - applyRanking(field, attribute); - } - - private void deriveImportedAttributes(ImmutableSDField field) { - for (Attribute attribute : field.getAttributes().values()) { - if (!importedAttributes.containsKey(field.getName())) { - importedAttributes.put(field.getName(), attribute); - } - } - } - - private void deriveArrayOfSimpleStruct(ImmutableSDField field) { - for (ImmutableSDField structField : field.getStructFields()) { - deriveAttributeAsArrayType(structField); - } - } - - private void deriveAttributeAsArrayType(ImmutableSDField field) { - if (field.isImportedField()) { - deriveImportedAttributes(field); - return; - } - Attribute attribute = field.getAttributes().get(field.getName()); - if (attribute != null) { - applyRanking(field, attribute); - attributes.put(attribute.getName(), attribute.convertToArray()); - } - } - - private void deriveMapOfSimpleStruct(ImmutableSDField field) { - deriveAttributeAsArrayType(field.getStructField("key")); - deriveMapValueField(field.getStructField("value")); - } - - private void deriveMapValueField(ImmutableSDField valueField) { - for (ImmutableSDField structField : valueField.getStructFields()) { - deriveAttributeAsArrayType(structField); - } - } - - private void deriveMapOfPrimitiveType(ImmutableSDField field) { - deriveAttributeAsArrayType(field.getStructField("key")); - deriveAttributeAsArrayType(field.getStructField("value")); - } - - /** Returns a read only attribute iterator */ - public Iterator attributeIterator() { - return attributes().iterator(); - } - - public Collection attributes() { - return Collections.unmodifiableCollection(attributes.values()); - } - - public Collection structFieldAttributes(String baseFieldName) { - String structPrefix = baseFieldName + "."; - return attributes().stream() - .filter(attribute -> attribute.getName().startsWith(structPrefix)) - .collect(Collectors.toList()); - } - - public String toString() { - return "attributes " + getName(); - } - - @Override - protected String getDerivedName() { - return "attributes"; - } - - @SuppressWarnings("removal") // TODO Vespa 8: remove - private Map toMap(List ls) { - Map ret = new LinkedHashMap<>(); - for (AttributesConfig.Attribute.Builder builder : ls) { - ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder); - } - return ret; - } - - @Override - public void getConfig(AttributesConfig.Builder builder) { - //TODO This is just to get some exporting tests to work, Should be undone and removed - getConfig(builder, FieldSet.ALL, 77777, false); - } - - private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { - return (fs == FieldSet.ALL) || ((fs == FieldSet.FAST_ACCESS) && attribute.isFastAccess()); - } - - private AttributesConfig.Attribute.Builder getConfig(String attrName, Attribute attribute, boolean imported) { - AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder() - .name(attrName) - .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName())) - .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName())); - if (attribute.isRemoveIfZero()) { - aaB.removeifzero(true); - } - if (attribute.isCreateIfNonExistent()) { - aaB.createifnonexistent(true); - } - aaB.enablebitvectors(attribute.isEnabledBitVectors()); - aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector()); - if (attribute.isFastSearch() || attribute.isFastRank()) { - // TODO make a separate fastrank flag in config instead of overloading fastsearch - aaB.fastsearch(true); - } - if (attribute.isFastAccess()) { - aaB.fastaccess(true); - } - if (attribute.isMutable()) { - aaB.ismutable(true); - } - if (attribute.isHuge()) { - aaB.huge(true); - } - if (attribute.isPaged()) { - aaB.paged(true); - } - if (attribute.getSorting().isDescending()) { - aaB.sortascending(false); - } - if (attribute.getSorting().getFunction() != Sorting.Function.UCA) { - aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString())); - } - if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) { - aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString())); - } - if (!attribute.getSorting().getLocale().isEmpty()) { - aaB.sortlocale(attribute.getSorting().getLocale()); - } - aaB.arity(attribute.arity()); - aaB.lowerbound(attribute.lowerBound()); - aaB.upperbound(attribute.upperBound()); - aaB.densepostinglistthreshold(attribute.densePostingListThreshold()); - if (attribute.tensorType().isPresent()) { - aaB.tensortype(attribute.tensorType().get().toString()); - } - aaB.imported(imported); - var dma = attribute.distanceMetric(); - aaB.distancemetric(AttributesConfig.Attribute.Distancemetric.Enum.valueOf(dma.toString())); - if (attribute.hnswIndexParams().isPresent()) { - var ib = new AttributesConfig.Attribute.Index.Builder(); - var params = attribute.hnswIndexParams().get(); - ib.hnsw.enabled(true); - ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); - ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); - ib.hnsw.multithreadedindexing(params.multiThreadedIndexing()); - aaB.index(ib); - } - Dictionary dictionary = attribute.getDictionary(); - if (dictionary != null) { - aaB.dictionary.type(convert(dictionary.getType())); - aaB.dictionary.match(convert(dictionary.getMatch())); - } - aaB.match(convertMatch(attribute.getCase())); - return aaB; - } - - private static AttributesConfig.Attribute.Dictionary.Type.Enum convert(Dictionary.Type type) { - switch (type) { - case BTREE: - return AttributesConfig.Attribute.Dictionary.Type.BTREE; - case HASH: - return AttributesConfig.Attribute.Dictionary.Type.HASH; - case BTREE_AND_HASH: - return AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; - } - return AttributesConfig.Attribute.Dictionary.Type.BTREE; - } - private static AttributesConfig.Attribute.Dictionary.Match.Enum convert(Case type) { - switch (type) { - case CASED: - return AttributesConfig.Attribute.Dictionary.Match.CASED; - case UNCASED: - return AttributesConfig.Attribute.Dictionary.Match.UNCASED; - } - return AttributesConfig.Attribute.Dictionary.Match.UNCASED; - } - private static AttributesConfig.Attribute.Match.Enum convertMatch(Case type) { - switch (type) { - case CASED: - return AttributesConfig.Attribute.Match.CASED; - case UNCASED: - return AttributesConfig.Attribute.Match.UNCASED; - } - return AttributesConfig.Attribute.Match.UNCASED; - } - - public void getConfig(AttributesConfig.Builder builder, FieldSet fs, long maxUnCommittedMemory, boolean enableBitVectors) { - for (Attribute attribute : attributes.values()) { - if (isAttributeInFieldSet(attribute, fs)) { - AttributesConfig.Attribute.Builder attrBuilder = getConfig(attribute.getName(), attribute, false); - attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); - if (enableBitVectors && attribute.isFastSearch()) { - attrBuilder.enablebitvectors(true); - } - builder.attribute(attrBuilder); - } - } - if (fs == FieldSet.ALL) { - for (Map.Entry entry : importedAttributes.entrySet()) { - AttributesConfig.Attribute.Builder attrBuilder = getConfig(entry.getKey(), entry.getValue(), true); - attrBuilder.maxuncommittedmemory(maxUnCommittedMemory); - builder.attribute(attrBuilder); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java deleted file mode 100644 index 99c40c2700f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.ConfigInstance.Builder; -import com.yahoo.document.Field; -import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.text.StringUtilities; - -import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Method; -import java.util.List; - -/** - * Superclass of all derived configurations - * - * @author bratseth - */ -public abstract class Derived implements Exportable { - - private String name; - - public Derived() { - this(""); - } - - public Derived(String name) { - this.name = name; - } - - public String getName() { return name; } - - protected final void setName(String name) { this.name = name; } - - /** - * Derives the content of this configuration. This - * default calls derive(Document) for each document - * and derive(SDField) for each search definition level field - * AND sets the name of this to the name of the input search definition - */ - protected void derive(Schema schema) { - setName(schema.getName()); - derive(schema.getDocument(), schema); - for (Index index : schema.getExplicitIndices()) - derive(index, schema); - for (SDField field : schema.allExtraFields()) - derive(field, schema); - schema.allImportedFields().forEach(importedField -> derive(importedField, schema)); - } - - - /** - * Derives the content of this configuration. This - * default calls derive(SDField) for each document field - */ - protected void derive(SDDocumentType document, Schema schema) { - for (Field field : document.fieldSet()) { - SDField sdField = (SDField) field; - if ( ! sdField.isExtraField()) { - derive(sdField, schema); - } - } - } - - /** - * Derives the content of this configuration. This - * default does nothing. - */ - protected void derive(ImmutableSDField field, Schema schema) {} - - /** - * Derives the content of this configuration. This - * default does nothing. - */ - protected void derive(Index index, Schema schema) { - } - - protected abstract String getDerivedName(); - - /** Returns the value of getName if true, the given number as a string otherwise */ - protected String getIndex(int number, boolean labels) { - return labels ? getName() : String.valueOf(number); - } - - /** - * Exports this derived configuration to its .cfg file - * in toDirectory - * - * @param toDirectory the directory to export to, or null - * - */ - public final void export(String toDirectory) throws IOException { - Writer writer = null; - try { - String fileName = getDerivedName() + ".cfg"; - if (toDirectory != null) - writer = IOUtils.createWriter(toDirectory + "/" + fileName,false); - try { - exportBuilderConfig(writer); - } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) { - throw new RuntimeException(e); - } - } - finally { - if (writer != null) IOUtils.closeWriter(writer); - } - } - - /** - * Checks what this is a producer of, instantiate that and export to writer - */ - // TODO move to ReflectionUtil, and move that to unexported pkg - private void exportBuilderConfig(Writer writer) throws ReflectiveOperationException, SecurityException, IllegalArgumentException, IOException { - for (Class intf : getClass().getInterfaces()) { - if (ConfigInstance.Producer.class.isAssignableFrom(intf)) { - Class configClass = intf.getEnclosingClass(); - String builderClassName = configClass.getCanonicalName()+"$Builder"; - Class builderClass = Class.forName(builderClassName); - ConfigInstance.Builder builder = (Builder) builderClass.getDeclaredConstructor().newInstance(); - Method getConfig = getClass().getMethod("getConfig", builderClass); - getConfig.invoke(this, builder); - ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder); - List payloadL = ConfigInstance.serialize(inst); - String payload = StringUtilities.implodeMultiline(payloadL); - writer.write(payload); - } - } - } - - @Override - public String getFileName() { - return getDerivedName() + ".cfg"; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java deleted file mode 100644 index c7b9b94a4b2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.document.config.DocumenttypesConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.io.IOUtils; -import com.yahoo.protect.Validator; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.validation.Validation; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.io.IOException; -import java.io.Writer; - -/** - * A set of all derived configuration of a schema. Use this as a facade to individual configurations when - * necessary. - * - * @author bratseth - */ -public class DerivedConfiguration implements AttributesConfig.Producer { - - private final Schema schema; - private Summaries summaries; - private SummaryMap summaryMap; - private Juniperrc juniperrc; - private AttributeFields attributeFields; - private RankProfileList rankProfileList; - private IndexingScript indexingScript; - private IndexInfo indexInfo; - private SchemaInfo schemaInfo; - private VsmFields streamingFields; - private VsmSummary streamingSummary; - private IndexSchema indexSchema; - private ImportedFields importedFields; - private final QueryProfileRegistry queryProfiles; - private final long maxUncommittedMemory; - private final boolean enableBitVectors; - - /** - * Creates a complete derived configuration from a search definition. - * Only used in tests. - * - * @param schema the search to derive a configuration from. Derived objects will be snapshots, but this argument is - * live. Which means that this object will be inconsistent when the given search definition is later - * modified. - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} - */ - public DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry) { - this(schema, rankProfileRegistry, new QueryProfileRegistry()); - } - - DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); - } - - /** - * Creates a complete derived configuration snapshot from a schema. - * - * @param schema the schema to derive a configuration from. Derived objects will be snapshots, but this - * argument is live. Which means that this object will be inconsistent if the given - * schema is later modified. - */ - public DerivedConfiguration(Schema schema, DeployState deployState) { - Validator.ensureNotNull("Schema", schema); - this.schema = schema; - this.queryProfiles = deployState.getQueryProfiles().getRegistry(); - this.maxUncommittedMemory = deployState.getProperties().featureFlags().maxUnCommittedMemory(); - this.enableBitVectors = deployState.getProperties().featureFlags().enableBitVectors(); - if ( ! schema.isDocumentsOnly()) { - streamingFields = new VsmFields(schema); - streamingSummary = new VsmSummary(schema); - } - if ( ! schema.isDocumentsOnly()) { - attributeFields = new AttributeFields(schema); - summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); - summaryMap = new SummaryMap(schema); - juniperrc = new Juniperrc(schema); - rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); - indexingScript = new IndexingScript(schema); - indexInfo = new IndexInfo(schema); - schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries, summaryMap); - indexSchema = new IndexSchema(schema); - importedFields = new ImportedFields(schema); - } - Validation.validate(this, schema); - } - - /** - * Exports a complete set of configuration-server format config files. - * - * @param toDirectory the directory to export to, current dir if null - * @throws IOException if exporting fails, some files may still be created - */ - public void export(String toDirectory) throws IOException { - if (!schema.isDocumentsOnly()) { - summaries.export(toDirectory); - summaryMap.export(toDirectory); - juniperrc.export(toDirectory); - attributeFields.export(toDirectory); - streamingFields.export(toDirectory); - streamingSummary.export(toDirectory); - indexSchema.export(toDirectory); - rankProfileList.export(toDirectory); - indexingScript.export(toDirectory); - indexInfo.export(toDirectory); - importedFields.export(toDirectory); - schemaInfo.export(toDirectory); - } - } - - public static void exportDocuments(DocumentmanagerConfig.Builder documentManagerCfg, String toDirectory) throws IOException { - exportCfg(new DocumentmanagerConfig(documentManagerCfg), toDirectory + "/" + "documentmanager.cfg"); - } - - public static void exportDocuments(DocumenttypesConfig.Builder documentTypesCfg, String toDirectory) throws IOException { - exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg"); - } - - public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException { - exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg"); - } - - public void exportConstants(String toDirectory) throws IOException { - RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); - rankProfileList.getConfig(b); - exportCfg(b.build(), toDirectory + "/" + "ranking-constants.cfg"); - } - - private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { - Writer writer = null; - try { - writer = IOUtils.createWriter(fileName, false); - writer.write(instance.toString()); - writer.write("\n"); - } finally { - if (writer != null) { - IOUtils.closeWriter(writer); - } - } - } - - public Summaries getSummaries() { - return summaries; - } - - public AttributeFields getAttributeFields() { - return attributeFields; - } - - @Override - public void getConfig(AttributesConfig.Builder builder) { - getConfig(builder, AttributeFields.FieldSet.ALL); - } - - public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { - attributeFields.getConfig(builder, fs, maxUncommittedMemory, enableBitVectors); - } - - public IndexingScript getIndexingScript() { - return indexingScript; - } - - public IndexInfo getIndexInfo() { - return indexInfo; - } - - public SchemaInfo getSchemaInfo() { return schemaInfo; } - - public void setIndexingScript(IndexingScript script) { - this.indexingScript = script; - } - - public Schema getSchema() { return schema; } - - public RankProfileList getRankProfileList() { - return rankProfileList; - } - - public VsmSummary getVsmSummary() { - return streamingSummary; - } - - public VsmFields getVsmFields() { - return streamingFields; - } - - public IndexSchema getIndexSchema() { - return indexSchema; - } - - public Juniperrc getJuniperrc() { - return juniperrc; - } - - public SummaryMap getSummaryMap() { - return summaryMap; - } - - public ImportedFields getImportedFields() { - return importedFields; - } - - public QueryProfileRegistry getQueryProfiles() { return queryProfiles; } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java deleted file mode 100644 index c90855896ec..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; -import com.yahoo.document.config.DocumenttypesConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -/** - * Facade for deriving configs from schemas - * - * @author bratseth - */ -public class Deriver { - - public static ApplicationBuilder getSchemaBuilder(List schemas) { - ApplicationBuilder builder = new ApplicationBuilder(); - try { - for (String schema : schemas) - builder.addSchemaFile(schema); - } catch (ParseException | IOException e) { - throw new IllegalArgumentException(e); - } - builder.build(true); - return builder; - } - - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) { - return getDocumentManagerConfig(Collections.singletonList(sd)); - } - - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List schemas) { - return new DocumentManager().produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); - } - - public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { - return getDocumentTypesConfig(Collections.singletonList(schema)); - } - - public static DocumenttypesConfig.Builder getDocumentTypesConfig(List schemas) { - return new DocumentTypes().produce(getSchemaBuilder(schemas).getModel(), new DocumenttypesConfig.Builder()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java deleted file mode 100644 index ee96161d443..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -/** - * Classes exportable to configurations - * - * @author bratseth - */ -public interface Exportable { - - /** - * Exports the configuration of this object - * - * - * @param toDirectory the directory to export to, does not write to disk if null - * @throws java.io.IOException if exporting fails, some files may still be created - */ - public void export(String toDirectory) throws java.io.IOException; - - /** - * The (short) name of the exported file - * @return a String with the (short) name of the exported file - */ - public String getFileName(); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java deleted file mode 100644 index 2a2950f71e9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.collections.Pair; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * The rank settings of a field used for native rank features. - * - * @author geirst - */ -public class FieldRankSettings { - - private final String fieldName; - - private final Map tables = new LinkedHashMap<>(); - - public FieldRankSettings(String fieldName) { - this.fieldName = fieldName; - } - - public void addTable(NativeTable table) { - NativeTable existing = tables.get(table.getType().getName()); - if (existing != null) { - // TODO: Throw? - return; - } - tables.put(table.getType().getName(), table); - } - - public static boolean isIndexFieldTable(NativeTable table) { - return isFieldMatchTable(table) || isProximityTable(table); - } - - public static boolean isAttributeFieldTable(NativeTable table) { - return isAttributeMatchTable(table); - } - - private static boolean isFieldMatchTable(NativeTable table) { - return (table.getType().equals(NativeTable.Type.FIRST_OCCURRENCE) || - table.getType().equals(NativeTable.Type.OCCURRENCE_COUNT)); - } - - private static boolean isAttributeMatchTable(NativeTable table) { - return (table.getType().equals(NativeTable.Type.WEIGHT)); - } - - private static boolean isProximityTable(NativeTable table) { - return (table.getType().equals(NativeTable.Type.PROXIMITY) || - table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY)); - } - - public List> deriveRankProperties() { - List> properties = new ArrayList<>(); - for (NativeTable table : tables.values()) { - if (isFieldMatchTable(table)) - properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); - if (isAttributeMatchTable(table)) - properties.add(new Pair<>("nativeAttributeMatch." + table.getType().getName() + "." + fieldName, table.getName())); - if (isProximityTable(table)) - properties.add(new Pair<>("nativeProximity." + table.getType().getName() + "." + fieldName, table.getName())); - } - return properties; - } - - @Override - public String toString() { - return "rank settings of field " + fieldName; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java deleted file mode 100644 index 39432fd0049..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.vespa.documentmodel.SummaryTransform; - -/** - * The result transformation of a named field - * - * @author bratseth - */ -public class FieldResultTransform { - - private final String fieldName; - - private SummaryTransform transform; - - private final String argument; - - public FieldResultTransform(String fieldName, SummaryTransform transform, String argument) { - this.fieldName = fieldName; - this.transform = transform; - this.argument = argument; - } - - public String getFieldName() { return fieldName; } - - public SummaryTransform getTransform() { return transform; } - - public void setTransform(SummaryTransform transform) { this.transform = transform; } - - /** Returns the argument of this (used as input to the backend docsum rewriter) */ - public String getArgument() { return argument; } - - public int hashCode() { - return fieldName.hashCode() + 11 * transform.hashCode() + 17 * argument.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (! (o instanceof FieldResultTransform)) return false; - FieldResultTransform other = (FieldResultTransform)o; - - return - this.fieldName.equals(other.fieldName) && - this.transform.equals(other.transform) && - this.argument.equals(other.argument); - } - - @Override - public String toString() { - String sourceString = ""; - if ( ! argument.equals(fieldName)) - sourceString = " (argument: " + argument + ")"; - return "field " + fieldName + ": " + transform + sourceString; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java deleted file mode 100644 index 433bfb108d6..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; - -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Constant values for ranking/model execution tied to a rank profile, - * to be distributed as files. - * - * @author bratseth - */ -public class FileDistributedConstants { - - private final Map constants; - - public FileDistributedConstants(FileRegistry fileRegistry, Collection constants) { - Map distributableConstants = new LinkedHashMap<>(); - for (var constant : constants) { - if ( ! constant.valuePath().isPresent()) continue; - - var distributableConstant = new DistributableConstant(constant.name().simpleArgument().get(), - constant.type(), - constant.valuePath().get(), - constant.pathType().get()); - distributableConstant.validate(); - distributableConstant.register(fileRegistry); - distributableConstants.put(distributableConstant.getName(), distributableConstant); - } - this.constants = Collections.unmodifiableMap(distributableConstants); - } - - /** Returns a read-only map of the constants in this indexed by name. */ - public Map asMap() { return constants; } - - public void getConfig(RankingConstantsConfig.Builder builder) { - for (var constant : constants.values()) { - builder.constant(new RankingConstantsConfig.Constant.Builder() - .name(constant.getName()) - .fileref(constant.getFileReference()) - .type(constant.getType())); - } - } - - public static class DistributableConstant extends DistributableResource { - - private final TensorType tensorType; - - public DistributableConstant(String name, TensorType type, String fileName) { - this(name, type, fileName, PathType.FILE); - } - - public DistributableConstant(String name, TensorType type, String fileName, PathType pathType) { - super(name, fileName, pathType); - this.tensorType = type; - validate(); - } - - public TensorType getTensorType() { return tensorType; } - public String getType() { return tensorType.toString(); } - - public void validate() { - super.validate(); - if (tensorType == null) - throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); - if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) - throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + - ": Dense tensor dimensions must have a size"); - } - - @Override - public String toString() { - return super.toString() + "' of type '" + tensorType + "'"; - } - - } - -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedOnnxModels.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedOnnxModels.java deleted file mode 100644 index a1310d6c18e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedOnnxModels.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * ONNX models distributed as files. - * - * @author bratseth - */ -public class FileDistributedOnnxModels { - - private static final Logger log = Logger.getLogger(FileDistributedOnnxModels.class.getName()); - - private final Map models; - - public FileDistributedOnnxModels(FileRegistry fileRegistry, Collection models) { - Map distributableModels = new LinkedHashMap<>(); - for (var model : models) { - model.validate(); - model.register(fileRegistry); - distributableModels.put(model.getName(), model); - } - this.models = Collections.unmodifiableMap(distributableModels); - } - - public Map asMap() { return models; } - - public void getConfig(OnnxModelsConfig.Builder builder) { - for (OnnxModel model : models.values()) { - if ("".equals(model.getFileReference())) - log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way - else { - OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); - modelBuilder.dry_run_on_setup(true); - modelBuilder.name(model.getName()); - modelBuilder.fileref(model.getFileReference()); - model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); - model.getOutputMap().forEach((name, as) -> modelBuilder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); - if (model.getStatelessExecutionMode().isPresent()) - modelBuilder.stateless_execution_mode(model.getStatelessExecutionMode().get()); - if (model.getStatelessInterOpThreads().isPresent()) - modelBuilder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); - if (model.getStatelessIntraOpThreads().isPresent()) - modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); - - builder.model(modelBuilder); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java deleted file mode 100644 index 3b8c0a9cff2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedComplexField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.vespa.config.search.ImportedFieldsConfig; - -import java.util.Optional; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; - -/** - * This class derives imported fields from search definition and produces imported-fields.cfg as needed by the search backend. - * - * @author geirst - */ -public class ImportedFields extends Derived implements ImportedFieldsConfig.Producer { - - private Optional importedFields = Optional.empty(); - - public ImportedFields(Schema schema) { - derive(schema); - } - - @Override - protected void derive(Schema schema) { - importedFields = schema.importedFields(); - } - - @Override - protected String getDerivedName() { - return "imported-fields"; - } - - @Override - public void getConfig(ImportedFieldsConfig.Builder builder) { - if (importedFields.isPresent()) { - importedFields.get().fields().forEach( (name, field) -> considerField(builder, field)); - } - } - - private static boolean isNestedFieldName(String fieldName) { - return fieldName.indexOf('.') != -1; - } - - private static void considerField(ImportedFieldsConfig.Builder builder, ImportedField field) { - if (field instanceof ImportedComplexField) { - considerComplexField(builder, (ImportedComplexField) field); - } else { - considerSimpleField(builder, field); - } - } - - private static void considerComplexField(ImportedFieldsConfig.Builder builder, ImportedComplexField field) { - ImmutableSDField targetField = field.targetField(); - if (GeoPos.isAnyPos(targetField)) { - // no action needed - } else if (isArrayOfSimpleStruct(targetField)) { - considerNestedFields(builder, field); - } else if (isMapOfSimpleStruct(targetField)) { - considerSimpleField(builder, field.getNestedField("key")); - considerNestedFields(builder, field.getNestedField("value")); - } else if (isMapOfPrimitiveType(targetField)) { - considerSimpleField(builder, field.getNestedField("key")); - considerSimpleField(builder, field.getNestedField("value")); - } - } - - private static void considerNestedFields(ImportedFieldsConfig.Builder builder, ImportedField field) { - if (field instanceof ImportedComplexField) { - ImportedComplexField complexField = (ImportedComplexField) field; - complexField.getNestedFields().forEach(nestedField -> considerSimpleField(builder, nestedField)); - } - } - - private static void considerSimpleField(ImportedFieldsConfig.Builder builder, ImportedField field) { - ImmutableSDField targetField = field.targetField(); - String targetFieldName = targetField.getName(); - if (!isNestedFieldName(targetFieldName)) { - if (targetField.doesAttributing()) { - builder.attribute.add(createAttributeBuilder(field)); - } - } else { - Attribute attribute = targetField.getAttribute(); - if (attribute != null) { - builder.attribute.add(createAttributeBuilder(field)); - } - } - } - - private static ImportedFieldsConfig.Attribute.Builder createAttributeBuilder(ImportedField field) { - ImportedFieldsConfig.Attribute.Builder result = new ImportedFieldsConfig.Attribute.Builder(); - result.name(field.fieldName()); - result.referencefield(field.reference().referenceField().getName()); - result.targetfield(field.targetField().getName()); - return result; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java deleted file mode 100644 index c19fc9ca942..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.NumericDataType; -import com.yahoo.document.datatypes.*; - -/** - * A type of an index structure - * - * @author bratseth - */ -public class Index { - - /** The index type enumeration */ - public static class Type { - - public static final Type TEXT=new Type("text"); - public static final Type INT64=new Type("long"); - public static final Type BOOLEANTREE=new Type("booleantree"); - - private String name; - - private Type(String name) { - this.name=name; - } - - public int hashCode() { - return name.hashCode(); - } - - public String getName() { return name; } - - public boolean equals(Object other) { - if ( ! (other instanceof Type)) return false; - return this.name.equals(((Type)other).name); - } - - public String toString() { - return "type: " + name; - } - - } - - /** Sets the right index type from a field type */ - public static Type convertType(DataType fieldType) { - FieldValue fval = fieldType.createFieldValue(); - if (fieldType instanceof NumericDataType) { - return Type.INT64; - } else if (fval instanceof StringFieldValue) { - return Type.TEXT; - } else if (fval instanceof Raw) { - return Type.BOOLEANTREE; - } else if (fval instanceof PredicateFieldValue) { - return Type.BOOLEANTREE; - } else if (fieldType instanceof CollectionDataType) { - return convertType(((CollectionDataType) fieldType).getNestedType()); - } else { - throw new IllegalArgumentException("Don't know which index type to " + - "convert " + fieldType + " to"); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java deleted file mode 100644 index b65af3a8f02..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ /dev/null @@ -1,595 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; -import com.yahoo.document.NumericDataType; -import com.yahoo.document.PrimitiveDataType; -import com.yahoo.document.StructuredDataType; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.processing.ExactMatch; -import com.yahoo.searchdefinition.processing.NGramMatch; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.search.config.IndexInfoConfig; - -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** - * Per-index commands which should be applied to queries prior to searching - * - * @author bratseth - */ -public class IndexInfo extends Derived implements IndexInfoConfig.Producer { - - private static final String CMD_ATTRIBUTE = "attribute"; - private static final String CMD_DEFAULT_POSITION = "default-position"; - private static final String CMD_DYNTEASER = "dynteaser"; - private static final String CMD_FULLURL = "fullurl"; - private static final String CMD_HIGHLIGHT = "highlight"; - private static final String CMD_INDEX = "index"; - private static final String CMD_LOWERCASE = "lowercase"; - private static final String CMD_NORMALIZE = "normalize"; - private static final String CMD_STEM = "stem"; - private static final String CMD_URLHOST = "urlhost"; - private static final String CMD_WORD = "word"; - private static final String CMD_PLAIN_TOKENS = "plain-tokens"; - private static final String CMD_MULTIVALUE = "multivalue"; - private static final String CMD_FAST_SEARCH = "fast-search"; - private static final String CMD_PREDICATE = "predicate"; - private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds"; - private static final String CMD_NUMERICAL = "numerical"; - private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; - private final Set commands = new java.util.LinkedHashSet<>(); - private final Map aliases = new java.util.LinkedHashMap<>(); - private final Map fieldSets; - private Schema schema; - - public IndexInfo(Schema schema) { - this.fieldSets = schema.fieldSets().userFieldSets(); - addIndexCommand("sddocname", CMD_INDEX); - addIndexCommand("sddocname", CMD_WORD); - derive(schema); - } - - @Override - protected void derive(Schema schema) { - super.derive(schema); // Derive per field - this.schema = schema; - // Populate fieldsets with actual field objects, bit late to do that here but - for (FieldSet fs : fieldSets.values()) { - for (String fieldName : fs.getFieldNames()) { - fs.fields().add(schema.getField(fieldName)); - } - } - // Must follow, because index settings overrides field settings - for (Index index : schema.getExplicitIndices()) { - derive(index, schema); - } - - // Commands for summary fields - // TODO: Move to fieldinfo and implement differently. This is not right - for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { - if (summaryField.getTransform().isTeaser()) { - addIndexCommand(summaryField.getName(), CMD_DYNTEASER); - } - if (summaryField.getTransform().isBolded()) { - addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); - } - } - } - - private static boolean isPositionField(ImmutableSDField field) { - return GeoPos.isAnyPos(field); - } - - @Override - protected void derive(ImmutableSDField field, Schema schema) { - derive(field, schema, false); - } - - protected void derive(ImmutableSDField field, Schema schema, boolean inPosition) { - if (field.getDataType().equals(DataType.PREDICATE)) { - addIndexCommand(field, CMD_PREDICATE); - Index index = field.getIndex(field.getName()); - if (index != null) { - BooleanIndexDefinition options = index.getBooleanIndexDefiniton(); - if (options.hasLowerBound() || options.hasUpperBound()) { - addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" + - (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." + - (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]"); - } - } - } - - // Field level aliases - for (Map.Entry e : field.getAliasToName().entrySet()) { - String alias = e.getKey(); - String name = e.getValue(); - addIndexAlias(alias, name); - } - boolean isPosition = isPositionField(field); - if (field.usesStructOrMap()) { - for (ImmutableSDField structField : field.getStructFields()) { - derive(structField, schema, isPosition); // Recursion - } - } - - if (isPosition) { - addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); - } - - addIndexCommand(field, CMD_INDEX); // List the indices - - if (needLowerCase(field)) { - addIndexCommand(field, CMD_LOWERCASE); - } - - if (field.getDataType().isMultivalue()) { - addIndexCommand(field, CMD_MULTIVALUE); - } - - Attribute attribute = field.getAttribute(); - if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { - addIndexCommand(field.getName(), CMD_ATTRIBUTE); - if (attribute != null && attribute.isFastSearch()) - addIndexCommand(field.getName(), CMD_FAST_SEARCH); - } else if (field.doesIndexing()) { - if (stemSomehow(field, schema)) { - addIndexCommand(field, stemCmd(field, schema), new StemmingOverrider(this, schema)); - } - if (normalizeAccents(field)) { - addIndexCommand(field, CMD_NORMALIZE); - } - if (field.getMatching() == null || field.getMatching().getType().equals(MatchType.TEXT)) { - addIndexCommand(field, CMD_PLAIN_TOKENS); - } - } - - if (isUriField(field)) { - addUriIndexCommands(field); - } - - if (field.getDataType().getPrimitiveType() instanceof NumericDataType) { - addIndexCommand(field, CMD_NUMERICAL); - } - - // Explicit commands - for (String command : field.getQueryCommands()) { - addIndexCommand(field, command); - } - - } - - private static boolean isAnyChildString(DataType dataType) { - PrimitiveDataType primitive = dataType.getPrimitiveType(); - if (primitive == PrimitiveDataType.STRING) return true; - if (primitive != null) return false; - if (dataType instanceof StructuredDataType) { - StructuredDataType structured = (StructuredDataType) dataType; - for (Field field : structured.getFields()) { - if (isAnyChildString(field.getDataType())) return true; - } - } else if (dataType instanceof MapDataType) { - MapDataType mapType = (MapDataType) dataType; - return isAnyChildString(mapType.getKeyType()) || isAnyChildString(mapType.getValueType()); - } - return false; - } - - private static boolean needLowerCase(ImmutableSDField field) { - return field.doesIndexing() - || field.doesLowerCasing() - || ((field.doesAttributing() || (field.getAttribute() != null)) - && isAnyChildString(field.getDataType()) - && field.getMatching().getCase().equals(Case.UNCASED)); - } - - static String stemCmd(ImmutableSDField field, Schema schema) { - return CMD_STEM + ":" + field.getStemming(schema).toStemMode(); - } - - private boolean stemSomehow(ImmutableSDField field, Schema schema) { - if (field.getStemming(schema).equals(Stemming.NONE)) return false; - return isTypeOrNested(field, DataType.STRING); - } - - private boolean normalizeAccents(ImmutableSDField field) { - return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); - } - - private boolean isTypeOrNested(ImmutableSDField field, DataType type) { - return field.getDataType().equals(type) || field.getDataType().equals(DataType.getArray(type)) || - field.getDataType().equals(DataType.getWeightedSet(type)); - } - - private boolean isUriField(ImmutableSDField field) { - DataType fieldType = field.getDataType(); - if (DataType.URI.equals(fieldType)) { - return true; - } - if (fieldType instanceof CollectionDataType && - DataType.URI.equals(((CollectionDataType)fieldType).getNestedType())) - { - return true; - } - return false; - } - - private void addUriIndexCommands(ImmutableSDField field) { - String fieldName = field.getName(); - addIndexCommand(fieldName, CMD_FULLURL); - addIndexCommand(fieldName, CMD_LOWERCASE); - addIndexCommand(fieldName + "." + fieldName, CMD_FULLURL); - addIndexCommand(fieldName + "." + fieldName, CMD_LOWERCASE); - addIndexCommand(fieldName + ".path", CMD_FULLURL); - addIndexCommand(fieldName + ".path", CMD_LOWERCASE); - addIndexCommand(fieldName + ".query", CMD_FULLURL); - addIndexCommand(fieldName + ".query", CMD_LOWERCASE); - addIndexCommand(fieldName + ".hostname", CMD_URLHOST); - addIndexCommand(fieldName + ".hostname", CMD_LOWERCASE); - - // XXX hack - Index index = field.getIndex("hostname"); - if (index != null) { - addIndexCommand(index, CMD_URLHOST); - } - } - - /** - * Sets a command for all indices of a field - */ - private void addIndexCommand(Index index, String command) { - addIndexCommand(index.getName(), command); - } - - /** - * Sets a command for all indices of a field - */ - private void addIndexCommand(ImmutableSDField field, String command) { - addIndexCommand(field, command, null); - } - - /** - * Sets a command for all indices of a field - */ - private void addIndexCommand(ImmutableSDField field, String command, IndexOverrider overrider) { - if (overrider == null || !overrider.override(field.getName(), command, field)) { - addIndexCommand(field.getName(), command); - } - } - - private void addIndexCommand(String indexName, String command) { - commands.add(new IndexCommand(indexName, command)); - } - - private void addIndexAlias(String alias, String indexName) { - aliases.put(alias, indexName); - } - - /** - * Returns whether a particular command is prsent in this index info - */ - public boolean hasCommand(String indexName, String command) { - return commands.contains(new IndexCommand(indexName, command)); - } - - private boolean notInCommands(String index) { - for (IndexCommand command : commands) { - if (command.getIndex().equals(index)) { - return false; - } - } - return true; - } - - @Override - public void getConfig(IndexInfoConfig.Builder builder) { - IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder(); - iiB.name(getName()); - for (IndexCommand command : commands) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(command.getIndex()) - .command(command.getCommand())); - } - // Make user defined field sets searchable - for (FieldSet fieldSet : fieldSets.values()) { - if (notInCommands(fieldSet.getName())) { - addFieldSetCommands(iiB, fieldSet); - } - } - - for (Map.Entry e : aliases.entrySet()) { - iiB.alias( - new IndexInfoConfig.Indexinfo.Alias.Builder() - .alias(e.getKey()) - .indexname(e.getValue())); - } - builder.indexinfo(iiB); - } - - // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. - private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { - for (String qc : fieldSet.queryCommands()) - iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder().indexname(fieldSet.getName()).command(qc)); - boolean anyIndexing = false; - boolean anyAttributing = false; - boolean anyLowerCasing = false; - boolean anyStemming = false; - boolean anyNormalizing = false; - String phraseSegmentingCommand = null; - String stemmingCommand = null; - Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching - // First a pass over the fields to read some params to decide field settings implicitly: - for (ImmutableSDField field : fieldSet.fields()) { - if (field.doesIndexing()) { - anyIndexing = true; - } - if (field.doesAttributing()) { - anyAttributing = true; - } - if (needLowerCase(field)) { - anyLowerCasing = true; - } - if (stemming(field)) { - anyStemming = true; - stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode(); - } - if (field.getNormalizing().doRemoveAccents()) { - anyNormalizing = true; - } - if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) { - fieldSetMatching = field.getMatching(); - } - Optional explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst(); - if (explicitPhraseSegmentingCommand.isPresent()) { - phraseSegmentingCommand = explicitPhraseSegmentingCommand.get(); - } - } - if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) { - // We have both attributes and indexes and no explicit match setting -> - // use default matching as that at least works if the data in the attribute consists - // of single tokens only. - fieldSetMatching = new Matching(); - } - if (anyLowerCasing) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_LOWERCASE)); - } - if (hasMultiValueField(fieldSet)) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_MULTIVALUE)); - } - if (anyIndexing) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_INDEX)); - if ( ! isExactMatch(fieldSetMatching)) { - if (fieldSetMatching == null || fieldSetMatching.getType().equals(MatchType.TEXT)) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_PLAIN_TOKENS)); - } - if (anyStemming) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(stemmingCommand)); - } - if (anyNormalizing) - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_NORMALIZE)); - if (phraseSegmentingCommand != null) - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(phraseSegmentingCommand)); - } - } else { - // Assume only attribute fields - iiB - .command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_ATTRIBUTE)) - .command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_INDEX)); - } - if (fieldSetMatching != null) { - // Explicit matching set on fieldset - if (fieldSetMatching.getType().equals(MatchType.EXACT)) { - String term = fieldSetMatching.getExactMatchTerminator(); - if (term==null) term=ExactMatch.DEFAULT_EXACT_TERMINATOR; - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command("exact "+term)); - } else if (fieldSetMatching.getType().equals(MatchType.WORD)) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(CMD_WORD)); - } else if (fieldSetMatching.getType().equals(MatchType.GRAM)) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); - } else if (fieldSetMatching.getType().equals(MatchType.TEXT)) { - - } - } - } - - private boolean hasMultiValueField(FieldSet fieldSet) { - for (ImmutableSDField field : fieldSet.fields()) { - if (field.getDataType().isMultivalue()) - return true; - } - return false; - } - - private Stemming getEffectiveStemming(ImmutableSDField field) { - Stemming active = field.getStemming(schema); - if (field.getIndex(field.getName()) != null) { - if (field.getIndex(field.getName()).getStemming()!=null) { - active = field.getIndex(field.getName()).getStemming(); - } - } - if (active != null) { - return active; - } - return Stemming.BEST; // assume default - } - - private boolean stemming(ImmutableSDField field) { - if (field.getStemming() != null) { - return !field.getStemming().equals(Stemming.NONE); - } - if (schema.getStemming() == Stemming.NONE) return false; - if (field.isImportedField()) return false; - if (field.getIndex(field.getName())==null) return true; - if (field.getIndex(field.getName()).getStemming()==null) return true; - return !(field.getIndex(field.getName()).getStemming().equals(Stemming.NONE)); - } - - private boolean isExactMatch(Matching m) { - if (m == null) return false; - if (m.getType().equals(MatchType.EXACT)) return true; - if (m.getType().equals(MatchType.WORD)) return true; - return false; - } - - @Override - protected String getDerivedName() { - return "index-info"; - } - - /** - * An index command. Null commands are also represented, to detect consistency issues. This is an (immutable) value - * object. - */ - public static class IndexCommand { - - private String index; - - private String command; - - public IndexCommand(String index, String command) { - this.index = index; - this.command = command; - } - - public String getIndex() { - return index; - } - - public String getCommand() { - return command; - } - - /** - * Returns true if this is the null command (do nothing) - */ - public boolean isNull() { - return command.equals(""); - } - - public int hashCode() { - return index.hashCode() + 17 * command.hashCode(); - } - - public boolean equals(Object object) { - if (!(object instanceof IndexCommand)) { - return false; - } - - IndexCommand other = (IndexCommand)object; - return - other.index.equals(this.index) && - other.command.equals(this.command); - } - - public String toString() { - return "index command " + command + " on index " + index; - } - - } - - /** - * A command which may override the command setting of a field for a particular index - */ - private static abstract class IndexOverrider { - - protected IndexInfo owner; - - public IndexOverrider(IndexInfo owner) { - this.owner = owner; - } - - /** - * Override the setting of this index for this field, returns true if overriden, false if this index should be - * set according to the field - */ - public abstract boolean override(String indexName, String command, ImmutableSDField field); - - } - - private static class StemmingOverrider extends IndexOverrider { - - private Schema schema; - - public StemmingOverrider(IndexInfo owner, Schema schema) { - super(owner); - this.schema = schema; - } - - public boolean override(String indexName, String command, ImmutableSDField field) { - if (schema == null) { - return false; - } - - Index index = schema.getIndex(indexName); - if (index == null) { - return false; - } - - Stemming indexStemming = index.getStemming(); - if (indexStemming == null) { - return false; - } - - if (Stemming.NONE.equals(indexStemming)) { - // Add nothing - } else { - owner.addIndexCommand(indexName, CMD_STEM + ":" + indexStemming.toStemMode()); - } - return true; - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java deleted file mode 100644 index ce5731bcdeb..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.document.StructuredDataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.config.search.IndexschemaConfig; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * Deriver of indexschema config containing information of all text index fields with name and data type. - * - * @author geirst - */ -public class IndexSchema extends Derived implements IndexschemaConfig.Producer { - - private final List fields = new ArrayList<>(); - private final Map collections = new LinkedHashMap<>(); - private final Map fieldSets = new LinkedHashMap<>(); - - public IndexSchema(Schema schema) { - fieldSets.putAll(schema.fieldSets().userFieldSets()); - derive(schema); - } - - public boolean containsField(String fieldName) { - return fields.stream().anyMatch(field -> field.getName().equals(fieldName)); - } - - @Override - protected void derive(Schema schema) { - super.derive(schema); - } - - private boolean isTensorField(ImmutableSDField field) { - return field.getDataType() instanceof TensorDataType; - } - - private void deriveIndexFields(ImmutableSDField field, Schema schema) { - // Note: Indexes for tensor fields are NOT part of the index schema for text fields. - if ((!field.doesIndexing() && !field.isIndexStructureField()) || - isTensorField(field)) - { - return; - } - List lst = flattenField(field.asField()); - if (lst.isEmpty()) { - return; - } - String fieldName = field.getName(); - for (Field flatField : lst) { - deriveIndexFields(flatField, schema); - } - if (lst.size() > 1) { - FieldSet fieldSet = new FieldSet(fieldName); - for (Field flatField : lst) { - fieldSet.addFieldName(flatField.getName()); - } - fieldSets.put(fieldName, fieldSet); - } - } - - private void deriveIndexFields(Field field, Schema schema) { - IndexField toAdd = new IndexField(field.getName(), Index.convertType(field.getDataType()), field.getDataType()); - com.yahoo.searchdefinition.Index definedIndex = schema.getIndex(field.getName()); - if (definedIndex != null) { - toAdd.setIndexSettings(definedIndex); - } - fields.add(toAdd); - addFieldToCollection(field.getName(), field.getName()); // implicit - } - - private FieldCollection getCollection(String collectionName) { - FieldCollection retval = collections.get(collectionName); - if (retval == null) { - collections.put(collectionName, new FieldCollection(collectionName)); - return collections.get(collectionName); - } - return retval; - } - - private void addFieldToCollection(String fieldName, String collectionName) { - FieldCollection collection = getCollection(collectionName); - collection.fields.add(fieldName); - } - - @Override - protected void derive(ImmutableSDField field, Schema schema) { - if (field.usesStructOrMap()) { - return; // unsupported - } - deriveIndexFields(field, schema); - } - - @Override - protected String getDerivedName() { - return "indexschema"; - } - - @Override - public void getConfig(IndexschemaConfig.Builder icB) { - for (int i = 0; i < fields.size(); ++i) { - IndexField f = fields.get(i); - IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder() - .name(f.getName()) - .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType())) - .prefix(f.hasPrefix()) - .phrases(f.hasPhrases()) - .positions(f.hasPositions()) - .interleavedfeatures(f.useInterleavedFeatures()); - if (!f.getCollectionType().equals("SINGLE")) { - ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); - } - icB.indexfield(ifB); - } - for (FieldSet fieldSet : fieldSets.values()) { - IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder() - .name(fieldSet.getName()); - for (String f : fieldSet.getFieldNames()) { - fsB.field(new IndexschemaConfig.Fieldset.Field.Builder() - .name(f)); - } - icB.fieldset(fsB); - } - } - - @SuppressWarnings("deprecation") - static List flattenField(Field field) { - DataType fieldType = field.getDataType(); - if (fieldType.getPrimitiveType() != null){ - return Collections.singletonList(field); - } - if (fieldType instanceof ArrayDataType) { - List ret = new LinkedList<>(); - Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType()); - for (Field flatField : flattenField(innerField)) { - ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()))); - } - return ret; - } - if (fieldType instanceof StructuredDataType) { - List ret = new LinkedList<>(); - String fieldName = field.getName(); - for (Field childField : ((StructuredDataType)fieldType).getFields()) { - for (Field flatField : flattenField(childField)) { - ret.add(new Field(fieldName + "." + flatField.getName(), flatField)); - } - } - return ret; - } - throw new UnsupportedOperationException(fieldType.getName()); - } - - public List getFields() { - return fields; - } - - /** - * Representation of an index field with name and data type. - */ - public static class IndexField { - private String name; - private Index.Type type; - private com.yahoo.searchdefinition.Index.Type sdType; // The index type in "user intent land" - private DataType sdFieldType; - private boolean prefix = false; - private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility? - private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility? - private BooleanIndexDefinition boolIndex = null; - // Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. - private boolean interleavedFeatures = false; - - public IndexField(String name, Index.Type type, DataType sdFieldType) { - this.name = name; - this.type = type; - this.sdFieldType = sdFieldType; - } - public void setIndexSettings(com.yahoo.searchdefinition.Index index) { - if (type.equals(Index.Type.TEXT)) { - prefix = index.isPrefix(); - interleavedFeatures = index.useInterleavedFeatures(); - } - sdType = index.getType(); - boolIndex = index.getBooleanIndexDefiniton(); - } - public String getName() { return name; } - public Index.Type getRawType() { return type; } - public String getType() { - return type.equals(Index.Type.INT64) - ? "INT64" : "STRING"; - } - public String getCollectionType() { - return (sdFieldType == null) - ? "SINGLE" - : (sdFieldType instanceof WeightedSetDataType) - ? "WEIGHTEDSET" - : (sdFieldType instanceof ArrayDataType) - ? "ARRAY" - : "SINGLE"; - } - public boolean hasPrefix() { return prefix; } - public boolean hasPhrases() { return phrases; } - public boolean hasPositions() { return positions; } - public boolean useInterleavedFeatures() { return interleavedFeatures; } - - public BooleanIndexDefinition getBooleanIndexDefinition() { - return boolIndex; - } - - /** - * The user set index type - * @return the type - */ - public com.yahoo.searchdefinition.Index.Type getSdType() { - return sdType; - } - } - - /** - * Representation of a collection of fields (aka index, physical view). - */ - @SuppressWarnings({ "UnusedDeclaration" }) - private static class FieldCollection { - - private final String name; - private final List fields = new ArrayList<>(); - - FieldCollection(String name) { - this.name = name; - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java deleted file mode 100644 index 23409729dbb..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; -import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; -import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * An indexing language script derived from a search definition. An indexing script contains a set of indexing - * statements, organized in a composite structure of indexing code snippets. - * - * @author bratseth - */ -public final class IndexingScript extends Derived implements IlscriptsConfig.Producer { - - private final List docFields = new ArrayList<>(); - private final List expressions = new ArrayList<>(); - private List fieldsSettingLanguage; - - public IndexingScript(Schema schema) { - derive(schema); - } - - @Override - protected void derive(Schema schema) { - fieldsSettingLanguage = fieldsSettingLanguage(schema); - if (fieldsSettingLanguage.size() == 1) // Assume this language should be used for all fields - addExpression(fieldsSettingLanguage.get(0).getIndexingScript()); - super.derive(schema); - } - - @Override - protected void derive(ImmutableSDField field, Schema schema) { - if (field.isImportedField()) return; - - if (field.hasFullIndexingDocprocRights()) - docFields.add(field.getName()); - - if (field.usesStructOrMap() && ! GeoPos.isAnyPos(field)) { - return; // unsupported - } - - if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field)) - return; // Already added - - addExpression(field.getIndexingScript()); - } - - private void addExpression(ScriptExpression expression) { - if ( expression.isEmpty()) return; - expressions.add(new StatementExpression(new ClearStateExpression(), new GuardExpression(expression))); - } - - private List fieldsSettingLanguage(Schema schema) { - return schema.allFieldsList().stream() - .filter(field -> ! field.isImportedField()) - .filter(field -> field.containsExpression(SetLanguageExpression.class)) - .collect(Collectors.toList()); - } - - public Iterable expressions() { - return Collections.unmodifiableCollection(expressions); - } - - @Override - public String getDerivedName() { - return "ilscripts"; - } - - @Override - public void getConfig(IlscriptsConfig.Builder configBuilder) { - IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); - ilscriptBuilder.doctype(getName()); - ilscriptBuilder.docfield(docFields); - addContentInOrder(ilscriptBuilder); - configBuilder.ilscript(ilscriptBuilder); - } - - private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { - ArrayList later = new ArrayList<>(); - Set touchedFields = new HashSet<>(); - for (Expression expression : expressions) { - if (modifiesSelf(expression) && ! setsLanguage(expression)) - later.add(expression); - else - ilscriptBuilder.content(expression.toString()); - - FieldScanVisitor fieldFetcher = new FieldScanVisitor(); - fieldFetcher.visit(expression); - touchedFields.addAll(fieldFetcher.touchedFields()); - } - for (Expression exp : later) - ilscriptBuilder.content(exp.toString()); - generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); - } - - private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set touchedFields) { - Set fieldsWithSyntheticStatements = new HashSet<>(docFields); - fieldsWithSyntheticStatements.removeAll(touchedFields); - List orderedFields = new ArrayList<>(fieldsWithSyntheticStatements); - Collections.sort(orderedFields); - for (String fieldName : orderedFields) { - StatementExpression copyField = new StatementExpression(new InputExpression(fieldName), - new PassthroughExpression(fieldName)); - ilscriptBuilder.content(copyField.toString()); - } - } - - private boolean setsLanguage(Expression expression) { - SetsLanguageVisitor visitor = new SetsLanguageVisitor(); - visitor.visit(expression); - return visitor.setsLanguage; - } - - private boolean modifiesSelf(Expression expression) { - ModifiesSelfVisitor visitor = new ModifiesSelfVisitor(); - visitor.visit(expression); - return visitor.modifiesSelf(); - } - - private static class ModifiesSelfVisitor extends ExpressionVisitor { - - private String inputField = null; - private String outputField = null; - - public boolean modifiesSelf() { return outputField != null && outputField.equals(inputField); } - - @Override - protected void doVisit(Expression expression) { - if (modifiesSelf()) return; - - if (expression instanceof InputExpression) { - inputField = ((InputExpression) expression).getFieldName(); - } - if (expression instanceof OutputExpression) { - outputField = ((OutputExpression) expression).getFieldName(); - } - } - } - - private static class SetsLanguageVisitor extends ExpressionVisitor { - - boolean setsLanguage = false; - - @Override - protected void doVisit(Expression expression) { - if (expression instanceof SetLanguageExpression) - setsLanguage = true; - } - - } - - private static class FieldScanVisitor extends ExpressionVisitor { - List touchedFields = new ArrayList(); - List candidates = new ArrayList(); - - @Override - protected void doVisit(Expression exp) { - if (exp instanceof OutputExpression) { - touchedFields.add(((OutputExpression) exp).getFieldName()); - } - if (exp instanceof InputExpression) { - candidates.add(((InputExpression) exp).getFieldName()); - } - if (exp instanceof ZCurveExpression) { - touchedFields.addAll(candidates); - } - } - - Collection touchedFields() { - Collection output = touchedFields; - touchedFields = null; // deny re-use to try and avoid obvious bugs - return output; - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java deleted file mode 100644 index 327d7df4d07..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.config.search.summary.JuniperrcConfig; - -import java.util.Set; - -/** - * Generated juniperrc-config for controlling juniper. - * - * @author Simon Thoresen Hult - */ -public class Juniperrc extends Derived implements JuniperrcConfig.Producer { - - // List of all fields that should be bolded. - private Set boldingFields = new java.util.LinkedHashSet<>(); - - /** - * Constructs a new juniper rc instance for a given search object. This will derive the configuration automatically, - * so there is no need to call {@link #derive(Schema)}. - * - * @param schema The search model to use for deriving. - */ - public Juniperrc(Schema schema) { - derive(schema); - } - - // Inherit doc from Derived. - @Override - protected void derive(Schema schema) { - super.derive(schema); - for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { - if (summaryField.getTransform() == SummaryTransform.BOLDED) { - boldingFields.add(summaryField.getName()); - } - } - } - - // Inherit doc from Derived. - @Override - protected String getDerivedName() { - return "juniperrc"; - } - - @Override - public void getConfig(JuniperrcConfig.Builder builder) { - if (boldingFields.size() != 0) { - builder.prefix(true); - for (String name : boldingFields) { - builder.override(new JuniperrcConfig.Override.Builder() - .fieldname(name) - .length(65536) - .max_matches(1) - .min_length(8192) - .surround_max(65536)); - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java deleted file mode 100644 index 66526684dc4..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.document.RankType; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -/** - * The definition of a rank type used for native rank features. - * - * @author geirst - */ -public class NativeRankTypeDefinition { - - /** The type this defines */ - private RankType type; - - /** The rank tables of this rank type */ - private List rankTables = new java.util.ArrayList<>(); - - public NativeRankTypeDefinition(RankType type) { - this.type = type; - } - - public RankType getType() { - return type; - } - - public void addTable(NativeTable table) { - rankTables.add(table); - } - - /** Returns an unmodifiable list of the tables in this type definition */ - public Iterator rankSettingIterator() { - return Collections.unmodifiableList(rankTables).iterator(); - } - - public String toString() { - return "native definition of rank type '" + type + "'"; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java deleted file mode 100644 index cadd3a9efb0..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.document.RankType; - -import java.util.Collections; -import java.util.Map; - -/** - * A set of rank type definitions used for native rank features. - * - * @author geirst - */ -public class NativeRankTypeDefinitionSet { - - /** The name of this rank definition set */ - private String name; - - /** The unmodifiable rank type implementations in this set */ - private final Map typeDefinitions; - - /** Returns the default rank type (about) */ - public static RankType getDefaultRankType() { return RankType.ABOUT; } - - public NativeRankTypeDefinitionSet(String name) { - this.name = name; - - Map typeDefinitions = new java.util.LinkedHashMap<>(); - typeDefinitions.put(RankType.IDENTITY, createIdentityRankType(RankType.IDENTITY)); - typeDefinitions.put(RankType.ABOUT, createAboutRankType(RankType.ABOUT)); - typeDefinitions.put(RankType.TAGS, createTagsRankType(RankType.TAGS)); - typeDefinitions.put(RankType.EMPTY, createEmptyRankType(RankType.EMPTY)); - this.typeDefinitions = Collections.unmodifiableMap(typeDefinitions); - } - - private NativeRankTypeDefinition createEmptyRankType(RankType type) { - NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); - rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)")); - rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)")); - rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)")); - rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)")); - rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)")); - return rank; - } - - private NativeRankTypeDefinition createAboutRankType(RankType type) { - NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); - rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)")); - rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); - rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)")); - rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)")); - rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)")); - return rank; - } - - private NativeRankTypeDefinition createIdentityRankType(RankType type) { - NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); - rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)")); - rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); - rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)")); - rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)")); - rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)")); - return rank; - } - - private NativeRankTypeDefinition createTagsRankType(RankType type) { - NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); - rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)")); - rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); - rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)")); - rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)")); - rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)")); - return rank; - } - - /** - * Returns a rank type definition if given an existing rank type name, - * or null if given a rank type which has no native implementation (meaning somebody forgot to add it), - */ - public NativeRankTypeDefinition getRankTypeDefinition(RankType type) { - if (type == RankType.DEFAULT) - type = getDefaultRankType(); - return typeDefinitions.get(type); - } - - /** Returns an unmodifiable map of the type definitions in this */ - public Map types() { return typeDefinitions; } - - public String toString() { - return "native rank type definitions " + name; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java deleted file mode 100644 index 3d1f565b793..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -/** - * A named rank table of a certain type. - * - * @author geirst - */ -public class NativeTable { - - private String name; - - private Type type; - - /** A table type enumeration */ - public static class Type { - - public static Type FIRST_OCCURRENCE = new Type("firstOccurrenceTable"); - public static Type OCCURRENCE_COUNT = new Type("occurrenceCountTable"); - public static Type WEIGHT = new Type("weightTable"); - public static Type PROXIMITY = new Type("proximityTable"); - public static Type REVERSE_PROXIMITY = new Type("reverseProximityTable"); - - private String name; - - private Type(String name) { - this.name = name; - } - - public String getName() { return name; } - - public boolean equals(Object object) { - if (!(object instanceof Type)) { - return false; - } - Type other = (Type)object; - return this.name.equals(other.name); - } - - public int hashCode() { - return name.hashCode(); - } - - public String toString() { - return getName(); - } - } - - public NativeTable(Type type, String name) { - this.type = type; - this.name = name; - } - - public Type getType() { return type; } - - public String getName() { return name; } - - public int hashCode() { - return type.hashCode() + 17*name.hashCode(); - } - - public boolean equals(Object object) { - if (! (object instanceof NativeTable)) return false; - NativeTable other = (NativeTable)object; - return other.getName().equals(this.getName()) && other.getType().equals(this.getType()); - } - - public String toString() { - return getType() + ": " + getName(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java deleted file mode 100644 index 081450275d1..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - -/** - * The derived rank profiles of a schema - * - * @author bratseth - */ -public class RankProfileList extends Derived implements RankProfilesConfig.Producer { - - private final Map rankProfiles; - private final FileDistributedConstants constants; - private final LargeRankExpressions largeRankExpressions; - private final FileDistributedOnnxModels onnxModels; - - public static final RankProfileList empty = new RankProfileList(); - - private RankProfileList() { - constants = new FileDistributedConstants(null, List.of()); - largeRankExpressions = new LargeRankExpressions(null); - onnxModels = new FileDistributedOnnxModels(null, List.of()); - rankProfiles = Map.of(); - } - - /** - * Creates a rank profile list - * - * @param schema the schema this is a rank profile from - * @param attributeFields the attribute fields to create a ranking for - */ - public RankProfileList(Schema schema, - LargeRankExpressions largeRankExpressions, - AttributeFields attributeFields, - DeployState deployState) { - setName(schema == null ? "default" : schema.getName()); - this.largeRankExpressions = largeRankExpressions; - this.rankProfiles = deriveRankProfiles(schema, attributeFields, deployState); - this.constants = deriveFileDistributedConstants(schema, rankProfiles.values(), deployState); - this.onnxModels = deriveFileDistributedOnnxModels(schema, rankProfiles.values(), deployState); - } - - private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry, Set processedProfiles) { - return rank.inheritedNames().isEmpty() || - processedProfiles.containsAll(rank.inheritedNames()) || - (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); - } - - private Map deriveRankProfiles(Schema schema, - AttributeFields attributeFields, - DeployState deployState) { - Map rawRankProfiles = new LinkedHashMap<>(); - if (schema != null) { // profiles belonging to a schema have a default profile - RawRankProfile rawRank = new RawRankProfile(deployState.rankProfileRegistry().get(schema, "default"), - largeRankExpressions, - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - attributeFields, - deployState.getProperties()); - rawRankProfiles.put(rawRank.getName(), rawRank); - } - - Map remaining = new LinkedHashMap<>(); - deployState.rankProfileRegistry().rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); - remaining.remove("default"); - while (!remaining.isEmpty()) { - List ready = new ArrayList<>(); - remaining.forEach((name, profile) -> { - if (areDependenciesReady(profile, deployState.rankProfileRegistry(), rawRankProfiles.keySet())) - ready.add(profile); - }); - rawRankProfiles.putAll(processRankProfiles(ready, - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - attributeFields, - deployState.getProperties(), - deployState.getExecutor())); - ready.forEach(rank -> remaining.remove(rank.name())); - } - return rawRankProfiles; - } - - private Map processRankProfiles(List profiles, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - AttributeFields attributeFields, - ModelContext.Properties deployProperties, - ExecutorService executor) { - Map> futureRawRankProfiles = new LinkedHashMap<>(); - for (RankProfile profile : profiles) { - futureRawRankProfiles.put(profile.name(), executor.submit(() -> new RawRankProfile(profile, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties))); - } - try { - Map rawRankProfiles = new LinkedHashMap<>(); - for (Future rawFuture : futureRawRankProfiles.values()) { - RawRankProfile rawRank = rawFuture.get(); - rawRankProfiles.put(rawRank.getName(), rawRank); - } - return rawRankProfiles; - } catch (InterruptedException | ExecutionException e) { - throw new IllegalStateException(e); - } - } - - private static FileDistributedConstants deriveFileDistributedConstants(Schema schema, - Collection rankProfiles, - DeployState deployState) { - Map allFileConstants = new HashMap<>(); - addFileConstants(schema != null ? schema.constants().values() : List.of(), - allFileConstants, - schema != null ? schema.toString() : "[global]"); - for (var profile : rankProfiles) - addFileConstants(profile.compiled().constants().values(), allFileConstants, profile.toString()); - return new FileDistributedConstants(deployState.getFileRegistry(), allFileConstants.values()); - } - - private static void addFileConstants(Collection source, - Map destination, - String sourceName) { - for (var constant : source) { - if (constant.valuePath().isEmpty()) continue; - var existing = destination.get(constant.name()); - if ( existing != null && ! constant.equals(existing)) { - throw new IllegalArgumentException("Duplicate constants: " + sourceName + " have " + constant + - ", but we already have " + existing + - ": Value reference constants must be unique across all rank profiles/models"); - } - destination.put(constant.name(), constant); - } - } - - private static FileDistributedOnnxModels deriveFileDistributedOnnxModels(Schema schema, - Collection rankProfiles, - DeployState deployState) { - Map allModels = new LinkedHashMap<>(); - addOnnxModels(schema != null ? schema.onnxModels().values() : List.of(), - allModels, - schema != null ? schema.toString() : "[global]"); - for (var profile : rankProfiles) - addOnnxModels(profile.compiled().onnxModels().values(), allModels, profile.toString()); - return new FileDistributedOnnxModels(deployState.getFileRegistry(), allModels.values()); - } - - private static void addOnnxModels(Collection source, - Map destination, - String sourceName) { - for (var model : source) { - var existing = destination.get(model.getName()); - if ( existing != null && ! model.equals(existing)) { - throw new IllegalArgumentException("Duplicate onnx model: " + sourceName + " have " + model + - ", but we already have " + existing + - ": Onnx models must be unique across all rank profiles/models"); - } - destination.put(model.getName(), model); - } - } - - public Map getRankProfiles() { return rankProfiles; } - public FileDistributedConstants constants() { return constants; } - public FileDistributedOnnxModels getOnnxModels() { return onnxModels; } - - @Override - public String getDerivedName() { return "rank-profiles"; } - - @Override - public void getConfig(RankProfilesConfig.Builder builder) { - for (RawRankProfile rank : rankProfiles.values() ) { - rank.getConfig(builder); - } - } - - public void getConfig(RankingExpressionsConfig.Builder builder) { - largeRankExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); - } - - public void getConfig(RankingConstantsConfig.Builder builder) { - constants.getConfig(builder); - } - - public void getConfig(OnnxModelsConfig.Builder builder) { - onnxModels.getConfig(builder); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java deleted file mode 100644 index 6a288ab477a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ /dev/null @@ -1,524 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.google.common.collect.ImmutableList; -import com.yahoo.collections.Pair; -import com.yahoo.compress.Compressor; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankExpressionBody; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.config.search.RankProfilesConfig; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.OptionalDouble; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * A rank profile derived from a search definition, containing exactly the features available natively in the server - * - * @author bratseth - */ -public class RawRankProfile implements RankProfilesConfig.Producer { - - /** A reusable compressor with default settings */ - private static final Compressor compressor = new Compressor(); - - private static final String keyEndMarker = "\r="; - private static final String valueEndMarker = "\r\n"; - - private final String name; - private final Compressor.Compression compressedProperties; - - /** The compiled profile this is created from. */ - private final RankProfile compiled; - - /** Creates a raw rank profile from the given rank profile. */ - public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, - QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.name = rankProfile.name(); - compiled = rankProfile.compile(queryProfiles, importedModels); - compressedProperties = compress(new Deriver(compiled, attributeFields, deployProperties, queryProfiles) - .derive(largeExpressions)); - } - - public RankProfile compiled() { return compiled; } - - private Compressor.Compression compress(List> properties) { - StringBuilder b = new StringBuilder(); - for (Pair property : properties) - b.append(property.getFirst()).append(keyEndMarker).append(property.getSecond()).append(valueEndMarker); - return compressor.compress(b.toString().getBytes(StandardCharsets.UTF_8)); - } - - private List> decompress(Compressor.Compression compression) { - String propertiesString = new String(compressor.decompress(compression), StandardCharsets.UTF_8); - if (propertiesString.isEmpty()) return ImmutableList.of(); - - ImmutableList.Builder> properties = new ImmutableList.Builder<>(); - for (int pos = 0; pos < propertiesString.length();) { - int keyEndPos = propertiesString.indexOf(keyEndMarker, pos); - String key = propertiesString.substring(pos, keyEndPos); - pos = keyEndPos + keyEndMarker.length(); - int valueEndPos = propertiesString.indexOf(valueEndMarker, pos); - String value = propertiesString.substring(pos, valueEndPos); - pos = valueEndPos + valueEndMarker.length(); - properties.add(new Pair<>(key, value)); - } - return properties.build(); - } - - public String getName() { return name; } - - private void getRankProperties(RankProfilesConfig.Rankprofile.Builder b) { - RankProfilesConfig.Rankprofile.Fef.Builder fefB = new RankProfilesConfig.Rankprofile.Fef.Builder(); - for (Pair p : decompress(compressedProperties)) - fefB.property(new RankProfilesConfig.Rankprofile.Fef.Property.Builder().name(p.getFirst()).value(p.getSecond())); - b.fef(fefB); - } - - /** - * Returns the properties of this as an unmodifiable list. - * Note: This method is expensive. - */ - public List> configProperties() { return decompress(compressedProperties); } - - @Override - public void getConfig(RankProfilesConfig.Builder builder) { - RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); - getRankProperties(b); - builder.rankprofile(b); - } - - @Override - public String toString() { - return " rank profile " + name; - } - - private static class Deriver { - - private final Map fieldRankSettings = new java.util.LinkedHashMap<>(); - private final Set summaryFeatures; - private final Set matchFeatures; - private final Set rankFeatures; - private final Map featureRenames = new java.util.LinkedHashMap<>(); - private final List rankProperties; - - /** - * Rank properties for weight settings to make these available to feature executors - */ - private final List boostAndWeightRankProperties = new ArrayList<>(); - - private final boolean ignoreDefaultRankFeatures; - private final RankProfile.MatchPhaseSettings matchPhaseSettings; - private final int rerankCount; - private final int keepRankCount; - private final int numThreadsPerSearch; - private final int minHitsPerThread; - private final int numSearchPartitions; - private final double termwiseLimit; - private final OptionalDouble postFilterThreshold; - private final OptionalDouble approximateThreshold; - private final double rankScoreDropLimit; - private final boolean mapBackRankingExpressionFeatures; - - /** - * The rank type definitions used to derive settings for the native rank features - */ - private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); - private final Map attributeTypes; - private final Map inputs; - private final Set filterFields = new java.util.LinkedHashSet<>(); - private final String rankprofileName; - - private RankingExpression firstPhaseRanking; - private RankingExpression secondPhaseRanking; - - /** - * Creates a raw rank profile from the given rank profile - */ - Deriver(RankProfile compiled, - AttributeFields attributeFields, - ModelContext.Properties deployProperties, - QueryProfileRegistry queryProfiles) { - rankprofileName = compiled.name(); - attributeTypes = compiled.getAttributeTypes(); - inputs = compiled.inputs(); - firstPhaseRanking = compiled.getFirstPhaseRanking(); - secondPhaseRanking = compiled.getSecondPhaseRanking(); - summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); - matchFeatures = new LinkedHashSet<>(compiled.getMatchFeatures()); - rankFeatures = compiled.getRankFeatures(); - rerankCount = compiled.getRerankCount(); - matchPhaseSettings = compiled.getMatchPhaseSettings(); - numThreadsPerSearch = compiled.getNumThreadsPerSearch(); - minHitsPerThread = compiled.getMinHitsPerThread(); - numSearchPartitions = compiled.getNumSearchPartitions(); - termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); - postFilterThreshold = compiled.getPostFilterThreshold(); - approximateThreshold = compiled.getApproximateThreshold(); - keepRankCount = compiled.getKeepRankCount(); - rankScoreDropLimit = compiled.getRankScoreDropLimit(); - mapBackRankingExpressionFeatures = deployProperties.featureFlags().avoidRenamingSummaryFeatures(); - ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); - rankProperties = new ArrayList<>(compiled.getRankProperties()); - - Map functions = compiled.getFunctions(); - List functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map functionProperties = new LinkedHashMap<>(); - SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, - Map.of(), - compiled.typeContext(queryProfiles)); - - if (firstPhaseRanking != null) { - functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); - } - if (secondPhaseRanking != null) { - functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext)); - } - - derivePropertiesAndFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); - deriveOnnxModelFunctionsAndFeatures(compiled); - - deriveRankTypeSetting(compiled, attributeFields); - deriveFilterFields(compiled); - deriveWeightProperties(compiled); - } - - private void deriveFilterFields(RankProfile rp) { - filterFields.addAll(rp.allFilterFields()); - } - - private void derivePropertiesAndFeaturesFromFunctions(Map functions, - Map functionProperties, - SerializationContext functionContext) { - if (functions.isEmpty()) return; - - replaceFunctionFeatures(summaryFeatures, functionContext); - replaceFunctionFeatures(matchFeatures, functionContext); - - // First phase, second phase and summary features should add all required functions to the context. - // However, we need to add any functions not referenced in those anyway for model-evaluation. - deriveFunctionProperties(functions, functionProperties, functionContext); - - for (Map.Entry e : functionProperties.entrySet()) { - rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); - } - } - - private void deriveFunctionProperties(Map functions, - Map functionProperties, - SerializationContext context) { - for (Map.Entry e : functions.entrySet()) { - String propertyName = RankingExpression.propertyName(e.getKey()); - if (context.serializedFunctions().containsKey(propertyName)) continue; - - String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); - - context.addFunctionSerialization(propertyName, expressionString); - for (Map.Entry argumentType : e.getValue().function().argumentTypes().entrySet()) - context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); - if (e.getValue().function().returnType().isPresent()) - context.addFunctionTypeSerialization(e.getKey(), e.getValue().function().returnType().get()); - // else if (e.getValue().function().arguments().isEmpty()) TODO: Enable this check when we resolve all types - // throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); - } - functionProperties.putAll(context.serializedFunctions()); - } - - private void replaceFunctionFeatures(Set features, SerializationContext context) { - if (features == null) return; - Map functionFeatures = new LinkedHashMap<>(); - for (Iterator i = features.iterator(); i.hasNext(); ) { - ReferenceNode referenceNode = i.next(); - // Is the feature a function? - ExpressionFunction function = context.getFunction(referenceNode.getName()); - if (function != null) { - String propertyName = RankingExpression.propertyName(referenceNode.getName()); - String expressionString = function.getBody().getRoot().toString(context).toString(); - context.addFunctionSerialization(propertyName, expressionString); - ReferenceNode backendReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", - referenceNode.getArguments().expressions(), - referenceNode.getOutput()); - if (mapBackRankingExpressionFeatures) { - // tell backend to map back to the name the user expects: - featureRenames.put(backendReferenceNode.toString(), referenceNode.toString()); - } - functionFeatures.put(referenceNode.getName(), backendReferenceNode); - i.remove(); // Will add the expanded one in next block - } - } - // Then, replace the features that were functions - for (Map.Entry e : functionFeatures.entrySet()) { - features.add(e.getValue()); - } - } - - private void deriveWeightProperties(RankProfile rankProfile) { - - for (RankProfile.RankSetting setting : rankProfile.rankSettings()) { - if (setting.getType() != RankProfile.RankSetting.Type.WEIGHT) continue; - boostAndWeightRankProperties.add(new RankProfile.RankProperty("vespa.fieldweight." + setting.getFieldName(), - String.valueOf(setting.getIntValue()))); - } - } - - /** - * Adds the type boosts from a rank profile - */ - private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) { - for (Iterator i = rankProfile.rankSettingIterator(); i.hasNext(); ) { - RankProfile.RankSetting setting = i.next(); - if (setting.getType() != RankProfile.RankSetting.Type.RANKTYPE) continue; - - deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields, - hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); - } - } - - private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, - boolean isDefaultSetting) { - if (isDefaultSetting) return; - - NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType); - if (definition == null) throw new IllegalArgumentException("In field '" + fieldName + "': " + - rankType + " is known but has no implementation. " + - "Supported rank types: " + - nativeRankTypeDefinitions.types().keySet()); - - FieldRankSettings settings = deriveFieldRankSettings(fieldName); - for (Iterator i = definition.rankSettingIterator(); i.hasNext(); ) { - NativeTable table = i.next(); - // only add index field tables if we are processing an index field and - // only add attribute field tables if we are processing an attribute field - if ((FieldRankSettings.isIndexFieldTable(table) && attributeFields.getAttribute(fieldName) == null) || - (FieldRankSettings.isAttributeFieldTable(table) && attributeFields.getAttribute(fieldName) != null)) { - settings.addTable(table); - } - } - } - - private boolean hasDefaultRankTypeSetting(RankProfile rankProfile, String fieldName) { - RankProfile.RankSetting setting = - rankProfile.getRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE); - return setting != null && setting.getValue().equals(RankType.DEFAULT); - } - - private FieldRankSettings deriveFieldRankSettings(String fieldName) { - FieldRankSettings settings = fieldRankSettings.get(fieldName); - if (settings == null) { - settings = new FieldRankSettings(fieldName); - fieldRankSettings.put(fieldName, settings); - } - return settings; - } - - /** Derives the properties this produces */ - public List> derive(LargeRankExpressions largeRankExpressions) { - List> properties = new ArrayList<>(); - for (RankProfile.RankProperty property : rankProperties) { - if (RankingExpression.propertyName(RankProfile.FIRST_PHASE).equals(property.getName())) { - // Could have been set by function expansion. Set expressions, then skip this property. - try { - firstPhaseRanking = new RankingExpression(property.getValue()); - } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse first phase expression", e); - } - } - else if (RankingExpression.propertyName(RankProfile.SECOND_PHASE).equals(property.getName())) { - try { - secondPhaseRanking = new RankingExpression(property.getValue()); - } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse second phase expression", e); - } - } - else { - properties.add(new Pair<>(property.getName(), property.getValue())); - } - } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, RankProfile.FIRST_PHASE)); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, RankProfile.SECOND_PHASE)); - for (FieldRankSettings settings : fieldRankSettings.values()) { - properties.addAll(settings.deriveRankProperties()); - } - for (RankProfile.RankProperty property : boostAndWeightRankProperties) { - properties.add(new Pair<>(property.getName(), property.getValue())); - } - for (ReferenceNode feature : summaryFeatures) { - properties.add(new Pair<>("vespa.summary.feature", feature.toString())); - } - for (ReferenceNode feature : matchFeatures) { - properties.add(new Pair<>("vespa.match.feature", feature.toString())); - } - for (ReferenceNode feature : rankFeatures) { - properties.add(new Pair<>("vespa.dump.feature", feature.toString())); - } - for (var entry : featureRenames.entrySet()) { - properties.add(new Pair<>("vespa.feature.rename", entry.getKey())); - properties.add(new Pair<>("vespa.feature.rename", entry.getValue())); - } - if (numThreadsPerSearch > 0) { - properties.add(new Pair<>("vespa.matching.numthreadspersearch", numThreadsPerSearch + "")); - } - if (minHitsPerThread > 0) { - properties.add(new Pair<>("vespa.matching.minhitsperthread", minHitsPerThread + "")); - } - if (numSearchPartitions >= 0) { - properties.add(new Pair<>("vespa.matching.numsearchpartitions", numSearchPartitions + "")); - } - if (termwiseLimit < 1.0) { - properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + "")); - } - if (postFilterThreshold.isPresent()) { - properties.add(new Pair<>("vespa.matching.global_filter.upper_limit", String.valueOf(postFilterThreshold.getAsDouble()))); - } - if (approximateThreshold.isPresent()) { - properties.add(new Pair<>("vespa.matching.global_filter.lower_limit", String.valueOf(approximateThreshold.getAsDouble()))); - } - if (matchPhaseSettings != null) { - properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); - properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); - properties.add(new Pair<>("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "")); - properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); - properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); - properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); - RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); - if (diversitySettings != null) { - properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); - properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); - } - } - if (rerankCount > -1) { - properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); - } - if (keepRankCount > -1) { - properties.add(new Pair<>("vespa.hitcollector.arraysize", keepRankCount + "")); - } - if (rankScoreDropLimit > -Double.MAX_VALUE) { - properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); - } - if (ignoreDefaultRankFeatures) { - properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); - } - for (String fieldName : filterFields) { - properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); - } - for (Map.Entry attributeType : attributeTypes.entrySet()) { - properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); - } - - for (var input : inputs.values()) { - if (FeatureNames.isQueryFeature(input.name())) { - if (input.type().rank() > 0) // Proton does not like representing the double type as a rank 0 tensor - properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), - input.type().toString())); - if (input.defaultValue().isPresent()) { - properties.add(new Pair<>(input.name().toString(), - input.type().rank() == 0 ? - String.valueOf(input.defaultValue().get().asDouble()) : - input.defaultValue().get().toString(true, false))); - } - } - } - if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); - distributeLargeExpressionsAsFiles(properties, largeRankExpressions); - return properties; - } - - private void distributeLargeExpressionsAsFiles(List> properties, LargeRankExpressions largeRankExpressions) { - for (ListIterator> iter = properties.listIterator(); iter.hasNext();) { - Pair property = iter.next(); - String expression = property.getSecond(); - if (expression.length() > largeRankExpressions.limit()) { - String propertyName = property.getFirst(); - String functionName = RankingExpression.extractScriptName(propertyName); - if (functionName != null) { - String mangledName = rankprofileName + "." + functionName; - largeRankExpressions.add(new RankExpressionBody(mangledName, ByteBuffer.wrap(expression.getBytes(StandardCharsets.UTF_8)))); - iter.set(new Pair<>(RankingExpression.propertyExpressionName(functionName), mangledName)); - } - } - } - } - - private List> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { - List> properties = new ArrayList<>(); - if (expression == null) return properties; - - String name = expression.getName(); - if ("".equals(name)) - name = phase; - - if (expression.getRoot() instanceof ReferenceNode) { - properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); - } else { - properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); - properties.add(new Pair<>(RankingExpression.propertyName(name), expression.getRoot().toString())); - } - return properties; - } - - private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { - if (rankProfile.schema() == null) return; - if (rankProfile.onnxModels().isEmpty()) return; - replaceOnnxFunctionInputs(rankProfile); - replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); - replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); - } - - private void replaceOnnxFunctionInputs(RankProfile rankProfile) { - Set functionNames = rankProfile.getFunctions().keySet(); - if (functionNames.isEmpty()) return; - for (OnnxModel onnxModel: rankProfile.onnxModels().values()) { - for (Map.Entry mapping : onnxModel.getInputMap().entrySet()) { - String source = mapping.getValue(); - if (functionNames.contains(source)) { - onnxModel.addInputNameMapping(mapping.getKey(), "rankingExpression(" + source + ")"); - } - } - } - } - - private void replaceImplicitOnnxConfigFeatures(Set features, RankProfile rankProfile) { - if (features == null || features.isEmpty()) return; - Set replacedFeatures = new HashSet<>(); - for (Iterator i = features.iterator(); i.hasNext(); ) { - ReferenceNode referenceNode = i.next(); - ReferenceNode replacedNode = (ReferenceNode) OnnxModelTransformer.transformFeature(referenceNode, rankProfile); - if (referenceNode != replacedNode) { - replacedFeatures.add(replacedNode); - i.remove(); - } - } - features.addAll(replacedFeatures); - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java deleted file mode 100644 index 0c1e5a76a89..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.config.search.SummarymapConfig; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.search.SearchCluster; - -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; - -/** - * Information about a schema. - * - * @author bratseth - */ -public final class SchemaInfo extends Derived implements SchemaInfoConfig.Producer { - - private final Schema schema; - - // Info about profiles needed in memory after build. - // The rank profile registry itself is not kept around due to its size. - private final Map rankProfiles; - - private final Summaries summaries; - private final SummaryMap summaryMap; - - public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, - Summaries summaries, SummaryMap summaryMap) { - this.schema = schema; - this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); - this.summaries = summaries; - this.summaryMap = summaryMap; - } - - public String name() { return schema.getName(); } - - @Override - public String getDerivedName() { return "schema-info"; } - - public Schema fullSchema() { return schema; } - - public Map rankProfiles() { return rankProfiles; } - - private Map toRankProfiles(Collection rankProfiles) { - Map rankProfileInfos = new LinkedHashMap<>(); - rankProfiles.forEach(profile -> rankProfileInfos.put(profile.name(), new RankProfileInfo(profile))); - return rankProfileInfos; - } - - @Override - public void getConfig(SchemaInfoConfig.Builder builder) { - var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); - schemaBuilder.name(schema.getName()); - addSummaryConfig(schemaBuilder); - addRankProfilesConfig(schemaBuilder); - builder.schema(schemaBuilder); - } - - private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { - for (var summary : summaries.asList()) { - var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); - summaryBuilder.name(summary.getName()); - for (var field : summary.fields().values()) { - var fieldsBuilder = new SchemaInfoConfig.Schema.Summaryclass.Fields.Builder(); - fieldsBuilder.name(field.getName()) - .type(field.getType().getName()) - .dynamic(isDynamic(field.getName())); - summaryBuilder.fields(fieldsBuilder); - } - schemaBuilder.summaryclass(summaryBuilder); - } - } - - /** Returns whether the given field is a dynamic summary field. */ - private boolean isDynamic(String fieldName) { - if (summaryMap == null) return false; // not know for streaming, but also not used - - var fieldTransform = summaryMap.resultTransforms().get(fieldName); - if (fieldTransform == null) return false; - // TODO: Move this into SummaryTransform and call it something else than "dynamic" - return fieldTransform.getTransform().isDynamic() || - fieldTransform.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || - fieldTransform.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER; - } - - private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { - for (RankProfileInfo rankProfile : rankProfiles().values()) { - var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); - for (var input : rankProfile.inputs().entrySet()) { - var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); - inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().type().toString()); - rankProfileConfig.input(inputConfig); - } - schemaBuilder.rankprofile(rankProfileConfig); - } - } - - /** A store of a *small* (in memory) amount of rank profile info. */ - public static final class RankProfileInfo { - - private final String name; - private final boolean hasSummaryFeatures; - private final boolean hasRankFeatures; - private final Map inputs; - - public RankProfileInfo(RankProfile profile) { - this.name = profile.name(); - this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); - this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); - this.inputs = profile.inputs(); - } - - public String name() { return name; } - public boolean hasSummaryFeatures() { return hasSummaryFeatures; } - public boolean hasRankFeatures() { return hasRankFeatures; } - public Map inputs() { return inputs; } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java deleted file mode 100644 index 8703eb11433..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.DataTypeName; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; - -import java.util.*; - -/** - *

A class which can reorder a list of search definitions such that any supertype - * always preceed any subtype. Subject to this condition the given order - * is preserved (the minimal reordering is done).

- * - *

This class is not multithread safe. Only one ordering must be done - * at the time in any instance.

- * - * @author bratseth - * @author bjorncs - */ -public class SearchOrderer { - - /** A map from DataTypeName to the Search defining them */ - private final Map documentNameToSearch = new HashMap<>(); - - /** - * Reorders the given list of search definitions such that any supertype - * always preceed any subtype. Subject to this condition the given order - * is preserved (the minimal reordering is done). - * - * @return a new list containing the same search instances in the right order - */ - public List order(List unordered) { - // Description above state that the original order should be preserved, except for the dependency constraint. - // Yet we botch that guarantee by sorting the list... - unordered.sort(Comparator.comparing(Schema::getName)); - - // No, this is not a fast algorithm... - indexOnDocumentName(unordered); - List ordered = new ArrayList<>(unordered.size()); - List moveOutwards = new ArrayList<>(); - for (Schema schema : unordered) { - if (allDependenciesAlreadyEmitted(ordered, schema)) { - addOrdered(ordered, schema, moveOutwards); - } - else { - moveOutwards.add(schema); - } - } - - // Any leftovers means we have search definitions with undefined inheritants. - // This is warned about elsewhere. - ordered.addAll(moveOutwards); - - documentNameToSearch.clear(); - return ordered; - } - - private void addOrdered(List ordered, Schema schema, List moveOutwards) { - ordered.add(schema); - Schema eligibleMove; - do { - eligibleMove = removeFirstEntryWithFullyEmittedDependencies(moveOutwards, ordered); - if (eligibleMove != null) { - ordered.add(eligibleMove); - } - } while (eligibleMove != null); - } - - /** Removes and returns the first search from the move list which can now be added, or null if none */ - private Schema removeFirstEntryWithFullyEmittedDependencies(List moveOutwards, List ordered) { - for (Schema move : moveOutwards) { - if (allDependenciesAlreadyEmitted(ordered, move)) { - moveOutwards.remove(move); - return move; - } - } - return null; - } - - private boolean allDependenciesAlreadyEmitted(List alreadyOrdered, Schema schema) { - if (schema.getDocument() == null) { - return true; - } - SDDocumentType document = schema.getDocument(); - return allInheritedDependenciesEmitted(alreadyOrdered, document) && allReferenceDependenciesEmitted(alreadyOrdered, document); - } - - private boolean allInheritedDependenciesEmitted(List alreadyOrdered, SDDocumentType document) { - for (SDDocumentType sdoc : document.getInheritedTypes() ) { - DataTypeName inheritedName = sdoc.getDocumentName(); - if ("document".equals(inheritedName.getName())) { - continue; - } - Schema inheritedSchema = documentNameToSearch.get(inheritedName); - if (!alreadyOrdered.contains(inheritedSchema)) { - return false; - } - } - return true; - } - - private static boolean allReferenceDependenciesEmitted(List alreadyOrdered, SDDocumentType document) { - DocumentReferences documentReferences = document.getDocumentReferences() - .orElseThrow(() -> new IllegalStateException("Missing document references. Should have been processed by now.")); - return documentReferences.stream() - .map(Map.Entry::getValue) - .map(DocumentReference::targetSearch) - .allMatch(alreadyOrdered::contains); - } - - private void indexOnDocumentName(List schemas) { - documentNameToSearch.clear(); - for (Schema schema : schemas) { - if (schema.getDocument() != null) { - documentNameToSearch.put(schema.getDocument().getDocumentName(), schema); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java deleted file mode 100644 index e259c07dbb5..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.config.search.SummaryConfig; - -import java.util.ArrayList; -import java.util.List; - -/** - * A list of derived summaries - * - * @author bratseth - */ -public class Summaries extends Derived implements SummaryConfig.Producer { - - private final boolean useV8GeoPositions; - private final List summaries; - - public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) { - super(); - this.useV8GeoPositions = featureFlags.useV8GeoPositions(); - - // Make sure the default is first - List summaries = new ArrayList<>(); - summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger)); - for (DocumentSummary summary : schema.getSummaries().values()) { - if (!summary.getName().equals("default")) - summaries.add(new SummaryClass(schema, summary, deployLogger)); - } - this.summaries = List.copyOf(summaries); - } - - public List asList() { return summaries; } - - @Override - protected String getDerivedName() { return "summary"; } - - @Override - public void getConfig(SummaryConfig.Builder builder) { - builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); - builder.usev8geopositions(useV8GeoPositions); - for (SummaryClass summaryClass : summaries) { - builder.classes(summaryClass.getSummaryClassConfig()); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java deleted file mode 100644 index ea8e2aac1f0..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.config.search.SummaryConfig; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; - -import java.util.Collections; -import java.util.Map; -import java.util.logging.Level; - -/** - * A summary derived from a search definition. - * Each summary definition have at least one summary, the default - * which has the same name as the search definition. - * - * @author bratseth - */ -public class SummaryClass extends Derived { - - public static final String DOCUMENT_ID_FIELD = "documentid"; - - private final int id; - - /** True if this summary class needs to access summary information on disk */ - private boolean accessingDiskSummary = false; - private final boolean rawAsBase64; - private final boolean omitSummaryFeatures; - - /** The summary fields of this indexed by name */ - private final Map fields; - - private final DeployLogger deployLogger; - - /** - * Creates a summary class from a search definition summary - * - * @param deployLogger a {@link DeployLogger} - */ - public SummaryClass(Schema schema, DocumentSummary summary, DeployLogger deployLogger) { - super(summary.getName()); - this.deployLogger = deployLogger; - this.rawAsBase64 = schema.isRawAsBase64(); - this.omitSummaryFeatures = summary.omitSummaryFeatures(); - Map fields = new java.util.LinkedHashMap<>(); - deriveFields(schema, summary, fields); - deriveImplicitFields(summary, fields); - this.fields = Collections.unmodifiableMap(fields); - this.id = deriveId(summary.getName(), fields); - } - - public int id() { return id; } - - /** MUST be called after all other fields are added */ - private void deriveImplicitFields(DocumentSummary summary, Map fields) { - if (summary.getName().equals("default")) { - addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields); - } - } - - private void deriveFields(Schema schema, DocumentSummary summary, Map fields) { - for (SummaryField summaryField : summary.getSummaryFields().values()) { - if (!accessingDiskSummary && schema.isAccessingDiskSummary(summaryField)) { - accessingDiskSummary = true; - } - addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), fields); - } - } - - private void addField(String name, DataType type, Map fields) { - addField(name, type, null, fields); - } - - private void addField(String name, DataType type, - SummaryTransform transform, - Map fields) { - if (fields.containsKey(name)) { - SummaryClassField sf = fields.get(name); - if ( SummaryClassField.convertDataType(type, transform, rawAsBase64) != sf.getType()) { - deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + - ". " + "Declared as type " + sf.getType() + " and " + type); - } - } else { - fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); - } - } - - public Map fields() { return fields; } - - private static int deriveId(String name, Map fields) { - int hash = name.hashCode(); - int number = 1; - for (var field : fields.values()) { - hash += number++ * (field.getName().hashCode() + - 17 * field.getType().getName().hashCode()); - } - hash = Math.abs(hash); - if (hash == DocsumDefinitionSet.SLIME_MAGIC_ID) - hash++; - return hash; - } - - public SummaryConfig.Classes.Builder getSummaryClassConfig() { - SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder(); - classBuilder. - id(id). - name(getName()). - omitsummaryfeatures(omitSummaryFeatures); - for (SummaryClassField field : fields.values() ) { - classBuilder.fields(new SummaryConfig.Classes.Fields.Builder(). - name(field.getName()). - type(field.getType().getName())); - } - return classBuilder; - } - - @Override - public int hashCode() { return id; } - - @Override - protected String getDerivedName() { return "summary"; } - - @Override - public String toString() { - return "summary class '" + getName() + "'"; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java deleted file mode 100644 index e9dd2ac321c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.MapDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.datatypes.BoolFieldValue; -import com.yahoo.document.datatypes.ByteFieldValue; -import com.yahoo.document.datatypes.DoubleFieldValue; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.Float16FieldValue; -import com.yahoo.document.datatypes.FloatFieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.LongFieldValue; -import com.yahoo.document.datatypes.PredicateFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.Struct; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.vespa.documentmodel.SummaryTransform; - -/** - * A summary field derived from a search definition - * - * @author bratseth - */ -public class SummaryClassField { - - private final String name; - - private final Type type; - - /** The summary field type enumeration */ - public enum Type { - - BOOL("bool"), - BYTE("byte"), - SHORT("short"), - INTEGER("integer"), - INT64("int64"), - FLOAT16("float16"), - FLOAT("float"), - DOUBLE("double"), - STRING("string"), - DATA("data"), - RAW("raw"), - LONGSTRING("longstring"), - LONGDATA("longdata"), - XMLSTRING("xmlstring"), - FEATUREDATA("featuredata"), - JSONSTRING("jsonstring"), - TENSOR("tensor"); - - private final String name; - - Type(String name) { - this.name = name; - } - - /** Returns the name of this type */ - public String getName() { - return name; - } - - public String toString() { - return "type: " + name; - } - } - - public SummaryClassField(String name, DataType type, SummaryTransform transform, boolean rawAsBase64) { - this.name = name; - this.type = convertDataType(type, transform, rawAsBase64); - } - - public String getName() { return name; } - - public Type getType() { return type; } - - /** Converts to the right summary field type from a field datatype and a transform*/ - public static Type convertDataType(DataType fieldType, SummaryTransform transform, boolean rawAsBase64) { - FieldValue fval = fieldType.createFieldValue(); - if (fval instanceof StringFieldValue) { - if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) { - return Type.FEATUREDATA; - } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { - return Type.FEATUREDATA; - } else { - return Type.LONGSTRING; - } - } else if (fval instanceof IntegerFieldValue) { - return Type.INTEGER; - } else if (fval instanceof LongFieldValue) { - return Type.INT64; - } else if (fval instanceof Float16FieldValue) { - return Type.FLOAT16; - } else if (fval instanceof FloatFieldValue) { - return Type.FLOAT; - } else if (fval instanceof DoubleFieldValue) { - return Type.DOUBLE; - } else if (fval instanceof BoolFieldValue) { - return Type.BOOL; - } else if (fval instanceof ByteFieldValue) { - return Type.BYTE; - } else if (fval instanceof Raw) { - return rawAsBase64 ? Type.RAW : Type.DATA; - } else if (fval instanceof Struct) { - return Type.JSONSTRING; - } else if (fval instanceof PredicateFieldValue) { - return Type.STRING; - } else if (fval instanceof TensorFieldValue) { - return Type.TENSOR; - } else if (fieldType instanceof CollectionDataType) { - if (transform != null && transform.equals(SummaryTransform.POSITIONS)) { - return Type.XMLSTRING; - } else { - return Type.JSONSTRING; - } - } else if (fieldType instanceof MapDataType) { - return Type.JSONSTRING; - } else if (fieldType instanceof NewDocumentReferenceDataType) { - return Type.LONGSTRING; - } else { - throw new IllegalArgumentException("Don't know which summary type to convert " + fieldType + " to"); - } - } - - public String toString() { - return "summary class field " + name; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java deleted file mode 100644 index 9e1740b4073..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.config.search.SummarymapConfig; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; - -import java.util.Collections; -import java.util.Iterator; -import java.util.Map; - -/** - * A summary map (describing search-time summary field transformations) - * derived from a Schema. - * - * @author bratseth - */ -public class SummaryMap extends Derived implements SummarymapConfig.Producer { - - private final Map resultTransforms = new java.util.LinkedHashMap<>(); - - /** Creates a summary map from a search definition */ - SummaryMap(Schema schema) { - derive(schema); - } - - protected void derive(Schema schema) { - for (DocumentSummary documentSummary : schema.getSummaries().values()) { - derive(documentSummary); - } - super.derive(schema); - } - - @Override - protected void derive(ImmutableSDField field, Schema schema) { - } - - private void derive(DocumentSummary documentSummary) { - for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { - if (summaryField.getTransform()== SummaryTransform.NONE) continue; - - if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || - summaryField.getTransform()==SummaryTransform.DISTANCE || - summaryField.getTransform()==SummaryTransform.GEOPOS || - summaryField.getTransform()==SummaryTransform.POSITIONS || - summaryField.getTransform()==SummaryTransform.MATCHED_ELEMENTS_FILTER || - summaryField.getTransform()==SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) - { - resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getSingleSource())); - } else { - // Note: Currently source mapping is handled in the indexing statement, - // by creating a summary field for each of the values - // This works, but is suboptimal. We could consolidate to a minimal set and - // use the right value from the minimal set as the third parameter here, - // and add "override" commands to multiple static values - resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getName())); - } - } - } - - /** Returns a read-only iterator of the FieldResultTransforms of this summary map */ - public Map resultTransforms() { - return Collections.unmodifiableMap(resultTransforms); - } - - protected String getDerivedName() { return "summarymap"; } - - /** Returns the command name of a transform */ - private String getCommand(SummaryTransform transform) { - if (transform == SummaryTransform.DISTANCE) - return "absdist"; - else if (transform.isDynamic()) - return "dynamicteaser"; - else - return transform.getName(); - } - - /** - * Does this summary command name stand for a dynamic transform? - * We need this because some model information is shared through configs instead of model - see usage - * A dynamic transform needs the query to perform its computations. - */ - // TODO/Note: "dynamic" here means something else than in SummaryTransform - public static boolean isDynamicCommand(String commandName) { - return (commandName.equals("dynamicteaser") || - commandName.equals(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()) || - commandName.equals(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName())); - } - - @Override - public void getConfig(SummarymapConfig.Builder builder) { - builder.defaultoutputclass(-1); - for (FieldResultTransform frt : resultTransforms.values()) { - SummarymapConfig.Override.Builder oB = new SummarymapConfig.Override.Builder() - .field(frt.getFieldName()) - .command(getCommand(frt.getTransform())); - if (frt.getTransform().isDynamic() || - frt.getTransform().equals(SummaryTransform.ATTRIBUTE) || - frt.getTransform().equals(SummaryTransform.DISTANCE) || - frt.getTransform().equals(SummaryTransform.GEOPOS) || - frt.getTransform().equals(SummaryTransform.POSITIONS) || - frt.getTransform().equals(SummaryTransform.TEXTEXTRACTOR) || - frt.getTransform().equals(SummaryTransform.MATCHED_ELEMENTS_FILTER) || - frt.getTransform().equals(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER)) - { - oB.arguments(frt.getArgument()); - } else { - oB.arguments(""); - } - builder.override(oB); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java deleted file mode 100644 index c9ad8ad2838..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.NumericDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.datatypes.BoolFieldValue; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.PredicateFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; - -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Vertical streaming matcher field specification - */ -public class VsmFields extends Derived implements VsmfieldsConfig.Producer { - - private final Map fields=new LinkedHashMap<>(); - private final Map doctypes=new LinkedHashMap<>(); - - public VsmFields(Schema schema) { - addSearchdefinition(schema); - } - - private void addSearchdefinition(Schema schema) { - derive(schema); - } - - @Override - protected void derive(SDDocumentType document, Schema schema) { - super.derive(document, schema); - StreamingDocumentType docType=getDocumentType(document.getName()); - if (docType == null) { - docType = new StreamingDocumentType(document.getName(), schema.fieldSets()); - doctypes.put(document.getName(), docType); - } - for (Object o : document.fieldSet()) { - derive(docType, (SDField) o); - } - } - - protected void derive(StreamingDocumentType document, SDField field) { - if (field.usesStructOrMap()) { - if (GeoPos.isAnyPos(field)) { - StreamingField streamingField = new StreamingField(field); - addField(streamingField.getName(), streamingField); - addFieldToIndices(document, field.getName(), streamingField); - } - for (SDField structField : field.getStructFields()) { - derive(document, structField); // Recursion - } - } else { - if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) ) - return; - - StreamingField streamingField = new StreamingField(field); - addField(streamingField.getName(),streamingField); - deriveIndices(document, field, streamingField); - } - } - - private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField) { - if (field.doesIndexing()) { - addFieldToIndices(document, field.getName(), streamingField); - } else if (field.doesAttributing()) { - for (String indexName : field.getAttributes().keySet()) { - addFieldToIndices(document, indexName, streamingField); - } - } - } - - private void addFieldToIndices(StreamingDocumentType document, String indexName, StreamingField streamingField) { - if (indexName.contains(".")) { - addFieldToIndices(document, indexName.substring(0,indexName.lastIndexOf(".")), streamingField); // Recursion - } - document.addIndexField(indexName, streamingField.getName()); - } - - private void addField(String name, StreamingField field) { - fields.put(name, field); - } - - /** Returns a streaming index, or null if there is none with this name */ - public StreamingDocumentType getDocumentType(String name) { - return doctypes.get(name); - } - - public String getDerivedName() { - return "vsmfields"; - } - - @Override - public void getConfig(VsmfieldsConfig.Builder vsB) { - for (StreamingField streamingField : fields.values()) { - vsB.fieldspec(streamingField.getFieldSpecConfig()); - } - for (StreamingDocumentType streamingDocType : doctypes.values()) { - vsB.documenttype(streamingDocType.getDocTypeConfig()); - } - } - - private static class StreamingField { - - private final String name; - - /** Whether this field does prefix matching by default */ - private final Matching matching; - - /** The type of this field */ - private final Type type; - - private final boolean isAttribute; - - /** The streaming field type enumeration */ - public static class Type { - - public static Type INT8 = new Type("int8","INT8"); - public static Type INT16 = new Type("int16","INT16"); - public static Type INT32 = new Type("int32","INT32"); - public static Type INT64 = new Type("int64","INT64"); - public static Type FLOAT16 = new Type("float16", "FLOAT16"); - public static Type FLOAT = new Type("float","FLOAT"); - public static Type DOUBLE = new Type("double","DOUBLE"); - public static Type STRING = new Type("string","AUTOUTF8"); - public static Type BOOL = new Type("bool","BOOL"); - public static Type UNSEARCHABLESTRING = new Type("string","NONE"); - public static Type GEO_POSITION = new Type("position", "GEOPOS"); - - private String name; - - private String searchMethod; - - private Type(String name, String searchMethod) { - this.name = name; - this.searchMethod = searchMethod; - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - /** Returns the name of this type */ - public String getName() { return name; } - - public String getSearchMethod() { return searchMethod; } - - @Override - public boolean equals(Object other) { - if ( ! (other instanceof Type)) return false; - return this.name.equals(((Type)other).name); - } - - @Override - public String toString() { - return "type: " + name; - } - - } - - public StreamingField(SDField field) { - this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing()); - } - - private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute) { - this.name = name; - this.type = convertType(sourceType); - this.matching = matching; - this.isAttribute = isAttribute; - } - - /** Converts to the right index type from a field datatype */ - private static Type convertType(DataType fieldType) { - FieldValue fval = fieldType.createFieldValue(); - if (fieldType.equals(DataType.FLOAT16)) { - return Type.FLOAT16; - } else if (fieldType.equals(DataType.FLOAT)) { - return Type.FLOAT; - } else if (fieldType.equals(DataType.LONG)) { - return Type.INT64; - } else if (fieldType.equals(DataType.DOUBLE)) { - return Type.DOUBLE; - } else if (fieldType.equals(DataType.BOOL)) { - return Type.BOOL; - } else if (fieldType.equals(DataType.BYTE)) { - return Type.INT8; - } else if (GeoPos.isAnyPos(fieldType)) { - return Type.GEO_POSITION; - } else if (fieldType instanceof NumericDataType) { - return Type.INT32; - } else if (fval instanceof StringFieldValue) { - return Type.STRING; - } else if (fval instanceof BoolFieldValue) { - return Type.BOOL; - } else if (fval instanceof Raw) { - return Type.STRING; - } else if (fval instanceof PredicateFieldValue) { - return Type.UNSEARCHABLESTRING; - } else if (fval instanceof TensorFieldValue) { - return Type.UNSEARCHABLESTRING; - } else if (fieldType instanceof CollectionDataType) { - return convertType(((CollectionDataType) fieldType).getNestedType()); - } else if (fieldType instanceof NewDocumentReferenceDataType) { - return Type.UNSEARCHABLESTRING; - } else { - throw new IllegalArgumentException("Don't know which streaming field type to convert " + - fieldType + " to"); - } - } - - public String getName() { return name; } - - public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() { - VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder(); - String matchingName = matching.getType().getName(); - if (matching.getType().equals(MatchType.TEXT)) - matchingName = ""; - if (matching.getType() != MatchType.EXACT) { - if (matching.isPrefix()) { - matchingName = "prefix"; - } else if (matching.isSubstring()) { - matchingName = "substring"; - } else if (matching.isSuffix()) { - matchingName = "suffix"; - } - } - if (type != Type.STRING) { - matchingName = ""; - } - fB.name(getName()) - .searchmethod(VsmfieldsConfig.Fieldspec.Searchmethod.Enum.valueOf(type.getSearchMethod())) - .arg1(matchingName) - .fieldtype(isAttribute - ? VsmfieldsConfig.Fieldspec.Fieldtype.ATTRIBUTE - : VsmfieldsConfig.Fieldspec.Fieldtype.INDEX); - if (matching.maxLength() != null) { - fB.maxlength(matching.maxLength()); - } - return fB; - } - - @Override - public boolean equals(Object o) { - if (o.getClass().equals(getClass())) { - StreamingField sf = (StreamingField)o; - return name.equals(sf.name) && - matching.equals(sf.matching) && - type.equals(sf.type); - } - return false; - } - - @Override public int hashCode() { - return java.util.Objects.hash(name, matching, type); - } - - } - - private static class StreamingDocumentType { - - private final String name; - private final Map fieldSets = new LinkedHashMap<>(); - private final Map userFieldSets; - - public StreamingDocumentType(String name, FieldSets fieldSets) { - this.name=name; - userFieldSets = fieldSets.userFieldSets(); - } - - public VsmfieldsConfig.Documenttype.Builder getDocTypeConfig() { - VsmfieldsConfig.Documenttype.Builder dtB = new VsmfieldsConfig.Documenttype.Builder(); - dtB.name(name); - Map all = new LinkedHashMap<>(); - all.putAll(fieldSets); - all.putAll(userFieldSets); - for (Map.Entry e : all.entrySet()) { - VsmfieldsConfig.Documenttype.Index.Builder indB = new VsmfieldsConfig.Documenttype.Index.Builder(); - indB.name(e.getValue().getName()); - for (String field : e.getValue().getFieldNames()) { - indB.field(new VsmfieldsConfig.Documenttype.Index.Field.Builder().name(field)); - } - dtB.index(indB); - } - return dtB; - } - - public String getName() { return name; } - - public void addIndexField(String indexName, String fieldName) { - FieldSet fs = fieldSets.get(indexName); - if (fs == null) { - fs = new FieldSet(indexName); - fieldSets.put(indexName, fs); - } - fs.addFieldName(fieldName); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java deleted file mode 100644 index 998343bc467..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; - -import java.util.*; - -/** - * Vertical streaming matcher summary specification - * - * @author bratseth - */ -public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { - - private final Map> summaryMap = new java.util.LinkedHashMap<>(1); - - public VsmSummary(Schema schema) { - derive(schema); - } - - @Override - protected void derive(Schema schema) { - // Use the default class, as it is the superset - derive(schema, schema.getSummary("default")); - } - - private void derive(Schema schema, DocumentSummary documentSummary) { - if (documentSummary == null) return; - for (SummaryField summaryField : documentSummary.getSummaryFields().values()) { - List from = toStringList(summaryField.sourceIterator()); - - if (doMapField(schema, summaryField)) { - SDField sdField = schema.getConcreteField(summaryField.getName()); - if (sdField != null && GeoPos.isAnyPos(sdField)) { - summaryMap.put(summaryField, Collections.singletonList(summaryField.getName())); - } else { - summaryMap.put(summaryField, from); - } - } - } - } - - /** - * Don't include field in map if sources are the same as the struct sub fields for the SDField. - * But do map if not all do summarying. - * Don't map if not struct either. - * @param summaryField a {@link SummaryField} - */ - private boolean doMapField(Schema schema, SummaryField summaryField) { - SDField sdField = schema.getConcreteField(summaryField.getName()); - SDDocumentType document = schema.getDocument(); - if (sdField==null || ((document != null) && (document.getField(summaryField.getName()) == sdField))) { - return true; - } - if (summaryField.getVsmCommand().equals(SummaryField.VsmCommand.FLATTENJUNIPER)) { - return true; - } - if (!sdField.usesStructOrMap()) { - return !(sdField.getName().equals(summaryField.getName())); - } - if (summaryField.getSourceCount()==sdField.getStructFields().size()) { - for (SummaryField.Source source : summaryField.getSources()) { - if (!sdField.getStructFields().contains(new SDField(schema.getDocument(), source.getName(), sdField.getDataType()))) { // equals() uses just name - return true; - } - if (sdField.getStructField(source.getName())!=null && !sdField.getStructField(source.getName()).doesSummarying()) { - return true; - } - } - // The sources in the summary field are the same as the sub-fields in the SD field. - // All sub fields do summarying. - // Don't map. - return false; - } - return true; - } - - private List toStringList(Iterator i) { - List ret = new ArrayList<>(); - while (i.hasNext()) { - ret.add(i.next().getName()); - } - return ret; - } - - @Override - public String getDerivedName() { - return "vsmsummary"; - } - - @Override - public void getConfig(VsmsummaryConfig.Builder vB) { - for (Map.Entry> entry : summaryMap.entrySet()) { - VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName()); - for (String field : entry.getValue()) { - fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field)); - } - fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString())); - vB.fieldmap(fmB); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java deleted file mode 100644 index e908045d777..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.searchdefinition.derived; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java deleted file mode 100644 index 76f52bce2a9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived.validation; - -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.derived.IndexingScript; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; - -/** - * @author Mathias M Lidal - */ -public class IndexStructureValidator extends Validator { - - public IndexStructureValidator(DerivedConfiguration config, Schema schema) { - super(config, schema); - } - - public void validate() { - IndexingScript script = config.getIndexingScript(); - for (Expression exp : script.expressions()) { - new OutputVisitor(schema.getDocument(), exp).visit(exp); - } - } - - private static class OutputVisitor extends ExpressionVisitor { - - final SDDocumentType docType; - final Expression exp; - - public OutputVisitor(SDDocumentType docType, Expression exp) { - this.docType = docType; - this.exp = exp; - } - - @Override - protected void doVisit(Expression exp) { - if (!(exp instanceof OutputExpression)) return; - - String fieldName = ((OutputExpression)exp).getFieldName(); - if (docType.getField(fieldName) != null) return; - - throw new IllegalArgumentException("Indexing expression '" + this.exp + "' refers to field '" + - fieldName + "' which does not exist in the index structure."); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java deleted file mode 100644 index fd75fbb2f48..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived.validation; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; - -public class Validation { - - public static void validate(DerivedConfiguration config, Schema schema) { - new IndexStructureValidator(config, schema).validate(); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java deleted file mode 100644 index 7d3e5b8e9ed..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived.validation; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; - -/** - * @author mathiasm - */ -public abstract class Validator { - - protected DerivedConfiguration config; - protected Schema schema; - - protected Validator(DerivedConfiguration config, Schema schema) { - this.config = config; - this.schema = schema; - } - - public abstract void validate(); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java deleted file mode 100644 index 2c4a2c06d02..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.PrimitiveDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.document.StructuredDataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.document.datatypes.BoolFieldValue; -import com.yahoo.document.datatypes.ByteFieldValue; -import com.yahoo.document.datatypes.DoubleFieldValue; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.FloatFieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.LongFieldValue; -import com.yahoo.document.datatypes.PredicateFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.Float16FieldValue; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.tensor.TensorType; - -import java.io.Serializable; -import java.util.function.Supplier; -import java.util.LinkedHashSet; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * A search-time document attribute (per-document in-memory value). - * This belongs to the field defining the attribute. - * - * @author bratseth - */ -public final class Attribute implements Cloneable, Serializable { - - public enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT, HAMMING } - - // Remember to change hashCode and equals when you add new fields - - private String name; - - private Type type; - private CollectionType collectionType; - - private boolean removeIfZero = false; - private boolean createIfNonExistent = false; - private boolean enableBitVectors = false; - private boolean enableOnlyBitVector = false; - - private boolean fastRank = false; - private boolean fastSearch = false; - private boolean fastAccess = false; - private boolean huge = false; - private boolean mutable = false; - private boolean paged = false; - private int arity = BooleanIndexDefinition.DEFAULT_ARITY; - private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND; - private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND; - private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD; - - /** This is set if the type of this is TENSOR */ - private Optional tensorType = Optional.empty(); - - /** This is set if the type of this is REFERENCE */ - private final Optional referenceDocumentType; - - private Optional distanceMetric = Optional.empty(); - - private Optional hnswIndexParams = Optional.empty(); - - private boolean isPosition = false; - private final Sorting sorting = new Sorting(); - - /** The aliases for this attribute */ - private final Set aliases = new LinkedHashSet<>(); - - private Dictionary dictionary = null; - private Case casing = Case.UNCASED; - - /** - * True if this attribute should be returned during first pass of search. - * Null means make the default decision for this kind of attribute - */ - private Boolean prefetch = null; - - /** The attribute type enumeration */ - public enum Type { - BYTE("byte", "INT8"), - SHORT("short", "INT16"), - INTEGER("integer", "INT32"), - LONG("long", "INT64"), - FLOAT16("float16", "FLOAT16"), - FLOAT("float", "FLOAT"), - DOUBLE("double", "DOUBLE"), - STRING("string", "STRING"), - BOOL("bool", "BOOL"), - PREDICATE("predicate", "PREDICATE"), - TENSOR("tensor", "TENSOR"), - REFERENCE("reference", "REFERENCE"); - - private final String myName; // different from what name() returns. - private final String exportAttributeTypeName; - - Type(String name, String exportAttributeTypeName) { - this.myName=name; - this.exportAttributeTypeName = exportAttributeTypeName; - } - - public String getName() { return myName; } - public String getExportAttributeTypeName() { return exportAttributeTypeName; } - - @Override - public String toString() { - return "type: " + myName; - } - } - - /** The attribute collection type enumeration */ - public enum CollectionType { - - SINGLE("SINGLE"), - ARRAY("ARRAY"), - WEIGHTEDSET ("WEIGHTEDSET"); - - private final String name; - - CollectionType(String name) { - this.name=name; - } - - public String getName() { return name; } - - @Override - public String toString() { - return "collectiontype: " + name; - } - - } - - /** Creates an attribute with default settings */ - public Attribute(String name, DataType fieldType) { - this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); - setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false); - setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false); - } - - public Attribute(String name, Type type, CollectionType collectionType) { - this(name, type, collectionType, Optional.empty(), Optional.empty()); - } - - public Attribute(String name, - Type type, - CollectionType collectionType, - Optional tensorType, - Optional referenceDocumentType) { - this.name=name; - setType(type); - setCollectionType(collectionType); - this.tensorType = tensorType; - this.referenceDocumentType = referenceDocumentType; - } - - public Attribute convertToArray() { - Attribute result = clone(); - result.collectionType = CollectionType.ARRAY; - return result; - } - - /** - *

Returns whether this attribute should be included in the "attributeprefetch" summary - * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. - * - *

Single value attributes are prefetched by default if summary is true. - * Multi value attributes are not.

- */ - public boolean isPrefetch() { - if (prefetch!=null) return prefetch.booleanValue(); - - if (CollectionType.SINGLE.equals(collectionType)) { - return true; - } - - return false; - } - - /** Returns the prefetch value of this, null if the default is used. */ - public Boolean getPrefetchValue() { return prefetch; } - - public boolean isRemoveIfZero() { return removeIfZero; } - public boolean isCreateIfNonExistent() { return createIfNonExistent; } - public boolean isEnabledBitVectors() { return enableBitVectors; } - public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; } - public boolean isFastSearch() { return fastSearch; } - public boolean isFastRank() { return fastRank; } - public boolean isFastAccess() { return fastAccess; } - public boolean isHuge() { return huge; } - public boolean isPaged() { return paged; } - public boolean isPosition() { return isPosition; } - public boolean isMutable() { return mutable; } - - public int arity() { return arity; } - public long lowerBound() { return lowerBound; } - public long upperBound() { return upperBound; } - public double densePostingListThreshold() { return densePostingListThreshold; } - public Optional tensorType() { return tensorType; } - public Optional referenceDocumentType() { return referenceDocumentType; } - - public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; - public DistanceMetric distanceMetric() { - return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); - } - public Optional hnswIndexParams() { return hnswIndexParams; } - - public Sorting getSorting() { return sorting; } - public Dictionary getDictionary() { return dictionary; } - public Case getCase() { return casing; } - - public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } - public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; } - - /** - * Sets whether this should be included in the "attributeprefetch" document summary. - * True or false to override default, null to use default - */ - public void setPrefetch(Boolean prefetch) { this.prefetch = prefetch; } - public void setEnableBitVectors(boolean enableBitVectors) { this.enableBitVectors = enableBitVectors; } - public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; } - public void setFastRank(boolean value) { - Supplier badGen = () -> new IllegalArgumentException("fast-rank is only valid for tensor attributes, invalid for: "+this); - var tt = tensorType.orElseThrow(badGen); - for (var dim : tt.dimensions()) { - if (dim.isMapped()) { - this.fastRank = value; - return; - } - } - throw badGen.get(); - } - public void setFastSearch(boolean fastSearch) { this.fastSearch = fastSearch; } - public void setHuge(boolean huge) { this.huge = huge; } - public void setPaged(boolean paged) { this.paged = paged; } - public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; } - public void setPosition(boolean position) { this.isPosition = position; } - public void setMutable(boolean mutable) { this.mutable = mutable; } - public void setArity(int arity) { this.arity = arity; } - public void setLowerBound(long lowerBound) { this.lowerBound = lowerBound; } - public void setUpperBound(long upperBound) { this.upperBound = upperBound; } - public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } - public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } - public void setDistanceMetric(DistanceMetric metric) { this.distanceMetric = Optional.of(metric); } - public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } - public void setDictionary(Dictionary dictionary) { this.dictionary = dictionary; } - public void setCase(Case casing) { this.casing = casing; } - - public String getName() { return name; } - public Type getType() { return type; } - public CollectionType getCollectionType() { return collectionType; } - - public void setName(String name) { this.name=name; } - private void setType(Type type) { this.type=type; } - public void setCollectionType(CollectionType type) { this.collectionType=type; } - - /** Converts to the right attribute type from a field datatype */ - public static Type convertDataType(DataType fieldType) { - if (fieldType instanceof NewDocumentReferenceDataType) { - return Type.REFERENCE; - } else if (fieldType instanceof CollectionDataType) { - return convertDataType(((CollectionDataType) fieldType).getNestedType()); - } - FieldValue fval = fieldType.createFieldValue(); - if (fval instanceof StringFieldValue) { - return Type.STRING; - } else if (fval instanceof IntegerFieldValue) { - return Type.INTEGER; - } else if (fval instanceof LongFieldValue) { - return Type.LONG; - } else if (fval instanceof FloatFieldValue) { - return Type.FLOAT; - } else if (fval instanceof DoubleFieldValue) { - return Type.DOUBLE; - } else if (fval instanceof BoolFieldValue) { - return Type.BOOL; - } else if (fval instanceof Float16FieldValue) { - return Type.FLOAT16; - } else if (fval instanceof ByteFieldValue) { - return Type.BYTE; - } else if (fval instanceof Raw) { - return Type.BYTE; - } else if (fval instanceof PredicateFieldValue) { - return Type.PREDICATE; - } else if (fval instanceof TensorFieldValue) { - return Type.TENSOR; - } else { - throw new IllegalArgumentException("Don't know which attribute type to convert " - + fieldType + " [" + fieldType.getClass() + "] to"); - } - } - - /** Converts to the right attribute type from a field datatype */ - private static CollectionType convertCollectionType(DataType fieldType) { - if (fieldType instanceof ArrayDataType) { - return CollectionType.ARRAY; - } else if (fieldType instanceof WeightedSetDataType) { - return CollectionType.WEIGHTEDSET; - } else if (fieldType instanceof TensorDataType) { - return CollectionType.SINGLE; - } else if (fieldType instanceof PrimitiveDataType) { - return CollectionType.SINGLE; - } else if (fieldType instanceof NewDocumentReferenceDataType) { - return CollectionType.SINGLE; - } else { - throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType"); - } - } - - private static Optional convertTensorType(DataType fieldType) { - if ( ! ( fieldType instanceof TensorDataType)) return Optional.empty(); - return Optional.of(((TensorDataType)fieldType).getTensorType()); - } - - private static Optional convertTargetType(DataType fieldType) { - return Optional.of(fieldType) - .filter(NewDocumentReferenceDataType.class::isInstance) - .map(NewDocumentReferenceDataType.class::cast) - .map(NewDocumentReferenceDataType::getTargetType); - } - - /** Converts to the right field type from an attribute type */ - private DataType toDataType(Type attributeType) { - switch (attributeType) { - case STRING : return DataType.STRING; - case INTEGER: return DataType.INT; - case LONG: return DataType.LONG; - case FLOAT16: return DataType.FLOAT16; - case FLOAT: return DataType.FLOAT; - case DOUBLE: return DataType.DOUBLE; - case BOOL: return DataType.BOOL; - case BYTE: return DataType.BYTE; - case PREDICATE: return DataType.PREDICATE; - case TENSOR: return DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new)); - case REFERENCE: return createReferenceDataType(); - default: throw new IllegalArgumentException("Unknown attribute type " + attributeType); - } - } - - @SuppressWarnings("deprecation") - private DataType createReferenceDataType() { - if (referenceDocumentType.isEmpty()) { - throw new IllegalStateException("Referenced document type is not set"); - } - StructuredDataType type = referenceDocumentType.get(); - if (type instanceof DocumentType) { - return new NewDocumentReferenceDataType((DocumentType) type); - } else { - return NewDocumentReferenceDataType.forDocumentName(type.getName()); - } - } - - public DataType getDataType() { - DataType dataType = toDataType(type); - if (collectionType == Attribute.CollectionType.ARRAY) { - return DataType.getArray(dataType); - } else if (collectionType == Attribute.CollectionType.WEIGHTEDSET) { - return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero); - } else { - return dataType; - } - } - - @Override - public int hashCode() { - return Objects.hash( - name, type, collectionType, sorting, dictionary, isPrefetch(), fastAccess, removeIfZero, - createIfNonExistent, isPosition, huge, mutable, paged, enableBitVectors, enableOnlyBitVector, - tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); - } - - @Override - public boolean equals(Object object) { - if (! (object instanceof Attribute)) return false; - - Attribute other = (Attribute)object; - if (!this.name.equals(other.name)) return false; - return isCompatible(other); - } - - /** Returns whether these attributes describes the same entity, even if they have different names */ - public boolean isCompatible(Attribute other) { - if (! this.type.equals(other.type)) return false; - if (! this.collectionType.equals(other.collectionType)) return false; - if (this.isPrefetch() != other.isPrefetch()) return false; - if (this.removeIfZero != other.removeIfZero) return false; - if (this.createIfNonExistent != other.createIfNonExistent) return false; - if (this.enableBitVectors != other.enableBitVectors) return false; - if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; - if (this.fastSearch != other.fastSearch) return false; - if (this.huge != other.huge) return false; - if (this.mutable != other.mutable) return false; - if (this.paged != other.paged) return false; - if (! this.sorting.equals(other.sorting)) return false; - if (! Objects.equals(dictionary, other.dictionary)) return false; - if (! Objects.equals(tensorType, other.tensorType)) return false; - if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; - if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; - if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false; - - return true; - } - - @Override - public Attribute clone() { - try { - return (Attribute)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error"); - } - } - - @Override - public String toString() { - return "attribute '" + name + "' (" + type + ")"; - } - - public Set getAliases() { - return aliases; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java deleted file mode 100644 index 3b6228029c7..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalInt; -import java.util.OptionalLong; - -/** - * Encapsulates values required for predicate fields. - * - * @author lesters - */ -public final class BooleanIndexDefinition { - - public static final int DEFAULT_ARITY = 8; - public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE; - public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE; - public static final double DEFAULT_DENSE_POSTING_LIST_THRESHOLD = 0.4; - - private final OptionalInt arity; // mandatory field value - private final OptionalLong lowerBound; - private final OptionalLong upperBound; - private final OptionalDouble densePostingListThreshold; - - public BooleanIndexDefinition(Optional arity, - Optional lowerBound, - Optional upperBound, - Optional densePLT) - { - this.arity = arity.isPresent() ? OptionalInt.of(arity.get()) : OptionalInt.empty(); - this.lowerBound = lowerBound.isPresent() ? OptionalLong.of(lowerBound.get()) : OptionalLong.empty(); - this.upperBound = upperBound.isPresent() ? OptionalLong.of(upperBound.get()) : OptionalLong.empty(); - this.densePostingListThreshold = densePLT.isPresent() ? OptionalDouble.of(densePLT.get()) : OptionalDouble.empty(); - } - - public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound, - OptionalLong upperBound, OptionalDouble densePostingListThreshold) { - this.arity = arity; - this.lowerBound = lowerBound; - this.upperBound = upperBound; - this.densePostingListThreshold = densePostingListThreshold; - } - - public int getArity() { - return arity.getAsInt(); - } - - public boolean hasArity() { - return arity.isPresent(); - } - - public long getLowerBound() { - return lowerBound.orElse(DEFAULT_LOWER_BOUND); - } - - public boolean hasLowerBound() { - return lowerBound.isPresent(); - } - - public long getUpperBound() { - return upperBound.orElse(DEFAULT_UPPER_BOUND); - } - - public boolean hasUpperBound() { - return upperBound.isPresent(); - } - - public double getDensePostingListThreshold() { - return densePostingListThreshold.orElse(DEFAULT_DENSE_POSTING_LIST_THRESHOLD); - } - - public boolean hasDensePostingListThreshold() { - return densePostingListThreshold.isPresent(); - } - - @Override - public String toString() { - return "BooleanIndexDefinition [arity=" + arity + ", lowerBound=" - + lowerBound + ", upperBound=" + upperBound + ", densePostingListThreshold=" - + densePostingListThreshold + "]"; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java deleted file mode 100644 index ef82a498e5e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -/** - * Describes if items should preserve lower/upper case, or shall be uncased - * which normally mean they are all normalized to lowercase. - * @author baldersheim - */ -public enum Case { - CASED("cased"), - UNCASED("uncased"); - private String name; - Case(String name) { this.name = name; } - public String getName() { return name;} -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java deleted file mode 100644 index feac6b9618e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.MapDataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.document.StructDataType; - -/** - * Utils used to check whether a complex field supports being represented as struct field attributes. - * - * Currently we support: - * - array of simple struct - * - map of primitive type to simple struct - * - map of primitive type to primitive type - * - * A simple struct can contain fields of any type, but only fields of primitive type can be defined as - * struct field attributes in the complex field using the simple struct. - * - * @author geirst - */ -public class ComplexAttributeFieldUtils { - - public static boolean isSupportedComplexField(ImmutableSDField field) { - return (isArrayOfSimpleStruct(field) || - isMapOfSimpleStruct(field) || - isMapOfPrimitiveType(field)); - } - - public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { - if (field.getDataType() instanceof ArrayDataType) { - ArrayDataType arrayType = (ArrayDataType)field.getDataType(); - return isStructWithPrimitiveStructFieldAttributes(arrayType.getNestedType(), field); - } else { - return false; - } - } - - public static boolean isMapOfSimpleStruct(ImmutableSDField field) { - if (field.getDataType() instanceof MapDataType) { - MapDataType mapType = (MapDataType)field.getDataType(); - return isPrimitiveType(mapType.getKeyType()) && - isStructWithPrimitiveStructFieldAttributes(mapType.getValueType(), - field.getStructField("value")); - } else { - return false; - } - } - - public static boolean isMapOfPrimitiveType(ImmutableSDField field) { - if (field.getDataType() instanceof MapDataType) { - MapDataType mapType = (MapDataType)field.getDataType(); - return isPrimitiveType(mapType.getKeyType()) && - isPrimitiveType(mapType.getValueType()); - } else { - return false; - } - } - - private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field) { - if (type instanceof StructDataType && ! GeoPos.isPos(type)) { - for (ImmutableSDField structField : field.getStructFields()) { - Attribute attribute = structField.getAttributes().get(structField.getName()); - if (attribute != null) { - if (!isPrimitiveType(attribute)) { - return false; - } - } else if (structField.wasConfiguredToDoAttributing()) { - if (!isPrimitiveType(structField.getDataType())) { - return false; - } - } - } - return true; - } else { - return false; - } - } - - public static boolean isPrimitiveType(Attribute attribute) { - return attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) && - isPrimitiveType(attribute.getDataType()); - } - - public static boolean isPrimitiveType(DataType dataType) { - return dataType.equals(DataType.BYTE) || - dataType.equals(DataType.INT) || - dataType.equals(DataType.LONG) || - dataType.equals(DataType.FLOAT) || - dataType.equals(DataType.DOUBLE) || - dataType.equals(DataType.STRING); - } - - public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { - if (isArrayOfSimpleStruct(field)) { - return hasOnlyStructFieldAttributes(field); - } else if (isMapOfSimpleStruct(field)) { - return hasSingleAttribute(field.getStructField("key")) && - hasOnlyStructFieldAttributes(field.getStructField("value")); - } else if (isMapOfPrimitiveType(field)) { - return hasSingleAttribute(field.getStructField("key")) && - hasSingleAttribute(field.getStructField("value")); - } - return false; - } - - private static boolean hasOnlyStructFieldAttributes(ImmutableSDField field) { - for (ImmutableSDField structField : field.getStructFields()) { - if (!hasSingleAttribute(structField)) { - return false; - } - } - return true; - } - - private static boolean hasSingleAttribute(ImmutableSDField field) { - if (field.getAttributes().size() != 1) { - return false; - } - return (field.getAttributes().get(field.getName()) != null); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java deleted file mode 100644 index 163651bf255..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.document; - -/** - * Represents settings for dictionary control - * - * @author baldersheim - */ -public class Dictionary { - public enum Type { BTREE, HASH, BTREE_AND_HASH }; - private Type type = null; - private Case casing= null; - - public void updateType(Type type) { - if (this.type == null) { - this.type = type; - } else if ((this.type == Type.BTREE) && (type == Type.HASH)) { - this.type = Type.BTREE_AND_HASH; - } else if ((this.type == Type.HASH) && (type == Type.BTREE)) { - this.type = Type.BTREE_AND_HASH; - } else { - throw new IllegalArgumentException("Can not combine previous dictionary setting " + this.type + - " with current " + type); - } - } - public void updateMatch(Case casing) { - if (this.casing != null) { - throw new IllegalArgumentException("dictionary match mode has already been set to " + this.casing); - } - this.casing = casing; - } - public Type getType() { return (type != null) ? type : Type.BTREE; } - public Case getMatch() { return (casing != null) ? casing : Case.UNCASED; } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java deleted file mode 100644 index 944564dbaa1..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.util.LinkedHashSet; -import java.util.Set; -import java.util.TreeSet; - -import static java.util.Comparator.comparing; - -/** - * Searchable collection of fields. - * - * @author baldersheim - */ -public class FieldSet { - - private final String name; - private final Set queryCommands = new LinkedHashSet<>(); - private final Set fieldNames = new TreeSet<>(); - private final Set fields = new TreeSet<>(comparing(ImmutableSDField::asField)); - private Matching matching = null; - - public FieldSet(String name) { this.name = name; } - public String getName() { return name; } - public FieldSet addFieldName(String field) { fieldNames.add(field); return this; } - public Set getFieldNames() { return fieldNames; } - public Set fields() { return fields; } - public Set queryCommands() { return queryCommands; } - - public void setMatching(Matching matching) { - this.matching = matching; - } - - public Matching getMatching() { - return matching; - } - - @Override - public String toString() { return "fieldset '" + name + "'"; } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java deleted file mode 100644 index 956d63a1cdf..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; - -/** - * Common utilities for recognizing fields with the built-in "position" datatype, - * possibly in array form. - * @author arnej - */ -public class GeoPos { - static public boolean isPos(DataType type) { - return PositionDataType.INSTANCE.equals(type); - } - static public boolean isPosArray(DataType type) { - return DataType.getArray(PositionDataType.INSTANCE).equals(type); - } - static public boolean isAnyPos(DataType type) { - return isPos(type) || isPosArray(type); - } - - static public boolean isPos(ImmutableSDField field) { return isPos(field.getDataType()); } - static public boolean isPosArray(ImmutableSDField field) { return isPosArray(field.getDataType()); } - static public boolean isAnyPos(ImmutableSDField field) { return isAnyPos(field.getDataType()); } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java deleted file mode 100644 index 4f717092f56..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.util.Locale; -import java.util.Optional; - -/** - * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. - * - * @author geirst - */ -public class HnswIndexParams { - - public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; - public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; - - private final Optional maxLinksPerNode; - private final Optional neighborsToExploreAtInsert; - private final Optional multiThreadedIndexing; - - public static class Builder { - private Optional maxLinksPerNode = Optional.empty(); - private Optional neighborsToExploreAtInsert = Optional.empty(); - private Optional multiThreadedIndexing = Optional.empty(); - - public void setMaxLinksPerNode(int value) { - maxLinksPerNode = Optional.of(value); - } - public void setNeighborsToExploreAtInsert(int value) { - neighborsToExploreAtInsert = Optional.of(value); - } - public void setMultiThreadedIndexing(boolean value) { - multiThreadedIndexing = Optional.of(value); - } - public HnswIndexParams build() { - return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert, multiThreadedIndexing); - } - } - - public HnswIndexParams() { - this.maxLinksPerNode = Optional.empty(); - this.neighborsToExploreAtInsert = Optional.empty(); - this.multiThreadedIndexing = Optional.empty(); - } - - public HnswIndexParams(Optional maxLinksPerNode, - Optional neighborsToExploreAtInsert, - Optional multiThreadedIndexing) { - this.maxLinksPerNode = maxLinksPerNode; - this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; - this.multiThreadedIndexing = multiThreadedIndexing; - } - - /** - * Creates a new instance where values from the given parameter instance are used where they are present, - * otherwise we use values from this. - */ - public HnswIndexParams overrideFrom(Optional other) { - if (! other.isPresent()) return this; - HnswIndexParams rhs = other.get(); - return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode), - rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert), - rhs.multiThreadedIndexing.or(() -> multiThreadedIndexing)); - } - - public int maxLinksPerNode() { - return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); - } - - public int neighborsToExploreAtInsert() { - return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); - } - - public boolean multiThreadedIndexing() { - return multiThreadedIndexing.orElse(true); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java deleted file mode 100644 index bdd5a4ea899..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.util.Collection; - -import static java.util.stream.Collectors.toList; - -/** - * Wraps {@link ImportedComplexField} as {@link ImmutableSDField}. - */ -public class ImmutableImportedComplexSDField extends ImmutableImportedSDField { - private final ImportedComplexField importedComplexField; - - public ImmutableImportedComplexSDField(ImportedComplexField importedField) { - super(importedField); - importedComplexField = importedField; - } - - @Override - public ImmutableSDField getStructField(String name) { - ImportedField field = importedComplexField.getNestedField(name); - return (field != null) ? field.asImmutableSDField() : null; - } - - @Override - public Collection getStructFields() { - return importedComplexField.getNestedFields().stream().map(field -> field.asImmutableSDField()).collect(toList()); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java deleted file mode 100644 index 71814c31087..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Wraps {@link ImportedField} as {@link ImmutableSDField}. - * Methods that are not meaningful or relevant for imported fields will throw {@link UnsupportedOperationException}. - * - * @author bjorncs - */ -public class ImmutableImportedSDField implements ImmutableSDField { - - private final ImportedField importedField; - - ImmutableImportedSDField(ImportedField importedField) { - this.importedField = importedField; - } - - public ImportedField getImportedField() { - return importedField; - } - - @Override - public boolean containsExpression(Class searchFor) { - throw createUnsupportedException(searchFor.getSimpleName()); - } - - @Override - public boolean doesAttributing() { - return importedField.targetField().doesAttributing(); - } - - @Override - public boolean doesIndexing() { - return importedField.targetField().doesIndexing(); - } - - @Override - public boolean doesLowerCasing() { - return importedField.targetField().doesLowerCasing(); - } - - @Override - public boolean isExtraField() { - return false; - } - - @Override - public boolean isImportedField() { - return true; - } - - @Override - public boolean isIndexStructureField() { - return importedField.targetField().isIndexStructureField(); - } - - @Override - public boolean hasIndex() { - return importedField.targetField().hasIndex(); - } - - @Override - public boolean usesStructOrMap() { - return importedField.targetField().usesStructOrMap(); - } - - @Override - public boolean wasConfiguredToDoAttributing() { - return importedField.targetField().wasConfiguredToDoAttributing(); - } - - @Override - public DataType getDataType() { - return importedField.targetField().getDataType(); - } - - @Override - public SummaryField getSummaryField(String name) { - return importedField.targetField().getSummaryField(name); - } - - @Override - public Index getIndex(String name) { - if ( ! importedField.fieldName().equals(name)) { - throw new IllegalArgumentException("Getting an index (" + name + ") with different name than the imported field (" - + importedField.fieldName() + ") is not supported"); - } - String targetIndexName = importedField.targetField().getName(); - return importedField.targetField().getIndex(targetIndexName); - } - - @Override - public List getQueryCommands() { - return importedField.targetField().getQueryCommands(); - } - - @Override - public Map getAttributes() { - return importedField.targetField().getAttributes(); - } - - @Override - public Attribute getAttribute() { return importedField.targetField().getAttribute(); } - - @Override - public Map getAliasToName() { - return Collections.emptyMap(); - } - - @Override - public ScriptExpression getIndexingScript() { - throw createUnsupportedException("indexing"); - } - - @Override - public Matching getMatching() { - return importedField.targetField().getMatching(); - } - - @Override - public NormalizeLevel getNormalizing() { - return importedField.targetField().getNormalizing(); - } - - @Override - public ImmutableSDField getStructField(String name) { - throw createUnsupportedException("struct"); - } - - @Override - public Collection getStructFields() { - throw createUnsupportedException("struct"); - } - - @Override - public Stemming getStemming() { - return importedField.targetField().getStemming(); - } - - @Override - public Stemming getStemming(Schema schema) { - throw createUnsupportedException("stemming"); - } - - @Override - public Ranking getRanking() { - throw createUnsupportedException("ranking"); - } - - @Override - public Map getSummaryFields() { - throw createUnsupportedException("summary fields"); - } - - @Override - public String getName() { - return importedField.fieldName(); // Name of the imported field, not the target field - } - - @Override - public int getWeight() { - return importedField.targetField().getWeight(); - } - - @Override - public int getLiteralBoost() { - return importedField.targetField().getLiteralBoost(); - } - - @Override - public RankType getRankType() { - return importedField.targetField().getRankType(); - } - - @Override - public Map getIndices() { - return importedField.targetField().getIndices(); - } - - @Override - public boolean existsIndex(String name) { - return importedField.targetField().existsIndex(name); - } - - /** - * Returns a field representation of the imported field. - * Changes to the returned instance are not propagated back to the underlying imported field! - */ - @Override - public Field asField() { - return new Field( - importedField.fieldName(), - importedField.targetField().getDataType()); - } - - private static UnsupportedOperationException createUnsupportedException(String aspect) { - return new UnsupportedOperationException("'" + aspect + "' is not meaningful or relevant for an imported field."); - } - - @Override - public boolean hasFullIndexingDocprocRights() { - return importedField.targetField().hasFullIndexingDocprocRights(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java deleted file mode 100644 index 6d1dee67641..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; - -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * An interface containing the non-mutating methods of {@link SDField}. - * For description of the methods see {@link SDField}. - * - * @author bjorncs - */ -public interface ImmutableSDField { - - boolean containsExpression(Class searchFor); - - boolean doesAttributing(); - - boolean doesIndexing(); - - boolean doesLowerCasing(); - - boolean isExtraField(); - - boolean isImportedField(); - - boolean isIndexStructureField(); - - boolean usesStructOrMap(); - - /** - * Whether this field at some time was configured to do attributing. - * - * This function can typically return a different value than doesAttributing(), - * which uses the final state of the underlying indexing script instead. - */ - boolean wasConfiguredToDoAttributing(); - - DataType getDataType(); - - Index getIndex(String name); - - List getQueryCommands(); - - Map getAttributes(); - - Attribute getAttribute(); - - Map getAliasToName(); - - ScriptExpression getIndexingScript(); - - Matching getMatching(); - - NormalizeLevel getNormalizing(); - - ImmutableSDField getStructField(String name); - - Collection getStructFields(); - - Stemming getStemming(); - - Stemming getStemming(Schema schema); - - Ranking getRanking(); - - String getName(); - - Map getSummaryFields(); - - /** Returns a {@link Field} representation (which is sadly not immutable) */ - Field asField(); - - boolean hasFullIndexingDocprocRights(); - int getWeight(); - int getLiteralBoost(); - RankType getRankType(); - Map getIndices(); - boolean existsIndex(String name); - SummaryField getSummaryField(String name); - boolean hasIndex(); -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java deleted file mode 100644 index 9435f7cdd48..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.DocumentReference; - -import java.util.Collection; -import java.util.Map; - -/** - * A complex field that is imported from a concrete field in a referenced document type and given an alias name. - */ -public class ImportedComplexField extends ImportedField { - - private Map nestedFields; - - public ImportedComplexField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { - super(fieldName, reference, targetField); - nestedFields = new java.util.LinkedHashMap<>(0); - } - - @Override - public ImmutableSDField asImmutableSDField() { - return new ImmutableImportedComplexSDField(this); - } - - public void addNestedField(ImportedField importedField) { - String prefix = fieldName() + "."; - assert(importedField.fieldName().substring(0, prefix.length()).equals(prefix)); - String suffix = importedField.fieldName().substring(prefix.length()); - nestedFields.put(suffix, importedField); - } - - public Collection getNestedFields() { - return nestedFields.values(); - } - - public ImportedField getNestedField(String name) { - if (name.contains(".")) { - String superFieldName = name.substring(0,name.indexOf(".")); - String subFieldName = name.substring(name.indexOf(".")+1); - ImportedField superField = nestedFields.get(superFieldName); - if (superField != null && superField instanceof ImportedComplexField) { - return ((ImportedComplexField)superField).getNestedField(subFieldName); - } - return null; - } - return nestedFields.get(name); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java deleted file mode 100644 index 6f2142c3cea..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.DocumentReference; - -/** - * A field that is imported from a concrete field in a referenced document type and given an alias name. - * - * @author geirst - */ -public abstract class ImportedField { - - private final String fieldName; - private final DocumentReference reference; - private final ImmutableSDField targetField; - - public ImportedField(String fieldName, - DocumentReference reference, - ImmutableSDField targetField) { - this.fieldName = fieldName; - this.reference = reference; - this.targetField = targetField; - } - - public String fieldName() { - return fieldName; - } - - public DocumentReference reference() { - return reference; - } - - public ImmutableSDField targetField() { - return targetField; - } - - public abstract ImmutableSDField asImmutableSDField(); -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java deleted file mode 100644 index 18bd3f43445..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.Schema; - -import java.util.Collections; -import java.util.Map; - -/** - * A set of fields that are imported from concrete fields in referenced document types. - * - * @author geirst - */ -public class ImportedFields { - - private final Map fields; - - public ImportedFields(Map fields) { - this.fields = fields; - } - - public Map fields() { - return Collections.unmodifiableMap(fields); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java deleted file mode 100644 index 6acd1661294..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.DocumentReference; - -/** - * A simple field that is imported from a concrete field in a referenced document type and given an alias name. - */ -public class ImportedSimpleField extends ImportedField { - public ImportedSimpleField(String fieldName, DocumentReference reference, ImmutableSDField targetField) { - super(fieldName, reference, targetField); - } - - @Override - public ImmutableSDField asImmutableSDField() { - return new ImmutableImportedSDField(this); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java deleted file mode 100644 index 545f2e293dc..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -/** Which match algorithm is used by this matching setup */ - -public enum MatchAlgorithm { - NORMAL("normal"), - PREFIX("prefix"), - SUBSTRING("substring"), - SUFFIX("suffix"); - - private String name; - MatchAlgorithm(String name) { this.name = name; } - - public String getName() { return name; } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java deleted file mode 100644 index dd4af0965d8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -public enum MatchType { - TEXT("text"), - WORD("word"), - EXACT("exact"), - GRAM("gram"); - - private String name; - MatchType(String name) { this.name = name; } - - public String getName() { return name; } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java deleted file mode 100644 index ffac35ba61e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.io.Serializable; - -/** - * Defines how a field should be matched. - * Matching objects can be compared based on their content, but they are not immutable. - * - * @author bratseth - */ -public class Matching implements Cloneable, Serializable { - - public static final MatchType defaultType = MatchType.TEXT; - - private MatchType type = MatchType.TEXT; - private Case casing = Case.UNCASED; - - /** The basic match algorithm */ - private MatchAlgorithm algorithm = MatchAlgorithm.NORMAL; - - private boolean typeUserSet = false; - - private boolean algorithmUserSet = false; - - /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ - private int gramSize=-1; - - /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ - private Integer maxLength; - - private String exactMatchTerminator=null; - - /** Creates a matching of type "text" */ - public Matching() {} - - public Matching(MatchType type) { - this.type = type; - } - - public MatchType getType() { return type; } - public Case getCase() { return casing; } - - public void setType(MatchType type) { - this.type = type; - typeUserSet = true; - } - - public void setCase(Case casing) { this.casing = casing; } - - public Integer maxLength() { return maxLength; } - public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } - public boolean isTypeUserSet() { return typeUserSet; } - - public MatchAlgorithm getAlgorithm() { return algorithm; } - - public void setAlgorithm(MatchAlgorithm algorithm) { - this.algorithm = algorithm; - algorithmUserSet = true; - } - - public boolean isAlgorithmUserSet() { return algorithmUserSet; } - - public boolean isPrefix() { return algorithm == MatchAlgorithm.PREFIX; } - - public boolean isSubstring() { return algorithm == MatchAlgorithm.SUBSTRING; } - - public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } - - /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ - public int getGramSize() { return gramSize; } - - public void setGramSize(int gramSize) { this.gramSize=gramSize; } - - /** - * Merge data from another matching object - */ - public void merge(Matching m) { - if (m == null) return; - if (m.isAlgorithmUserSet()) { - this.setAlgorithm(m.getAlgorithm()); - } - if (m.isTypeUserSet()) { - this.setType(m.getType()); - if (m.getType() == MatchType.GRAM) - gramSize = m.gramSize; - } - if (m.getExactMatchTerminator() != null) { - this.setExactMatchTerminator(m.getExactMatchTerminator()); - } - } - - /** - * If exact matching is used, this returns the terminator string - * which terminates an exact matched sequence in queries. If exact - * matching is not used, or no terminator is set, this is null - */ - public String getExactMatchTerminator() { return exactMatchTerminator; } - - /** - * Sets the terminator string which terminates an exact matched - * sequence in queries (used if type is EXACT). - */ - public void setExactMatchTerminator(String exactMatchTerminator) { - this.exactMatchTerminator = exactMatchTerminator; - } - - @Override - public String toString() { - return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + - "], [exact-terminator "+exactMatchTerminator+"]"; - } - - public Matching clone() { - try { - return (Matching)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error"); - } - } - - @Override - public boolean equals(Object o) { - if (! (o instanceof Matching)) return false; - - Matching other=(Matching)o; - if ( ! other.type.equals(this.type)) return false; - if ( ! other.algorithm.equals(this.algorithm)) return false; - if ( this.exactMatchTerminator == null && other.exactMatchTerminator != null) return false; - if ( this.exactMatchTerminator != null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) - return false; - if ( gramSize != other.gramSize) return false; - return true; - } - - @Override public int hashCode() { - return java.util.Objects.hash(type, algorithm, exactMatchTerminator, gramSize); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java deleted file mode 100644 index a1e2986b572..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -/** - * class representing the character normalization - * we want to do on query and indexed text. - * Levels are strict subsets, so doing accent - * removal means doing codepoint normalizing - * and case normalizing also. - */ -// TODO: Missing author -public class NormalizeLevel { - - /** - * The current levels are as follows: - * NONE: no changes to input text - * CODEPOINT: convert text into Unicode - * Normalization Form Compatibility Composition - * LOWERCASE: also convert text into lowercase letters - * ACCENT: do both above and remove accents on characters - */ - public enum Level { - NONE, CODEPOINT, LOWERCASE, ACCENT - } - - private boolean userSpecified = false; - private Level level = Level.ACCENT; - - /** - * Returns whether accents should be removed from text - */ - public boolean doRemoveAccents() { return level == Level.ACCENT; } - - /** - * Construct a default (full) normalizelevel, - */ - public NormalizeLevel() {} - - /** - * Construct for a specific level, possibly user specified - * - * @param level which level to use - * @param fromUser whether this was specified by the user - */ - public NormalizeLevel(Level level, boolean fromUser) { - this.level = level; - this.userSpecified = fromUser; - } - - /** - * Change the current level to CODEPOINT as inferred - * by other features' needs. If the current level - * was user specified it will not change; also this - * will not increase the level. - */ - public void inferCodepoint() { - if (userSpecified) { - // ignore inferred changes if user specified something - return; - } - // do not increase level - if (level != Level.NONE) level = Level.CODEPOINT; - } - - /** - * Change the current level to LOWERCASE as inferred - * by other features' needs. If the current level - * was user specified it will not change; also this - * will not increase the level. - */ - public void inferLowercase() { - if (userSpecified) { - // ignore inferred changes if user specified something - return; - } - // do not increase level - if (level == Level.NONE) return; - if (level == Level.CODEPOINT) return; - - level = Level.LOWERCASE; - } - - public Level getLevel() { - return level; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java deleted file mode 100644 index 6d84a9e97fc..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -/** - * The rank type of a field. For now this is just a container of a string name. - * This class is immutable. - * - * @author bratseth - */ -public enum RankType { - - /** *implicit* default: No type has been set. */ - DEFAULT, - - // Rank types which can be set explicitly. These are defined for Vespa in NativeRankTypeDefinitionSet - IDENTITY, ABOUT, TAGS, EMPTY; - - @Override - public String toString() { - return "rank type " + name().toLowerCase(); - } - - /** - * Returns the rank type from a string, regardless of its case. - * - * @param rankTypeName a rank type name in any casing - * @return the rank type found - * @throws IllegalArgumentException if not found - */ - public static RankType fromString(String rankTypeName) { - try { - return RankType.valueOf(rankTypeName.toUpperCase()); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Unknown rank type '" + rankTypeName + "'. Supported rank types are " + - "'identity', 'about', 'tags' and 'empty'."); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java deleted file mode 100644 index 235677cd05e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.io.Serializable; - -/** - * The rank settings given in a rank clause in the search definition. - * - * @author Vegard Havdal - */ -public class Ranking implements Cloneable, Serializable { - - private boolean literal = false; - private boolean filter = false; - private boolean normal = false; - - /** - *

Returns whether literal (non-stemmed, non-normalized) forms of the words should - * be indexed in a separate index which is searched by a automatically added rank term - * during searches.

- * - *

Default is false.

- */ - public boolean isLiteral() { return literal; } - - public void setLiteral(boolean literal) { this.literal = literal; } - - /** - *

Returns whether this is a filter. Filters will only tell if they are matched or not, - * no detailed relevance information will be available about the match.

- * - *

Matching a filter is much cheaper for the search engine than matching a regular field.

- * - *

Default is false.

- */ - public boolean isFilter() { return filter && !normal; } - - public void setFilter(boolean filter) { this.filter = filter; } - - /** Whether user has explicitly requested normal (non-filter) behavior */ - public boolean isNormal() { return normal; } - public void setNormal(boolean n) { this.normal = n; } - - /** Returns true if the given rank settings are the same */ - @Override - public boolean equals(Object o) { - if ( ! (o instanceof Ranking)) return false; - - Ranking other=(Ranking)o; - if (this.filter != other.filter) return false; - if (this.literal != other.literal) return false; - if (this.normal != other.normal) return false; - return true; - } - - @Override - public int hashCode() { - return java.util.Objects.hash(filter, literal, normal); - } - - @Override - public String toString() { - return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]"; - } - - @Override - public Ranking clone() { - try { - return (Ranking)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error", e); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java deleted file mode 100644 index b87bdd8907e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataType; -import com.yahoo.document.DataTypeName; -import com.yahoo.document.DocumentType; -import com.yahoo.document.Field; -import com.yahoo.document.PositionDataType; -import com.yahoo.document.StructDataType; -import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.document.annotation.AnnotationTypeRegistry; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.searchdefinition.Schema; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** - * A document definition is a list of fields. Documents may inherit other documents, - * implicitly acquiring their fields as it's own. If a document is not set to inherit - * any document, it will always inherit the document "document.0". - * - * @author Thomas Gundersen - * @author bratseth - */ -public class SDDocumentType implements Cloneable, Serializable { - - public static final SDDocumentType VESPA_DOCUMENT; - private final Map inheritedTypes = new LinkedHashMap<>(); - private final Map ownedTypes = new LinkedHashMap<>(); - private final AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry(); - private DocumentType docType; - private DataType structType; - // The field sets here are set from the processing step in SD, - // to ensure that the full Search and this SDDocumentType is built first. - private FieldSets fieldSets; - // Document references - private Optional documentReferences = Optional.empty(); - private TemporaryImportedFields temporaryImportedFields; - - static { - VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); - VESPA_DOCUMENT.addType(createSDDocumentType(PositionDataType.INSTANCE)); - } - - public SDDocumentType clone() throws CloneNotSupportedException { - SDDocumentType type = (SDDocumentType) super.clone(); - type.docType = docType.clone(); - type.inheritedTypes.putAll(inheritedTypes); - type.structType = structType; - // TODO this isn't complete; should it be..?! - return type; - } - - /** - * For adding structs defined in document scope - * - * @param dt the struct to add - * @return self, for chaining - */ - public SDDocumentType addType(SDDocumentType dt) { - NewDocumentType.Name name = new NewDocumentType.Name(dt.getName()); - if (getType(name) != null) - throw new IllegalArgumentException("Data type '" + name + "' has already been used."); - if (name.getName() == docType.getName()) - throw new IllegalArgumentException("Data type '" + name + "' can not have same name as its defining document."); - ownedTypes.put(name, dt); - return this; - } - public final SDDocumentType getOwnedType(String name) { - return getOwnedType(new NewDocumentType.Name(name)); - } - public SDDocumentType getOwnedType(DataTypeName name) { - return getOwnedType(name.getName()); - } - - public SDDocumentType getOwnedType(NewDocumentType.Name name) { - return ownedTypes.get(name); - } - - public final SDDocumentType getType(String name) { - return getType(new NewDocumentType.Name(name)); - } - - public SDDocumentType getType(NewDocumentType.Name name) { - SDDocumentType type = ownedTypes.get(name); - if (type == null) { - for (SDDocumentType inherited : inheritedTypes.values()) { - type = inherited.getType(name); - if (type != null) { - return type; - } - } - } - return type; - } - - public SDDocumentType addAnnotation(AnnotationType annotation) { - annotationTypes.register(annotation); - return this; - } - - /** Returns all owned datatypes. */ - public Collection getTypes() { return ownedTypes.values(); } - - // TODO: Include inherited - public Map getAnnotations() { return annotationTypes.getTypes(); } - public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); } - - public Collection getAllTypes() { - Collection list = new ArrayList<>(); - list.addAll(getTypes()); - for (SDDocumentType inherited : inheritedTypes.values()) { - list.addAll(inherited.getAllTypes()); - } - return list; - } - - public Map allTypes() { - Map map = new LinkedHashMap<>(); - for (SDDocumentType inherited : inheritedTypes.values()) - map.putAll(inherited.allTypes()); - map.putAll(ownedTypes); - return map; - } - - /** - * Creates a new document type. - * The document type id will be generated as a hash from the document type name. - * - * @param name The name of the new document type - */ - public SDDocumentType(String name) { - this(name,null); - } - - public SDDocumentType(DataTypeName name) { - this(name.getName()); - } - - /** - * Creates a new document type. - * The document type id will be generated as a hash from the document type name. - * - * @param name the name of the new document type - * @param schema check for type ID collisions in this search definition - */ - public SDDocumentType(String name, Schema schema) { - docType = new DocumentType(name); - validateId(schema); - inherit(VESPA_DOCUMENT); - } - - public boolean isStruct() { return getStruct() != null; } - public DataType getStruct() { return structType; } - public SDDocumentType setStruct(DataType structType) { - if (structType != null) { - this.structType = structType; - inheritedTypes.remove(VESPA_DOCUMENT.getDocumentName()); - } else { - if (docType.contentStruct() != null) { - this.structType = docType.contentStruct(); - inheritedTypes.clear(); - } else { - throw new IllegalArgumentException("You can not set a null struct"); - } - } - return this; - } - - public String getName() { return docType.getName(); } - public DataTypeName getDocumentName() { return docType.getDataTypeName(); } - public DocumentType getDocumentType() { return docType; } - - public void inherit(DataTypeName name) { - inherit(new TemporarySDDocumentType(name)); - } - - public void inherit(SDDocumentType type) { - if (type == null) return; - if (type.getName().equals(this.getName())) - throw new IllegalArgumentException("Document type '" + getName() + "' cannot inherit itself"); - if ( ! inheritedTypes.containsKey(type.getDocumentName()) || - (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) { - inheritedTypes.put(type.getDocumentName(), type); - } - } - - public Collection getInheritedTypes() { return inheritedTypes.values(); } - - public Map inheritedTypes() { return inheritedTypes; } - - protected void validateId(Schema schema) { - if (schema == null) return; - if (schema.getDocument(getName()) == null) return; - SDDocumentType doc = schema.getDocument(); - throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " + - "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); - } - - public void setFieldId(SDField field, int id) { - field.setId(id, docType); - } - - /** Override getField, as it may need to ask inherited types that isn't registered in document type. */ - public Field getField(String name) { - if (name.contains(".")) { - String superFieldName = name.substring(0,name.indexOf(".")); - String subFieldName = name.substring(name.indexOf(".")+1); - Field f = docType.getField(superFieldName); - if (f != null) { - if (f instanceof SDField) { - SDField superField = (SDField)f; - return superField.getStructField(subFieldName); - } else { - throw new IllegalArgumentException("Field " + f.getName() + " is not an SDField"); - } - } - } - Field f = docType.getField(name); - if (f == null) { - for(SDDocumentType parent : inheritedTypes.values()) { - f = parent.getField(name); - if (f != null) return f; - } - } - return f; - } - - public void addField(Field field) { - verifyInheritance(field); - for (Iterator i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) { - if (field.getName().equalsIgnoreCase((i.next()).getName())) { - throw new IllegalArgumentException("Duplicate (case insensitively) " + field + " in " + this); - } - } - docType.addField(field); - } - - /** Parse-time inheritance check. */ - private void verifyInheritance(Field field) { - for (SDDocumentType parent : inheritedTypes.values()) { - for (Field pField : parent.fieldSet()) { - if (pField.getName().equals(field.getName())) { - if (!pField.getDataType().equals(field.getDataType())) { - throw new IllegalArgumentException("For " + this + ", field '" + field.getName() + - "': Datatype can not be different from that of same field " + - "in the supertype '" + parent.getName() + "'"); - } - } - } - } - } - - public SDField addField(String string, DataType dataType) { - SDField field = new SDField(this, string, dataType); - addField(field); - return field; - } - - public Field addField(String fName, DataType dataType, boolean header, int code) { - SDField field = new SDField(this, fName, code, dataType); - addField(field); - return field; - } - - private Map fieldsInherited() { - Map map = new LinkedHashMap<>(); - for (SDDocumentType parent : inheritedTypes.values()) { - for (Field field : parent.fieldSet()) { - map.put(field.getName(), field); - } - } - return map; - } - - public Set fieldSet() { - Map map = fieldsInherited(); - Iterator it = docType.fieldIteratorThisTypeOnly(); - while (it.hasNext()) { - Field field = it.next(); - map.put(field.getName(), field); - } - return new LinkedHashSet<>(map.values()); - } - - public Iterator fieldIterator() { - return fieldSet().iterator(); - } - - /** Returns the number of fields in this only, not including inherited fields */ - // TODO: Remove - public int getFieldCount() { - return docType.getFieldCount(); - } - - @Override - public String toString() { - return "document type '" + docType.getName() + "'"; - } - - private static SDDocumentType createSDDocumentType(StructDataType structType) { - SDDocumentType docType = new SDDocumentType(structType.getName()); - for (Field field : structType.getFields()) { - docType.addField(new SDField(docType, field.getName(), field.getDataType())); - } - docType.setStruct(structType); - return docType; - } - - /** The field sets defined for this type and its {@link Schema} */ - public FieldSets getFieldSets() { - return fieldSets; - } - - /** Sets the field sets for this */ - public void setFieldSets(FieldSets fieldSets) { - this.fieldSets = fieldSets; - } - - public Optional getDocumentReferences() { - return documentReferences; - } - - public void setDocumentReferences(DocumentReferences documentReferences) { - this.documentReferences = Optional.of(documentReferences); - } - - public TemporaryImportedFields getTemporaryImportedFields() { - return temporaryImportedFields; - } - - public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { - this.temporaryImportedFields = temporaryImportedFields; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java deleted file mode 100644 index 382c0d0ceb5..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ /dev/null @@ -1,802 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; -import com.yahoo.document.StructDataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.documentmodel.OwnedTemporaryType; -import com.yahoo.documentmodel.TemporaryUnknownType; -import com.yahoo.language.Linguistics; -import com.yahoo.language.process.Embedder; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.fieldoperation.FieldOperation; -import com.yahoo.searchdefinition.fieldoperation.FieldOperationContainer; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.ScriptParserContext; -import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; -import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; -import com.yahoo.vespa.indexinglanguage.parser.ParseException; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.TreeMap; - - -/** - * The field class represents a document field. It is used in - * the Document class to get and set fields. Each SDField has a name, a numeric ID, - * a data type. The numeric ID is used when the fields are stored - * in serialized form. - * - * @author bratseth - */ -public class SDField extends Field implements TypedKey, FieldOperationContainer, ImmutableSDField { - - /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ - private boolean indexStructureField = false; - - /** The indexing statements to be applied to this value during indexing */ - private ScriptExpression indexingScript = new ScriptExpression(); - - /** The default rank type for indices of this field */ - private RankType rankType = RankType.DEFAULT; - - /** Rank settings in a "rank" block for the field. */ - private final Ranking ranking = new Ranking(); - - /** - * The literal boost of this field. This boost is added to a rank score - * when a query term matched as query term exactly (unnormalized and unstemmed). - * Non-positive boosts causes no boosting, 0 allows boosts - * to be specified in other rank profiles, while negative values - * turns the capability off. - */ - private int literalBoost = -1; - - /** - * The weight of this field. This is a percentage, - * so 100 is default to provide the identity transform. - */ - private int weight = 100; - - /** - * Indicates what kind of matching should be done on this field - */ - private Matching matching = new Matching(); - - private Dictionary dictionary = null; - - /** Attribute settings, or null if there are none */ - private final Map attributes = new TreeMap<>(); - - /** - * The stemming setting of this field, or null to use the default. - * Default is determined by the owning search definition. - */ - private Stemming stemming = null; - - /** How content of this field should be accent normalized etc. */ - private NormalizeLevel normalizing = new NormalizeLevel(); - - /** Extra query commands of this field */ - private final List queryCommands = new java.util.ArrayList<>(0); - - /** Summary fields defined in this field */ - private final Map summaryFields = new java.util.LinkedHashMap<>(0); - - /** The explicitly index settings on this field */ - private final Map indices = new java.util.LinkedHashMap<>(); - - private boolean idOverride = false; - - /** Struct fields defined in this field */ - private final Map structFields = new java.util.LinkedHashMap<>(0); - - /** The document that this field was declared in, or null */ - private SDDocumentType repoDocType = null; - - /** The aliases declared for this field. May pertain to indexes or attributes */ - private final Map aliasToName = new HashMap<>(); - - /** Pending operations that must be applied after parsing, due to use of not-yet-defined structs. */ - private final List pendingOperations = new LinkedList<>(); - - private boolean isExtraField = false; - - private boolean wasConfiguredToDoAttributing = false; - - /** - * Creates a new field. This method is only used to create reserved fields. - * - * @param name the name of the field - * @param dataType the datatype of the field - */ - public SDField(SDDocumentType repo, String name, int id, DataType dataType) { - super(name, id, dataType); - this.repoDocType = repo; - populate(name, dataType); - } - - public SDField(String name, DataType dataType) { - this(null, name, dataType); - } - - /** Creates a new field */ - public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name, dataType, null); - } - - /** Creates a new field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { - this(repo, name, dataType, owner, null, 0); - } - - /** - * Creates a new field - * - * @param name the name of the field - * @param dataType the datatype of the field - * @param owner the owning document (used to check for id collisions) - * @param fieldMatching the matching object to set for the field - */ - protected SDField(SDDocumentType repo, - String name, - DataType dataType, - SDDocumentType owner, - Matching fieldMatching, - int recursion) - { - super(name, dataType, owner == null ? null : owner.getDocumentType()); - this.repoDocType = repo; - this.structFieldDepth = recursion; - if (fieldMatching != null) - this.setMatching(fieldMatching); - populate(name, dataType); - } - - private int structFieldDepth = 0; - - private void populate(String name, DataType dataType) { - if (dataType instanceof TensorDataType) { - TensorType type = ((TensorDataType)dataType).getTensorType(); - if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) - throw new IllegalArgumentException("Illegal type in field " + name + " type " + type + - ": Dense tensor dimensions must have a size"); - addQueryCommand("type " + type); - } - else if (dataType instanceof WeightedSetDataType) { - var nested = ((WeightedSetDataType) dataType).getNestedType().getName(); - addQueryCommand("type WeightedSet<" + nested + ">"); - } - else { - addQueryCommand("type " + dataType.getName()); - } - } - - public void setIsExtraField(boolean isExtra) { - isExtraField = isExtra; - } - - @Override - public boolean isExtraField() { - return isExtraField; - } - - @Override - public boolean isImportedField() { - return false; - } - - @Override - public boolean doesAttributing() { - return containsExpression(AttributeExpression.class); - } - - @Override - public boolean doesIndexing() { - return containsExpression(IndexExpression.class); - } - - public boolean doesSummarying() { - if (usesStruct()) { - for (SDField structField : getStructFields()) { - if (structField.doesSummarying()) { - return true; - } - } - } - return containsExpression(SummaryExpression.class); - } - - @Override - public boolean doesLowerCasing() { - return containsExpression(LowerCaseExpression.class); - } - - @Override - public boolean containsExpression(Class searchFor) { - return findExpression(searchFor) != null; - } - - private T findExpression(Class searchFor) { - return new ExpressionSearcher<>(searchFor).searchIn(indexingScript); - } - - public void addSummaryFieldSources(SummaryField summaryField) { - if (usesStruct()) { - /* - * How this works for structs: When at least one sub-field in a struct is to - * be used for summary, that whole struct field is included in summary.cfg. Then, - * vsmsummary.cfg specifies the sub-fields used for each struct field. - * So we recurse into each struct, adding the destination classes set for each sub-field - * to the main summary-field for the struct field. - */ - for (SDField structField : getStructFields()) { - for (SummaryField sumF : structField.getSummaryFields().values()) { - for (String dest : sumF.getDestinations()) { - summaryField.addDestination(dest); - } - } - structField.addSummaryFieldSources(summaryField); - } - } else { - if (doesSummarying()) { - summaryField.addSource(getName()); - } - } - } - - private boolean doneStructFields = false; - - @SuppressWarnings("deprecation") - private void actuallyMakeStructFields() { - if (doneStructFields) return; - if (getFirstStructOrMapRecursive() == null) { - doneStructFields = true; - return; - } - var sdoc = repoDocType; - var dataType = getDataType(); - - java.util.function.BiConsumer supplyStructField = (fieldName, fieldType) -> { - if (structFields.containsKey(fieldName)) return; - Matching subFieldMatching = new Matching(); - subFieldMatching.merge(this.matching); - String subName = getName().concat(".").concat(fieldName); - var subField = new SDField(sdoc, subName, fieldType, null, - subFieldMatching, structFieldDepth + 1); - structFields.put(fieldName, subField); - }; - - if (dataType instanceof MapDataType) { - MapDataType mdt = (MapDataType) dataType; - supplyStructField.accept("key", mdt.getKeyType()); - supplyStructField.accept("value", mdt.getValueType()); - } else { - if (structFieldDepth >= 10) { - // too risky, infinite recursion - doneStructFields = true; - return; - } - if (dataType instanceof CollectionDataType) { - dataType = ((CollectionDataType)dataType).getNestedType(); - } - if ((dataType instanceof MapDataType) || (dataType instanceof CollectionDataType)) { - // "array of map" or "array of array" will not have any struct fields - // TODO: consider what this would mean - doneStructFields = true; - return; - } - SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (dataType instanceof TemporaryUnknownType && subType != null) { - for (Field field : subType.fieldSet()) { - supplyStructField.accept(field.getName(), field.getDataType()); - } - } else if (dataType instanceof OwnedTemporaryType && subType != null) { - for (Field field : subType.fieldSet()) { - supplyStructField.accept(field.getName(), field.getDataType()); - } - } else if (dataType instanceof StructDataType) { - var sdt = (StructDataType) dataType; - for (Field field : sdt.getFields()) { - supplyStructField.accept(field.getName(), field.getDataType()); - } - } - if ((subType == null) && (structFields.size() > 0)) { - throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " - + this + " [" + getDataType() + getDataType().getClass() + - "] with " + structFields.size() + " struct fields"); - } - // populate struct fields with matching - if (subType != null) { - for (Field f : subType.fieldSet()) { - if (f instanceof SDField) { - SDField field = (SDField) f; - SDField subField = structFields.get(field.getName()); - if (subField != null) { - // we just made this with a copy of our matching (see above) - Matching subFieldMatching = subField.getMatching(); - subFieldMatching.merge(field.getMatching()); - subField.setMatching(subFieldMatching); - } - } else { - throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); - } - } - } - // else ("missing subtype for struct fields in: " + this + " type " + getDataType() + " [" + getDataType().getClass().getSimpleName() + "]"); - } - doneStructFields = true; - } - - private Matching matchingForStructFields = null; - - public void addOperation(FieldOperation op) { - pendingOperations.add(op); - } - - @Override - public void applyOperations(SDField field) { - if (pendingOperations.isEmpty()) return; - - Collections.sort(pendingOperations); - ListIterator ops = pendingOperations.listIterator(); - while (ops.hasNext()) { - FieldOperation op = ops.next(); - ops.remove(); - op.apply(field); - } - } - - public void applyOperations() { - applyOperations(this); - } - - public void setId(int fieldId, DocumentType owner) { - super.setId(fieldId, owner); - idOverride = true; - } - - public StructDataType getFirstStructRecursive() { - DataType dataType = getDataType(); - while (true) { // Currently no nesting of collections - if (dataType instanceof CollectionDataType) { - dataType = ((CollectionDataType)dataType).getNestedType(); - } else if (dataType instanceof MapDataType) { - dataType = ((MapDataType)dataType).getValueType(); - } else { - break; - } - } - return (dataType instanceof StructDataType) ? (StructDataType)dataType : null; - } - - private DataType getFirstStructOrMapRecursive() { - DataType dataType = getDataType(); - while (dataType instanceof CollectionDataType) { // Currently no nesting of collections - dataType = ((CollectionDataType)dataType).getNestedType(); - } - return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null; - } - - private boolean usesStruct() { - DataType dt = getFirstStructRecursive(); - return (dt != null); - } - - @Override - public boolean usesStructOrMap() { - DataType dt = getFirstStructOrMapRecursive(); - return (dt != null); - } - - @Override - public boolean wasConfiguredToDoAttributing() { - return wasConfiguredToDoAttributing; - } - - /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ - public void parseIndexingScript(String script) { - parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); - } - - public void parseIndexingScript(String script, Linguistics linguistics, Map embedders) { - try { - ScriptParserContext config = new ScriptParserContext(linguistics, embedders); - config.setInputStream(new IndexingInput(script)); - setIndexingScript(ScriptExpression.newInstance(config)); - } catch (ParseException e) { - throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); - } - } - - /** Sets the indexing script of this, or null to not use a script */ - public void setIndexingScript(ScriptExpression exp) { - if (exp == null) { - exp = new ScriptExpression(); - } - indexingScript = exp; - if (indexingScript.isEmpty()) { - return; // TODO: This causes empty expressions not to be propagate to struct fields!! BAD BAD BAD!! - } - if (!wasConfiguredToDoAttributing()) { - wasConfiguredToDoAttributing = doesAttributing(); - } - if (!usesStructOrMap()) { - new ExpressionVisitor() { - - @Override - protected void doVisit(Expression exp) { - if (!(exp instanceof AttributeExpression)) { - return; - } - String fieldName = ((AttributeExpression)exp).getFieldName(); - if (fieldName == null) { - fieldName = getName(); - } - Attribute attribute = attributes.get(fieldName); - if (attribute == null) { - addAttribute(new Attribute(fieldName, getDataType())); - } - } - }.visit(indexingScript); - } - for (SDField structField : getStructFields()) { - structField.setIndexingScript(exp); - } - } - - @Override - public ScriptExpression getIndexingScript() { return indexingScript; } - - @SuppressWarnings("deprecation") - @Override - public void setDataType(DataType type) { - if (type.equals(DataType.URI)) { // Different defaults, naturally - normalizing.inferLowercase(); - stemming = Stemming.NONE; - } - this.dataType = type; - if ( ! idOverride) { - this.fieldId = calculateIdV7(null); - } - } - - @Override - public boolean isIndexStructureField() { - return indexStructureField; - } - - public void setIndexStructureField(boolean indexStructureField) { - this.indexStructureField = indexStructureField; - } - - @Override - public boolean hasIndex() { - return (getIndexingScript() != null) && doesIndexing(); - } - - /** Sets the literal boost of this field */ - public void setLiteralBoost(int literalBoost) { this.literalBoost=literalBoost; } - - /** - * Returns the literal boost of this field. This boost is added to a literal score - * when a query term matched as query term exactly (unnormalized and unstemmed). - * Default is non-positive. - */ - @Override - public int getLiteralBoost() { return literalBoost; } - - /** Sets the weight of this field */ - public void setWeight(int weight) { this.weight=weight; } - - /** Returns the weight of this field, or 0 if nothing is set */ - @Override - public int getWeight() { return weight; } - - /** - * Returns what kind of matching type should be applied. - */ - @Override - public Matching getMatching() { return matching; } - - /** - * Sets what kind of matching type should be applied. - * (Token matching is default, PREFIX, SUBSTRING, SUFFIX are alternatives) - */ - public void setMatching(Matching matching) { this.matching=matching; } - - /** - * Returns Dictionary settings. - */ - public Dictionary getDictionary() { return dictionary; } - public Dictionary getOrSetDictionary() { - if (dictionary == null) { - dictionary = new Dictionary(); - } - return dictionary; - } - - /** - * Set the matching type for this field and all subfields. - */ - // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right - // Matching object for struct fields at lookup time instead. - public void setMatchingType(MatchType type) { - this.getMatching().setType(type); - for (SDField structField : getStructFields()) { - structField.setMatchingType(type); - } - } - - /** - * Set the matching type for this field and all subfields. - */ - // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right - // Matching object for struct fields at lookup time instead. - public void setMatchingCase(Case casing) { - this.getMatching().setCase(casing); - for (SDField structField : getStructFields()) { - structField.setMatchingCase(casing); - } - } - /** - * Set matching algorithm for this field and all subfields. - */ - // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right - // Matching object for struct fields at lookup time instead. - public void setMatchingAlgorithm(MatchAlgorithm algorithm) { - this.getMatching().setAlgorithm(algorithm); - for (SDField structField : getStructFields()) { - structField.getMatching().setAlgorithm(algorithm); - } - } - - /** Adds an explicit index defined in this field */ - public void addIndex(Index index) { - indices.put(index.getName(),index); - } - - /** - * Returns an index, or null if no index with this name has had - * some explicit settings applied in this field (even if this returns null, - * the index may be implicitly defined by an indexing statement) - */ - @Override - public Index getIndex(String name) { - return indices.get(name); - } - - /** - * Returns an index if this field has one (implicitly or - * explicitly) targeting the given name. - */ - @Override - public boolean existsIndex(String name) { - if (indices.get(name) != null) return true; - return name.equals(getName()) && doesIndexing(); - } - - /** - * Defined indices on this field - * @return defined indices on this - */ - @Override - public Map getIndices() { - return indices; - } - - /** - * Sets the default rank type of this fields indices, and sets this rank type - * to all indices explicitly defined here which has no index set. - * (This complex behavior is dues to the fact than we would prefer to have rank types - * per field, not per index) - */ - public void setRankType(RankType rankType) { - this.rankType=rankType; - for (Index index : getIndices().values()) { - if (index.getRankType()==null) - index.setRankType(rankType); - } - - } - - /** Returns the rank settings set in a "rank" block for this field. This is never null. */ - @Override - public Ranking getRanking() { return ranking; } - - /** Returns the default rank type of indices of this field, or null if nothing is set */ - @Override - public RankType getRankType() { return this.rankType; } - - /** - * Returns the search-time attribute settings of this field or null if none is set. - * - *

TODO: Make unmodifiable.

- */ - @Override - public Map getAttributes() { return attributes; } - - public Attribute getAttribute() { - return attributes.get(getName()); - } - - public void addAttribute(Attribute attribute) { - String name = attribute.getName(); - if (name == null || "".equals(name)) { - name = getName(); - attribute.setName(name); - } - attributes.put(attribute.getName(),attribute); - } - - /** - * Returns the stemming setting of this field. - * Default is determined by the owning search definition. - * - * @return the stemming setting of this, or null, to use the default - */ - @Override - public Stemming getStemming() { return stemming; } - - /** - * Whether this field should be stemmed in this search definition - */ - @Override - public Stemming getStemming(Schema schema) { - if (stemming != null) - return stemming; - else - return schema.getStemming(); - } - - @Override - public Field asField() { - return this; - } - - /** - * Sets how this field should be stemmed, or set to null to use the default. - */ - public void setStemming(Stemming stemming) { - this.stemming = stemming; - } - - /** Returns an unmodifiable map of the summary fields defined in this */ - @Override - public Map getSummaryFields() { - return Collections.unmodifiableMap(summaryFields); - } - - public void removeSummaryFields() { - summaryFields.clear(); - } - - /** Adds a summary field */ - public void addSummaryField(SummaryField summaryField) { - summaryFields.put(summaryField.getName(),summaryField); - } - - /** - * Returns a summary field defined (implicitly or explicitly) by this field. - * Returns null if there is no such summary field defined. - */ - @Override - public SummaryField getSummaryField(String name) { - return summaryFields.get(name); - } - - /** - * Returns a summary field defined (implicitly or explicitly) by this field. - * - * @param create true to create the summary field and add it to this field before returning if it is missing - * @return the summary field, or null if not present and create is false - */ - public SummaryField getSummaryField(String name, boolean create) { - SummaryField summaryField=summaryFields.get(name); - if (summaryField==null && create) { - summaryField=new SummaryField(name, getDataType()); - addSummaryField(summaryField); - } - return summaryFields.get(name); - } - - /** Returns list of static struct fields */ - @Override - public Collection getStructFields() { - actuallyMakeStructFields(); - return structFields.values(); - } - - /** - * Returns a struct field defined in this field, - * potentially traversing into nested structs. - * Returns null if there is no such struct field defined. - */ - @Override - public SDField getStructField(String name) { - actuallyMakeStructFields(); - if (name.contains(".")) { - String superFieldName = name.substring(0,name.indexOf(".")); - String subFieldName = name.substring(name.indexOf(".")+1); - SDField superField = structFields.get(superFieldName); - if (superField != null) { - return superField.getStructField(subFieldName); - } - return null; - } - return structFields.get(name); - } - - /** - * Returns how the content of this field should be accent normalized etc - */ - @Override - public NormalizeLevel getNormalizing() { return normalizing; } - - /** - * Change how the content of this field should be accent normalized etc - */ - public void setNormalizing(NormalizeLevel level) { normalizing = level; } - - public void addQueryCommand(String name) { - queryCommands.add(name); - } - - public boolean hasQueryCommand(String name) { - return queryCommands.contains(name); - } - - /** Returns a list of query commands */ - @Override - public List getQueryCommands() { return queryCommands; } - - @Override - public boolean equals(Object other) { - if ( ! (other instanceof SDField)) return false; - return super.equals(other); - } - - @Override - public int hashCode() { - return getName().hashCode(); - } - - @Override - public String toString() { - return "field '" + getName() + "'"; - } - - /** The aliases declared for this field */ - @Override - public Map getAliasToName() { - return aliasToName; - } - - @Override - public boolean hasFullIndexingDocprocRights() { - Attribute self = getAttributes().get(getName()); - return (!isExtraField() || ((self != null) && self.isMutable())); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java deleted file mode 100644 index ef175f68ab8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import java.io.Serializable; - -/** - * A search-time document attribute sort specification(per-document in-memory value). - * This belongs to the attribute or field(implicitt attribute). - * - * @author baldersheim - */ -public final class Sorting implements Cloneable, Serializable { - - // Remember to change hashCode and equals when you add new fields - public enum Function {UCA, RAW, LOWERCASE} - public enum Strength {PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL} - private boolean ascending = true; - private Function function = Function.UCA; - private String locale = ""; - private Strength strength = Strength.PRIMARY; - - public boolean isAscending() { return ascending; } - public boolean isDescending() { return ! ascending; } - public String getLocale() { return locale; } - public Function getFunction() { return function; } - public Strength getStrength() { return strength; } - - public void setAscending() { ascending = true; } - public void setDescending() { ascending = false; } - public void setFunction(Function function) { this.function = function; } - public void setLocale(String locale) { this.locale = locale; } - public void setStrength(Strength strength) { this.strength = strength; } - - public int hashCode() { - return locale.hashCode() + - strength.hashCode() + - function.hashCode() + - (isDescending() ? 13 : 0); - } - - public boolean equals(Object object) { - if (! (object instanceof Sorting)) return false; - - Sorting other=(Sorting)object; - return this.locale.equals(other.locale) && - (ascending == other.ascending) && - (function == other.function) && - (strength == other.strength); - } - - @Override - public Sorting clone() { - try { - return (Sorting)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Programming error"); - } - } - - public String toString() { - return "sorting '" + (isAscending() ? '+' : '-') + function.toString() + "(" + strength.toString() + ", " + locale + ")"; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java deleted file mode 100644 index 5b4862d4643..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.language.process.StemMode; - -/** - * The stemming setting of a field. This describes how the search engine - * should transform content of this field into base forms (stems) to increase - * recall (find "car" when you search for "cars" etc.). - * - * @author bratseth - */ -public enum Stemming { - - /** No stemming */ - NONE("none"), - - /** select shortest possible stem */ - SHORTEST("shortest"), - - /** select the "best" stem alternative */ - BEST("best"), - - /** index multiple stems */ - MULTIPLE("multiple"); - - private final String name; - - /** - * Returns the stemming object for the given string. - * The legal stemming names are the stemming constants in any capitalization. - * - * @throws IllegalArgumentException if there is no stemming type with the given name - */ - public static Stemming get(String stemmingName) { - try { - return Stemming.valueOf(stemmingName.toUpperCase()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting"); - } - } - - Stemming(String name) { - this.name = name; - } - - public String getName() { return name; } - - @Override - public String toString() { - return "stemming " + name; - } - - public StemMode toStemMode() { - switch(this) { - case SHORTEST: return StemMode.SHORTEST; - case MULTIPLE: return StemMode.ALL; - case BEST : return StemMode.BEST; - case NONE: return StemMode.NONE; - default: throw new IllegalStateException("Inconvertible stem mode " + this); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedField.java deleted file mode 100644 index fef5e25c402..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedField.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -/** - * A field that is imported from a field in a referenced document type and given an alias name. - * - * This is temporary AST structure that only refers to the imported field by name. - * - * @author geirst - */ -public class TemporaryImportedField { - - private final String fieldName; - private final String referenceFieldName; - private final String targetFieldName; - - public TemporaryImportedField(String fieldName, - String referenceFieldName, - String targetFieldName) { - this.fieldName = fieldName; - this.referenceFieldName = referenceFieldName; - this.targetFieldName = targetFieldName; - } - - public String fieldName() { - return fieldName; - } - - public String referenceFieldName() { - return referenceFieldName; - } - - public String targetFieldName() { - return targetFieldName; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java deleted file mode 100644 index b4d76445507..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.Schema; - -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * A set of fields that are imported from referenced document types. - * - * This is temporary AST structure that only refers to the imported fields by name. - * - * @author geirst - */ -public class TemporaryImportedFields { - - private final Schema owner; - private final Map fields = new LinkedHashMap<>(); - - public TemporaryImportedFields(Schema owner) { - this.owner = owner; - } - - public void add(TemporaryImportedField importedField) { - fields.put(importedField.fieldName(), importedField); - } - - public boolean hasField(String fieldName) { - return fields.get(fieldName) != null; - } - - public Map fields() { - if (owner.inherited().isEmpty()) return Collections.unmodifiableMap(fields); - if (owner.inherited().get().temporaryImportedFields().isEmpty()) return Collections.unmodifiableMap(fields); - - var allFields = new HashMap<>(owner.inherited().get().temporaryImportedFields().get().fields()); - allFields.putAll(fields); - return allFields; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java deleted file mode 100644 index 1452ff430f3..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataTypeName; - -/** - * @author baldersheim - */ -public class TemporarySDDocumentType extends SDDocumentType { - public TemporarySDDocumentType(DataTypeName name) { - super(name); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java deleted file mode 100644 index 8c17b607f94..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataType; - -/** - * @author Einar M R Rosenvinge - */ -public class TemporarySDField extends SDField { - - public TemporarySDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner) { - super(repo, name, dataType, owner); - } - - public TemporarySDField(SDDocumentType repo, String name, DataType dataType) { - super(repo, name, dataType); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java deleted file mode 100644 index 767d6cc8009..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.document.DataType; - -/** - * Common interface for various typed key (or field definitions). - * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping - * - * @author bratseth - */ -public interface TypedKey { - - String getName(); - - void setDataType(DataType type); - - DataType getDataType(); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java deleted file mode 100644 index b7ba2333b3c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document.annotation; - -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.document.annotation.AnnotationType; - -/** - * @author Einar M R Rosenvinge - */ -public class SDAnnotationType extends AnnotationType { - - private SDDocumentType sdDocType; - private String inherits; - - public SDAnnotationType(String name) { - super(name); - } - - public SDAnnotationType(String name, SDDocumentType dataType, String inherits) { - super(name); - this.sdDocType = dataType; - this.inherits = inherits; - } - - public SDDocumentType getSdDocType() { - return sdDocType; - } - - public void setSdDocType(SDDocumentType value) { - assert(sdDocType == null); - sdDocType = value; - } - - public String getInherits() { - return inherits; - } - - public void inherit(String inherits) { - this.inherits = inherits; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java deleted file mode 100644 index 717e778adb9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document.annotation; - -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.document.annotation.AnnotationType; - -/** - * @author Einar M R Rosenvinge - */ -public class TemporaryAnnotationReferenceDataType extends AnnotationReferenceDataType { - - private final String target; - - public TemporaryAnnotationReferenceDataType(String target) { - this.target = target; - } - - public String getTarget() { - return target; - } - - @Override - public void setAnnotationType(AnnotationType type) { - super.setName("annotationreference<" + type.getName() + ">"); - super.setAnnotationType(type); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java deleted file mode 100644 index f6ed5abaa7f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; - -import java.util.ArrayList; -import java.util.List; - -/** - * Transforms named references to constant tensors with the rank feature 'constant'. - * - * @author geirst - */ -public class ConstantTensorTransformer extends ExpressionTransformer { - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) { - return transformFeature((ReferenceNode) node, context); - } else if (node instanceof CompositeNode) { - return transformChildren((CompositeNode) node, context); - } else { - return node; - } - } - - private ExpressionNode transformFeature(ReferenceNode node, RankProfileTransformContext context) { - if ( ! node.getArguments().isEmpty() && ! FeatureNames.isSimpleFeature(node.reference())) { - return transformArguments(node, context); - } else { - return transformConstantReference(node, context); - } - } - - private ExpressionNode transformArguments(ReferenceNode node, RankProfileTransformContext context) { - List arguments = node.getArguments().expressions(); - List transformedArguments = new ArrayList<>(arguments.size()); - for (ExpressionNode argument : arguments) { - transformedArguments.add(transform(argument, context)); - } - return node.setArguments(transformedArguments); - } - - private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) { - String constantName = node.getName(); - Reference constantReference = node.reference(); - if (FeatureNames.isConstantFeature(constantReference)) { - constantName = constantReference.simpleArgument().orElse(null); - } else if (constantReference.isIdentifier()) { - constantReference = FeatureNames.asConstantFeature(constantName); - } else { - return node; - } - Value value = context.constants().get(constantName); - if (value == null || value.type().rank() == 0) return node; - - TensorValue tensorValue = (TensorValue)value; - String tensorType = tensorValue.asTensor().type().toString(); - context.rankProperties().put(constantReference + ".value", tensorValue.toString()); - context.rankProperties().put(constantReference + ".type", tensorType); - return new ReferenceNode(constantReference); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java deleted file mode 100644 index 1334a7a3aad..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.google.common.collect.ImmutableList; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.searchlib.rankingexpression.transform.Simplifier; -import com.yahoo.searchlib.rankingexpression.transform.TensorMaxMinTransformer; - -import java.util.List; - -/** - * The transformations done on ranking expressions done at config time before passing them on to the Vespa - * engine for execution. - * - * An instance of this class has scope of a compilation of a single rank profile. - * - * @author bratseth - */ -public class ExpressionTransforms { - - private final List transforms; - - public ExpressionTransforms() { - transforms = - ImmutableList.of(new TensorFlowFeatureConverter(), - new OnnxFeatureConverter(), - new OnnxModelTransformer(), - new XgboostFeatureConverter(), - new LightGBMFeatureConverter(), - new TokenTransformer(), - new ConstantDereferencer(), - new ConstantTensorTransformer(), - new FunctionInliner(), - new FunctionShadower(), - new TensorMaxMinTransformer(), - new Simplifier()); - } - - public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { - for (ExpressionTransformer transformer : transforms) - expression = transformer.transform(expression, context); - return expression; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java deleted file mode 100644 index 340eec278e3..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; - -/** - * Inlines functions in ranking expressions - * - * @author bratseth - */ -public class FunctionInliner extends ExpressionTransformer { - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeatureNode((ReferenceNode)node, context); - if (node instanceof CompositeNode) - return transformChildren((CompositeNode)node, context); - return node; - } - - private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { - if (feature.getArguments().size() > 0) return feature; // From RankProfile: only inline no-arg functions - RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName()); - if (rankingExpressionFunction == null) return feature; - return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java deleted file mode 100644 index 1f0112194cf..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; - -/** - * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function. - * This has the effect of allowing rank profile functions to redefine built-in functions. - * Another effect is that we can add built-in functions over time - * without fear of breaking existing users' functions with the same name. - * - * However, there is a (largish) caveat. If a user has a function with a certain number - * of arguments, and we add in a built-in function with a different arity, - * this will cause parse errors as the Java parser gives precedence to - * built-in functions. - * - * @author lesters - */ -public class FunctionShadower extends ExpressionTransformer { - - @Override - public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { - ExpressionNode node = expression.getRoot(); - ExpressionNode result = transform(node, context); - return (result == node) - ? expression - : new RankingExpression(expression.getName(), result); - } - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof FunctionNode) - return transformFunctionNode((FunctionNode) node, context); - if (node instanceof CompositeNode) - return transformChildren((CompositeNode)node, context); - return node; - } - - private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { - String name = function.getFunction().toString(); - RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().findFunction(name); - if (rankingExpressionFunction == null) - return transformChildren(function, context); - - int functionArity = function.getFunction().arity(); - if (functionArity != rankingExpressionFunction.function().arguments().size()) - return transformChildren(function, context); - - ReferenceNode node = new ReferenceNode(name, function.children(), null); - return transformChildren(node, context); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java deleted file mode 100644 index 1bccac237d5..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.path.Path; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.vespa.model.ml.ConvertedModel; -import com.yahoo.vespa.model.ml.FeatureArguments; - -import java.io.UncheckedIOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Replaces instances of the lightgbm(model-path) pseudofeature with the - * native Vespa ranking expression implementing the same computation. - * - * @author lesters - */ -public class LightGBMFeatureConverter extends ExpressionTransformer { - - /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ - private final Map convertedLightGBMModels = new HashMap<>(); - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeature((ReferenceNode) node, context); - else if (node instanceof CompositeNode) - return super.transformChildren((CompositeNode) node, context); - else - return node; - } - - private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if ( ! feature.getName().equals("lightgbm")) return feature; - - try { - FeatureArguments arguments = asFeatureArguments(feature.getArguments()); - ConvertedModel convertedModel = - convertedLightGBMModels.computeIfAbsent(arguments.path(), - path -> ConvertedModel.fromSourceOrStore(path, true, context)); - return convertedModel.expression(arguments, context); - } catch (IllegalArgumentException | UncheckedIOException e) { - throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e); - } - } - - private FeatureArguments asFeatureArguments(Arguments arguments) { - if (arguments.size() != 1) - throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " + - "the LightGBM model file under [application]/models"); - return new FeatureArguments(arguments); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java deleted file mode 100644 index 6944a1f9dd1..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.path.Path; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.vespa.model.ml.ConvertedModel; -import com.yahoo.vespa.model.ml.FeatureArguments; - -import java.io.UncheckedIOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Replaces instances of the onnx(model-path, output) - * pseudofeature with the native Vespa ranking expression implementing - * the same computation. - * - * @author bratseth - * @author lesters - */ -public class OnnxFeatureConverter extends ExpressionTransformer { - - /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ - private final Map convertedOnnxModels = new HashMap<>(); - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeature((ReferenceNode) node, context); - else if (node instanceof CompositeNode) - return super.transformChildren((CompositeNode) node, context); - else - return node; - } - - private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if ( ! feature.getName().equals("onnx_vespa")) return feature; - try { - FeatureArguments arguments = asFeatureArguments(feature.getArguments()); - ConvertedModel convertedModel = - convertedOnnxModels.computeIfAbsent(arguments.path(), - path -> ConvertedModel.fromSourceOrStore(path, true, context)); - return convertedModel.expression(arguments, context); - } - catch (IllegalArgumentException | UncheckedIOException e) { - throw new IllegalArgumentException("Could not use Onnx model from " + feature, e); - } - } - - private FeatureArguments asFeatureArguments(Arguments arguments) { - if (arguments.isEmpty()) - throw new IllegalArgumentException("An ONNX node must take an argument pointing to " + - "the ONNX model file under [application]/models"); - if (arguments.expressions().size() > 3) - throw new IllegalArgumentException("An onnx feature can have at most 3 arguments"); - - return new FeatureArguments(arguments); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java deleted file mode 100644 index 58a9c78254a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.path.Path; -import com.yahoo.searchdefinition.ImmutableSchema; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.vespa.model.ml.ConvertedModel; -import com.yahoo.vespa.model.ml.FeatureArguments; -import com.yahoo.vespa.model.ml.ModelName; - -import java.util.List; - -/** - * Transforms ONNX model features of the forms: - * - * onnxModel(config_name) - * onnxModel(config_name).output - * onnxModel("path/to/model") - * onnxModel("path/to/model").output - * onnxModel("path/to/model", "path/to/output") - * onnxModel("path/to/model", "unused", "path/to/output") // signature is unused - * onnx(...) // same as with onnxModel, onnx is an alias of onnxModel - * - * To the format expected by the backend: - * - * onnxModel(config_name).output - * - * @author lesters - */ -public class OnnxModelTransformer extends ExpressionTransformer { - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeature((ReferenceNode) node, context); - else if (node instanceof CompositeNode) - return super.transformChildren((CompositeNode) node, context); - else - return node; - } - - private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if (context.rankProfile() == null) return feature; - if (context.rankProfile().schema() == null) return feature; - return transformFeature(feature, context.rankProfile()); - } - - public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile profile) { - String featureName = feature.getName(); - if ( ! featureName.equals("onnxModel") && ! featureName.equals("onnx")) return feature; - - Arguments arguments = feature.getArguments(); - if (arguments.isEmpty()) - throw new IllegalArgumentException("An " + featureName + " feature must take an argument referring to a " + - "onnx-model config or an ONNX file."); - if (arguments.expressions().size() > 3) - throw new IllegalArgumentException("An " + featureName + " feature can have at most 3 arguments."); - - // Check that the model configuration "onnx-model" exists. If not defined, it should have been added - // by the "OnnxModelConfigGenerator" processor. If it still doesn't exist, it is because we can't find - // the actual ONNX file, which can happen if we are restarting or upgrading an application using an - // ONNX file that was transformed to Vespa ranking expressions. We then assume it is in the model store. - - String modelConfigName = getModelConfigName(feature.reference()); - OnnxModel onnxModel = profile.onnxModels().get(modelConfigName); - if (onnxModel == null) { - String path = asString(arguments.expressions().get(0)); - ModelName modelName = new ModelName(null, Path.fromString(path), true); - ConvertedModel convertedModel = ConvertedModel.fromStore(profile.schema().applicationPackage(), modelName, path, profile); - FeatureArguments featureArguments = new FeatureArguments(arguments); - return convertedModel.expression(featureArguments, null); - } - - String defaultOutput = onnxModel.getOutputMap().get(onnxModel.getDefaultOutput()); - String output = getModelOutput(feature.reference(), defaultOutput); - if (! onnxModel.getOutputMap().containsValue(output)) { - throw new IllegalArgumentException(featureName + " argument '" + output + - "' output not found in model '" + onnxModel.getFileName() + "'"); - } - return new ReferenceNode("onnxModel", List.of(new ReferenceNode(modelConfigName)), output); - } - - public static String getModelConfigName(Reference reference) { - if (reference.arguments().size() > 0) { - ExpressionNode expr = reference.arguments().expressions().get(0); - if (expr instanceof ReferenceNode) { // refers to onnx-model config - return expr.toString(); - } - if (expr instanceof ConstantNode) { // refers to a file path - return asValidIdentifier(expr); - } - } - return null; - } - - public static String getModelOutput(Reference reference, String defaultOutput) { - if (reference.output() != null) { - return reference.output(); - } else if (reference.arguments().expressions().size() == 2) { - return asValidIdentifier(reference.arguments().expressions().get(1)); - } else if (reference.arguments().expressions().size() > 2) { - return asValidIdentifier(reference.arguments().expressions().get(2)); - } - return defaultOutput; - } - - public static String stripQuotes(String s) { - if (isNotQuoteSign(s.codePointAt(0))) return s; - if (isNotQuoteSign(s.codePointAt(s.length() - 1))) - throw new IllegalArgumentException("argument [" + s + "] is missing end quote"); - return s.substring(1, s.length()-1); - } - - public static String asValidIdentifier(String str) { - return str.replaceAll("[^\\w\\d\\$@_]", "_"); - } - - private static String asValidIdentifier(ExpressionNode node) { - return asValidIdentifier(asString(node)); - } - - private static boolean isNotQuoteSign(int c) { - return c != '\'' && c != '"'; - } - - public static String asString(ExpressionNode node) { - if ( ! (node instanceof ConstantNode)) - throw new IllegalArgumentException("Expected a constant string as argument, but got '" + node); - return stripQuotes(node.toString()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java deleted file mode 100644 index b284f9df325..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.transform.TransformContext; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Extends the transform context with rank profile information - * - * @author bratseth - */ -public class RankProfileTransformContext extends TransformContext { - - private final RankProfile rankProfile; - private final QueryProfileRegistry queryProfiles; - private final ImportedMlModels importedModels; - private final Map inlineFunctions; - private final Map rankProperties = new HashMap<>(); - - public RankProfileTransformContext(RankProfile rankProfile, - QueryProfileRegistry queryProfiles, - Map featureTypes, - ImportedMlModels importedModels, - Map constants, - Map inlineFunctions) { - super(valuesOf(constants), rankProfile.typeContext(queryProfiles, featureTypes)); - this.rankProfile = rankProfile; - this.queryProfiles = queryProfiles; - this.importedModels = importedModels; - this.inlineFunctions = inlineFunctions; - } - - public RankProfile rankProfile() { return rankProfile; } - public QueryProfileRegistry queryProfiles() { return queryProfiles; } - public ImportedMlModels importedModels() { return importedModels; } - public Map inlineFunctions() { return inlineFunctions; } - public Map rankProperties() { return rankProperties; } - - private static Map valuesOf(Map constants) { - return constants.values().stream() - .filter(constant -> constant.value().isPresent()) - .collect(Collectors.toMap(constant -> constant.name().simpleArgument().get(), - constant -> asValue(constant.value().get()))); - } - - private static Value asValue(Tensor tensor) { - if (tensor.type().rank() == 0) - return DoubleValue.of(tensor.asDouble()); - else - return TensorValue.of(tensor); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java deleted file mode 100644 index 7b165d94cae..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.path.Path; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.vespa.model.ml.ConvertedModel; -import com.yahoo.vespa.model.ml.FeatureArguments; - -import java.io.UncheckedIOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Replaces instances of the tensorflow(model-path, signature, output) - * pseudofeature with the native Vespa ranking expression implementing - * the same computation. - * - * @author bratseth - */ -public class TensorFlowFeatureConverter extends ExpressionTransformer { - - /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ - private final Map convertedTensorFlowModels = new HashMap<>(); - - public TensorFlowFeatureConverter() {} - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeature((ReferenceNode) node, context); - else if (node instanceof CompositeNode) - return super.transformChildren((CompositeNode) node, context); - else - return node; - } - - private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if ( ! feature.getName().equals("tensorflow")) return feature; - - try { - FeatureArguments arguments = asFeatureArguments(feature.getArguments()); - ConvertedModel convertedModel = - convertedTensorFlowModels.computeIfAbsent(arguments.path(), - path -> ConvertedModel.fromSourceOrStore(path, false, context)); - return convertedModel.expression(arguments, context); - } - catch (IllegalArgumentException | UncheckedIOException e) { - throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); - } - } - - private FeatureArguments asFeatureArguments(Arguments arguments) { - if (arguments.isEmpty()) - throw new IllegalArgumentException("A tensorflow node must take an argument pointing to " + - "the tensorflow model directory under [application]/models"); - if (arguments.expressions().size() > 3) - throw new IllegalArgumentException("A tensorflow feature can have at most 3 arguments"); - - return new FeatureArguments(arguments); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java deleted file mode 100644 index dbb32d88ef6..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; -import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; -import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; -import com.yahoo.searchlib.rankingexpression.rule.ComparisonNode; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; -import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.IfNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; -import com.yahoo.searchlib.rankingexpression.rule.TruthOperator; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.functions.Generate; -import com.yahoo.tensor.functions.Slice; -import com.yahoo.tensor.functions.TensorFunction; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode.wrapScalar; - -/** - * Convenience feature transforms for inputs to Transformer type models. - * - * Replaces features of the form - * - * tokenInputIds - * tokenTypeIds - * tokenAttentionMask - * - * to tensor generation expressions that generate the required input. - * In general, these models expect input of the form: - * - * CLS + arg1 + SEP + arg2 + SEP + 0's - * - * @author lesters - */ -public class TokenTransformer extends ExpressionTransformer { - - static private final ConstantNode ZERO = new ConstantNode(new DoubleValue(0.0)); - static private final ConstantNode ONE = new ConstantNode(new DoubleValue(1.0)); - static private final ConstantNode TWO = new ConstantNode(new DoubleValue(2.0)); - static private final ConstantNode CLS = new ConstantNode(new DoubleValue(101)); - static private final ConstantNode SEP = new ConstantNode(new DoubleValue(102)); - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeature((ReferenceNode) node, context); - else if (node instanceof CompositeNode) - return super.transformChildren((CompositeNode) node, context); - else - return node; - } - - private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if (feature.getName().equals("tokenInputIds") && shouldTransform(feature, context)) - return transformTokenInputIds(feature, context); - if (feature.getName().equals("tokenTypeIds") && shouldTransform(feature, context)) - return transformTokenTypeIds(feature, context); - if (feature.getName().equals("tokenAttentionMask") && shouldTransform(feature, context)) - return transformTokenAttentionMask(feature, context); - return feature; - } - - /** - * Transforms a feature of the form - * - * tokenInputIds(128, a, b, ...) - * - * to an expression that concatenates the arguments a, b, ... using the - * special Transformers sequences of CLS and SEP, up to length 128, so - * that the sequence becomes - * - * CLS + a + SEP + b + SEP + 0's - * - * Concretely, transforms to a tensor generation expression: - * - * tensor(d0[1],d1[128])( - * if (d1 < 1, - * 101, - * if (d1 < 1 + length_a, - * a{d0:(d1 - (1)}, - * if (d1 < 1 + length_a + 1, - * 102, - * if (d1 < 1 + length_a + 1 + length_b, - * b{d0:(d1 - (1 + length_a + 1))}, - * if (d1 < 1 + length_a + 1 + length_b + 1, - * 102, - * 0.0 - * )))))) - * - * Functions calculating lengths of arguments are added to the rank profile. - */ - private ExpressionNode transformTokenInputIds(ReferenceNode feature, RankProfileTransformContext context) { - checkArguments(feature); - - TensorType type = createTensorType(feature.getName(), feature.getArguments().expressions().get(0)); - - // we need to add functions calculating the token lengths of the arguments - createTokenLengthFunctions(feature, context); - - // create token sequence: CLS + arg1 + SEP + arg2 + SEP + .... - ExpressionNode tokenSequenceExpr = createTokenSequenceExpr(0, createTokenSequence(feature)); - return new TensorFunctionNode(Generate.bound(type, wrapScalar(tokenSequenceExpr))); - } - - /** - * Transforms a feature of the form - * - * tokenTypeIds(128, a, b, ...) - * - * to an expression that generates a tensor that has values 0 for "a" - * (including CLS and SEP tokens) and 1 for the rest of the sequence. - * - * Concretely, transforms to a tensor generation expression: - * - * tensor(d0[1],d1[128])( - * if (d1 < 1 + length_a + 1, - * 0, - * if (d1 < 1 + length_a + 1 + length_b + 1 + ..., - * 1, - * 0 - * ))) - */ - private ExpressionNode transformTokenTypeIds(ReferenceNode feature, RankProfileTransformContext context) { - checkArguments(feature); - - TensorType type = createTensorType(feature.getName(), feature.getArguments().expressions().get(0)); - - // we need to add functions calculating the token lengths of the arguments - createTokenLengthFunctions(feature, context); - - List tokenSequence = createTokenSequence(feature); - ExpressionNode queryLengthExpr = createLengthExpr(2, tokenSequence); - ExpressionNode restLengthExpr = createLengthExpr(tokenSequence.size() - 1, tokenSequence); - ExpressionNode expr = new IfNode( - new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, queryLengthExpr), - ZERO, - new IfNode( - new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, restLengthExpr), - ONE, - ZERO - ) - ); - return new TensorFunctionNode(Generate.bound(type, wrapScalar(expr))); - } - - /** - * Transforms a feature of the form - * - * tokenAttentionMask(128, a, b, ...) - * - * to an expression that generates a tensor that has values 1 for all - * arguments (including CLS and SEP tokens) and 0 for the rest of the - * sequence. - * - * Concretely, transforms to a tensor generation expression: - * - * tensor(d0[1],d1[128])(if(d1 < 1 + length_a + 1 + length_b + 1 + ..., 1, 0)) - * - */ - private ExpressionNode transformTokenAttentionMask(ReferenceNode feature, RankProfileTransformContext context) { - checkArguments(feature); - - TensorType type = createTensorType(feature.getName(), feature.getArguments().expressions().get(0)); - - // we need to add functions calculating the token lengths of the arguments - createTokenLengthFunctions(feature, context); - - List tokenSequence = createTokenSequence(feature); - ExpressionNode lengthExpr = createLengthExpr(tokenSequence.size() - 1, tokenSequence); - ComparisonNode comparison = new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, lengthExpr); - ExpressionNode expr = new IfNode(comparison, ONE, ZERO); - return new TensorFunctionNode(Generate.bound(type, wrapScalar(expr))); - } - - private boolean shouldTransform(ReferenceNode feature, RankProfileTransformContext context) { - if (context.rankProfile().getFunctions().containsKey(feature.getName())) - return false; - if (feature.getArguments().size() < 2) - return false; - return true; - } - - private void checkArguments(ReferenceNode feature) { - for (int i = 1; i < feature.getArguments().size(); ++i) { - ExpressionNode arg = feature.getArguments().expressions().get(i); - if ( ! (arg instanceof ReferenceNode)) { - throw new IllegalArgumentException("Invalid argument " + i + " to " + feature.getName() + ": " + - "the argument must be a reference. Got " + arg.toString()); - } - } - } - - public static TensorType createTensorType(String featureName, ExpressionNode argument) { - try { - int length = Integer.parseInt(argument.toString()); - return new TensorType.Builder(TensorType.Value.FLOAT).indexed("d0", 1).indexed("d1", length).build(); - } catch (NumberFormatException ex) { - throw new IllegalArgumentException("Invalid argument to " + featureName + ": the first argument must be " + - "the length to the token sequence to generate. Got " + argument); - } - } - - private String lengthFunctionName(ReferenceNode arg) { - return "__token_length@" + arg.hashCode(); - } - - private List createTokenSequence(ReferenceNode feature) { - List sequence = new ArrayList<>(); - sequence.add(CLS); - for (int i = 1; i < feature.getArguments().size(); ++i) { - sequence.add(feature.getArguments().expressions().get(i)); - sequence.add(SEP); - } - return sequence; - } - - /** - * Adds functions for calculating the token length input. Assumes that - * token sequences are 0-padded, so this returns the number of non-0 - * tokens using a map and reduce-sum. - */ - private void createTokenLengthFunctions(ReferenceNode feature, RankProfileTransformContext context) { - for (int i = 1; i < feature.getArguments().size(); ++i) { - ExpressionNode arg = feature.getArguments().expressions().get(i); - if ( ! (arg instanceof ReferenceNode)) { - throw new IllegalArgumentException("Invalid argument " + i + " to " + feature.getName() + ": " + - "the argument must be a reference. Got " + arg.toString()); - } - ReferenceNode ref = (ReferenceNode) arg; - String functionName = lengthFunctionName(ref); - if ( ! context.rankProfile().getFunctions().containsKey(functionName)) { - context.rankProfile().addFunction(functionName, List.of(), "sum(map(" + ref + ", f(x)(x > 0)))", false); - } - } - } - - /** - * Recursively creates partial expressions of the form - * - * if (d1 < 1 + length_a, - * a{d0:(d1 - 1}, - * ... - * - * for each part of the token sequence. CLS and SEP are added directly, - * and we create a slice expression for each argument to extract the - * actual tokens. - */ - private ExpressionNode createTokenSequenceExpr(int iter, List sequence) { - ExpressionNode lengthExpr = createLengthExpr(iter, sequence); - ComparisonNode comparison = new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, lengthExpr); - - ExpressionNode trueExpr = sequence.get(iter); - if (sequence.get(iter) instanceof ReferenceNode) { - trueExpr = createTokenExtractExpr(iter, sequence); - } - - ExpressionNode falseExpr; - if (iter < sequence.size() - 1) { - falseExpr = createTokenSequenceExpr(iter + 1, sequence); - } else { - falseExpr = ZERO; // 0-padding for rest of sequence - } - - return new IfNode(comparison, trueExpr, falseExpr); - } - - /** - * Creates an expression for the length of the token sequence so far, where - * the lengths of CLS and SEP are 1, and the length of the arguments are - * calculated using auxiliary functions. - */ - private ExpressionNode createLengthExpr(int iter, List sequence) { - List factors = new ArrayList<>(); - List operators = new ArrayList<>(); - for (int i = 0; i < iter + 1; ++i) { - if (sequence.get(i) instanceof ConstantNode) { - factors.add(ONE); - } else if (sequence.get(i) instanceof ReferenceNode) { - factors.add(new ReferenceNode(lengthFunctionName((ReferenceNode) sequence.get(i)))); - } - if (i >= 1) { - operators.add(ArithmeticOperator.PLUS); - } - } - return new ArithmeticNode(factors, operators); - } - - /** - * Create the slice expression to extract the tokens from arguments - */ - private ExpressionNode createTokenExtractExpr(int iter, List sequence) { - ExpressionNode expr; - if (iter >= 1) { - ExpressionNode lengthExpr = new EmbracedNode(createLengthExpr(iter - 1, sequence)); - expr = new EmbracedNode(new ArithmeticNode(new ReferenceNode("d1"), ArithmeticOperator.MINUS, lengthExpr)); - } else { - expr = new ReferenceNode("d1"); - } - List> slices = List.of(new Slice.DimensionValue<>("d0", wrapScalar(expr)) ); - TensorFunction argument = new TensorFunctionNode.ExpressionTensorFunction(sequence.get(iter)); - return new TensorFunctionNode(new Slice<>(argument, slices)); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/XgboostFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/XgboostFeatureConverter.java deleted file mode 100644 index 296ffd5f7fe..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/XgboostFeatureConverter.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.path.Path; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.vespa.model.ml.ConvertedModel; -import com.yahoo.vespa.model.ml.FeatureArguments; - -import java.io.UncheckedIOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Replaces instances of the xgboost(model-path) - * pseudofeature with the native Vespa ranking expression implementing - * the same computation. - * - * @author grace-lam - * @author bratseth - */ -public class XgboostFeatureConverter extends ExpressionTransformer { - - /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ - private final Map convertedXGBoostModels = new HashMap<>(); - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeature((ReferenceNode) node, context); - else if (node instanceof CompositeNode) - return super.transformChildren((CompositeNode) node, context); - else - return node; - } - - private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if ( ! feature.getName().equals("xgboost")) return feature; - - try { - FeatureArguments arguments = asFeatureArguments(feature.getArguments()); - ConvertedModel convertedModel = - convertedXGBoostModels.computeIfAbsent(arguments.path(), - path -> ConvertedModel.fromSourceOrStore(path, true, context)); - return convertedModel.expression(arguments, context); - } catch (IllegalArgumentException | UncheckedIOException e) { - throw new IllegalArgumentException("Could not use XGBoost model from " + feature, e); - } - } - - private FeatureArguments asFeatureArguments(Arguments arguments) { - if (arguments.size() != 1) - throw new IllegalArgumentException("An xgboost node must take a single argument pointing to " + - "the xgboost model directory under [application]/models"); - return new FeatureArguments(arguments); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java deleted file mode 100644 index 1eb27cd31c2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class AliasOperation implements FieldOperation { - - private String aliasedName; - private String alias; - - public AliasOperation(String aliasedName, String alias) { - this.aliasedName = aliasedName; - this.alias = alias; - } - - public String getAliasedName() { - return aliasedName; - } - - public void setAliasedName(String aliasedName) { - this.aliasedName = aliasedName; - } - - public String getAlias() { - return alias; - } - - public void setAlias(String alias) { - this.alias = alias; - } - - public void apply(SDField field) { - if (aliasedName == null) { - aliasedName = field.getName(); - } - field.getAliasToName().put(alias, aliasedName); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java deleted file mode 100644 index e528b1a9971..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.tensor.TensorType; - -import java.util.Locale; -import java.util.Optional; - -/** - * @author Einar M R Rosenvinge - */ -public class AttributeOperation implements FieldOperation, FieldOperationContainer { - - private final String name; - private Boolean huge; - private Boolean fastSearch; - private Boolean fastAccess; - private Boolean mutable; - private Boolean paged; - private Boolean enableBitVectors; - private Boolean enableOnlyBitVector; - //TODO: Remember sorting!! - private boolean doAlias = false; - private String alias; - private String aliasedName; - private Optional tensorType = Optional.empty(); - private Optional distanceMetric = Optional.empty(); - - public AttributeOperation(String name) { - this.name = name; - } - - @Override - public void addOperation(FieldOperation op) { - //TODO: Implement this method: - - } - - @Override - public void applyOperations(SDField field) { - //TODO: Implement this method: - } - - @Override - public String getName() { - return name; - } - - public Boolean getHuge() { - return huge; - } - - public void setHuge(Boolean huge) { - this.huge = huge; - } - - public Boolean getFastSearch() { - return fastSearch; - } - - public void setFastSearch(Boolean fastSearch) { - this.fastSearch = fastSearch; - } - - public Boolean getFastAccess() { - return fastAccess; - } - - public void setFastAccess(Boolean fastAccess) { - this.fastAccess = fastAccess; - } - public void setMutable(Boolean mutable) { - this.mutable = mutable; - } - public void setPaged(Boolean paged) { - this.paged = paged; - } - - public Boolean getEnableBitVectors() { - return enableBitVectors; - } - - public void setEnableBitVectors(Boolean enableBitVectors) { - this.enableBitVectors = enableBitVectors; - } - - public Boolean getEnableOnlyBitVector() { - return enableOnlyBitVector; - } - - public void setEnableOnlyBitVector(Boolean enableOnlyBitVector) { - this.enableOnlyBitVector = enableOnlyBitVector; - } - - public void setDoAlias(boolean doAlias) { - this.doAlias = doAlias; - } - - public String getAlias() { - return alias; - } - - public void setAlias(String alias) { - this.alias = alias; - } - - - public void setAliasedName(String aliasedName) { - this.aliasedName = aliasedName; - } - - public void setTensorType(TensorType tensorType) { - this.tensorType = Optional.of(tensorType); - } - - public void setDistanceMetric(String value) { - this.distanceMetric = Optional.of(value); - } - - public void apply(SDField field) { - Attribute attribute = null; - if (attributeIsSuffixOfStructField(field.getName())) { - attribute = field.getAttributes().get(field.getName()); - } - if (attribute == null) { - attribute = field.getAttributes().get(name); - if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); - field.addAttribute(attribute); - } - } - - if (huge != null) { - attribute.setHuge(huge); - } - if (paged != null) { - attribute.setPaged(paged); - } - if (fastSearch != null) { - attribute.setFastSearch(fastSearch); - } - if (fastAccess != null) { - attribute.setFastAccess(fastAccess); - } - if (mutable != null) { - attribute.setMutable(mutable); - } - if (enableBitVectors != null) { - attribute.setEnableBitVectors(enableBitVectors); - } - if (enableOnlyBitVector != null) { - attribute.setEnableOnlyBitVector(enableOnlyBitVector); - } - if (doAlias) { - field.getAliasToName().put(alias, aliasedName); - } - if (tensorType.isPresent()) { - attribute.setTensorType(tensorType.get()); - } - if (distanceMetric.isPresent()) { - String upper = distanceMetric.get().toUpperCase(Locale.ENGLISH); - attribute.setDistanceMetric(Attribute.DistanceMetric.valueOf(upper)); - } - } - - private boolean attributeIsSuffixOfStructField(String fieldName) { - return ((fieldName.indexOf('.') != -1) && fieldName.endsWith(name)); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java deleted file mode 100644 index 63486a7656b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; - -/** - * @author Einar M R Rosenvinge - */ -public class BoldingOperation implements FieldOperation { - - private final boolean bold; - - public BoldingOperation(boolean bold) { - this.bold = bold; - } - - public void apply(SDField field) { - SummaryField summaryField = field.getSummaryField(field.getName(), true); - summaryField.addSource(field.getName()); - summaryField.addDestination("default"); - summaryField.setTransform(bold ? summaryField.getTransform().bold() : summaryField.getTransform().unbold()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java deleted file mode 100644 index 5b2e0fb004f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.SDField; - -/** - * Represents operations controlling setup of dictionary used for queries - * - * @author baldersheim - */ -public class DictionaryOperation implements FieldOperation { - public enum Operation { HASH, BTREE, CASED, UNCASED } - private final Operation operation; - - public DictionaryOperation(Operation type) { - this.operation = type; - } - @Override - public void apply(SDField field) { - Dictionary dictionary = field.getOrSetDictionary(); - switch (operation) { - case HASH: - dictionary.updateType(Dictionary.Type.HASH); - break; - case BTREE: - dictionary.updateType(Dictionary.Type.BTREE); - break; - case CASED: - dictionary.updateMatch(Case.CASED); - break; - case UNCASED: - dictionary.updateMatch(Case.UNCASED); - break; - default: - throw new IllegalArgumentException("Unhandled operation " + operation); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java deleted file mode 100644 index 2778a147a18..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -/** - * An operation on a field. - * Operations has a natural order of execution. - * - * @author Einar M R Rosenvinge - */ -public interface FieldOperation extends Comparable { - - /** Apply this operation on the given field */ - void apply(SDField field); - - @Override - default int compareTo(FieldOperation other) { - return 0; // no order by default - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java deleted file mode 100644 index f81b1e42449..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public interface FieldOperationContainer { - - /** Adds an operation */ - void addOperation(FieldOperation op); - - /** Apply all operations. Operations must be sorted in their natural order before applying each operation. */ - void applyOperations(SDField field); - - String getName(); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java deleted file mode 100644 index 2d9d2d0b8db..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class IdOperation implements FieldOperation { - - private SDDocumentType document; - private int fieldId; - - public SDDocumentType getDocument() { - return document; - } - - public void setDocument(SDDocumentType document) { - this.document = document; - } - - public int getFieldId() { - return fieldId; - } - - public void setFieldId(int fieldId) { - this.fieldId = fieldId; - } - - public void apply(SDField field) { - document.setFieldId(field, fieldId); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java deleted file mode 100644 index 4611f11f023..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Index.Type; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; - -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalInt; -import java.util.OptionalLong; - -/** - * @author Einar M R Rosenvinge - */ -public class IndexOperation implements FieldOperation { - - private String indexName; - private Optional prefix = Optional.empty(); - private List aliases = new LinkedList<>(); - private Optional stemming = Optional.empty(); - private Optional type = Optional.empty(); - - private OptionalInt arity = OptionalInt.empty(); // For predicate data type - private OptionalLong lowerBound = OptionalLong.empty(); - private OptionalLong upperBound = OptionalLong.empty(); - private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); - private Optional enableBm25 = Optional.empty(); - - private Optional hnswIndexParams = Optional.empty(); - - public String getIndexName() { - return indexName; - } - - public void setIndexName(String indexName) { - this.indexName = indexName; - } - - public boolean getPrefix() { - return prefix.get(); - } - - public void setPrefix(Boolean prefix) { - this.prefix = Optional.of(prefix); - } - - public void addAlias(String alias) { - aliases.add(alias); - } - - public String getStemming() { - return stemming.get(); - } - - public void setStemming(String stemming) { - this.stemming = Optional.of(stemming); - } - - public void apply(SDField field) { - Index index = field.getIndex(indexName); - - if (index == null) { - index = new Index(indexName); - field.addIndex(index); - } - - applyToIndex(index); - } - - public void applyToIndex(Index index) { - if (prefix.isPresent()) { - index.setPrefix(prefix.get()); - } - for (String alias : aliases) { - index.addAlias(alias); - } - if (stemming.isPresent()) { - index.setStemming(Stemming.get(stemming.get())); - } - if (type.isPresent()) { - index.setType(type.get()); - } - if (arity.isPresent() || lowerBound.isPresent() || - upperBound.isPresent() || densePostingListThreshold.isPresent()) { - index.setBooleanIndexDefiniton( - new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold)); - } - if (enableBm25.isPresent()) { - index.setInterleavedFeatures(enableBm25.get()); - } - if (hnswIndexParams.isPresent()) { - index.setHnswIndexParams(hnswIndexParams.get().build()); - } - } - - public Type getType() { - return type.get(); - } - - public void setType(Type type) { - this.type = Optional.of(type); - } - - public void setArity(int arity) { - this.arity = OptionalInt.of(arity); - } - - public void setLowerBound(long value) { - this.lowerBound = OptionalLong.of(value); - } - - public void setUpperBound(long value) { - this.upperBound = OptionalLong.of(value); - } - - public void setDensePostingListThreshold(double densePostingListThreshold) { - this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); - } - - public void setEnableBm25(boolean value) { - enableBm25 = Optional.of(value); - } - - public void setHnswIndexParams(HnswIndexParams.Builder params) { - this.hnswIndexParams = Optional.of(params); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java deleted file mode 100644 index cdd3cc386a4..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.language.Linguistics; -import com.yahoo.language.process.Embedder; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.parser.SimpleCharStream; -import com.yahoo.vespa.indexinglanguage.ScriptParserContext; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; -import com.yahoo.yolean.Exceptions; - -import java.util.Map; - -/** - * @author Einar M R Rosenvinge - */ -public class IndexingOperation implements FieldOperation { - - private final ScriptExpression script; - - public IndexingOperation(ScriptExpression script) { - this.script = script; - } - - public ScriptExpression getScript() { return script; } - - public void apply(SDField field) { - field.setIndexingScript(script); - } - - /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ - public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException { - return fromStream(input, multiLine, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); - } - - public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, - Linguistics linguistics, Map embedders) - throws ParseException { - ScriptParserContext config = new ScriptParserContext(linguistics, embedders); - config.setAnnotatorConfig(new AnnotatorConfig()); - config.setInputStream(input); - ScriptExpression exp; - try { - if (multiLine) { - exp = ScriptExpression.newInstance(config); - } else { - exp = new ScriptExpression(StatementExpression.newInstance(config)); - } - } catch (com.yahoo.vespa.indexinglanguage.parser.ParseException e) { - ParseException t = new ParseException("Could not parse indexing statement: " + Exceptions.toMessageString(e)); - t.initCause(e); - throw t; - } - return new IndexingOperation(exp); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java deleted file mode 100644 index 322f26bc0e3..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.MatchAlgorithm; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class MatchOperation implements FieldOperation { - - private MatchType matchingType; - private Case casing; - private Integer gramSize; - private Integer maxLength; - private MatchAlgorithm matchingAlgorithm; - private String exactMatchTerminator; - - public void setMatchingType(MatchType matchingType) { - this.matchingType = matchingType; - } - - public void setGramSize(Integer gramSize) { - this.gramSize = gramSize; - } - public void setMaxLength(Integer maxLength) { - this.maxLength = maxLength; - } - - public void setMatchingAlgorithm(MatchAlgorithm matchingAlgorithm) { - this.matchingAlgorithm = matchingAlgorithm; - } - - public void setExactMatchTerminator(String exactMatchTerminator) { - this.exactMatchTerminator = exactMatchTerminator; - } - - public void setCase(Case casing) { - this.casing = casing; - } - - public void apply(SDField field) { - if (matchingType != null) { - field.setMatchingType(matchingType); - } - if (casing != null) { - field.setMatchingCase(casing); - } - if (gramSize != null) { - field.getMatching().setGramSize(gramSize); - } - if (maxLength != null) { - field.getMatching().maxLength(maxLength); - } - if (matchingAlgorithm != null) { - field.setMatchingAlgorithm(matchingAlgorithm); - } - if (exactMatchTerminator != null) { - field.getMatching().setExactMatchTerminator(exactMatchTerminator); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java deleted file mode 100644 index cca7693119e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class NormalizingOperation implements FieldOperation { - - private final NormalizeLevel.Level level; - - public NormalizingOperation(String setting) { - if ("none".equals(setting)) { - this.level = NormalizeLevel.Level.NONE; - } else if ("codepoint".equals(setting)) { - this.level = NormalizeLevel.Level.CODEPOINT; - } else if ("lowercase".equals(setting)) { - this.level = NormalizeLevel.Level.LOWERCASE; - } else if ("accent".equals(setting)) { - this.level = NormalizeLevel.Level.ACCENT; - } else if ("all".equals(setting)) { - this.level = NormalizeLevel.Level.ACCENT; - } else { - throw new IllegalArgumentException("invalid normalizing setting: " + setting); - } - } - - public void apply(SDField field) { - field.setNormalizing(new NormalizeLevel(level, true)); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java deleted file mode 100644 index fc2dd303746..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -import java.util.List; - -/** - * @author Einar M R Rosenvinge - */ -public class QueryCommandOperation implements FieldOperation { - - private final List queryCommands = new java.util.ArrayList<>(0); - - public void addQueryCommand(String name) { - queryCommands.add(name); - } - - public void apply(SDField field) { - for (String command : queryCommands) { - field.addQueryCommand(command); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java deleted file mode 100644 index df9095ffcc5..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class RankOperation implements FieldOperation { - - private Boolean literal = null; - private Boolean filter = null; - private Boolean normal = null; - - public Boolean getLiteral() { return literal; } - public void setLiteral(Boolean literal) { this.literal = literal; } - - public Boolean getFilter() { return filter; } - public void setFilter(Boolean filter) { this.filter = filter; } - - public Boolean getNormal() { return normal; } - public void setNormal(Boolean n) { this.normal = n; } - - public void apply(SDField field) { - if (literal != null) { - field.getRanking().setLiteral(literal); - } - if (filter != null) { - field.getRanking().setFilter(filter); - } - if (normal != null) { - field.getRanking().setNormal(normal); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java deleted file mode 100644 index 970e5bc0678..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; - -/** - * @author Einar M R Rosenvinge - */ -public class RankTypeOperation implements FieldOperation { - - private String indexName; - private RankType type; - - public String getIndexName() { - return indexName; - } - public void setIndexName(String indexName) { - this.indexName = indexName; - } - - public RankType getType() { - return type; - } - public void setType(RankType type) { - this.type = type; - } - - public void apply(SDField field) { - if (indexName == null) { - field.setRankType(type); // Set default if the index is not specified. - } else { - Index index = field.getIndex(indexName); - if (index == null) { - index = new Index(indexName); - field.addIndex(index); - } - index.setRankType(type); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java deleted file mode 100644 index 148919c6de3..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; - -/** - * @author Einar M R Rosenvinge - */ -public class SortingOperation implements FieldOperation { - - private final String attributeName; - private Boolean ascending; - private Boolean descending; - private Sorting.Function function; - private Sorting.Strength strength; - private String locale; - - public SortingOperation(String attributeName) { - this.attributeName = attributeName; - } - - public String getAttributeName() { - return attributeName; - } - - public Boolean getAscending() { - return ascending; - } - - public void setAscending() { - this.ascending = true; - } - - public Boolean getDescending() { - return descending; - } - - public void setDescending() { - this.descending = true; - } - - public Sorting.Function getFunction() { - return function; - } - - public void setFunction(Sorting.Function function) { - this.function = function; - } - - public Sorting.Strength getStrength() { - return strength; - } - - public void setStrength(Sorting.Strength strength) { - this.strength = strength; - } - - public String getLocale() { - return locale; - } - - public void setLocale(String locale) { - this.locale = locale; - } - - public void apply(SDField field) { - Attribute attribute = field.getAttributes().get(attributeName); - if (attribute == null) { - attribute = new Attribute(attributeName, field.getDataType()); - field.addAttribute(attribute); - } - Sorting sorting = attribute.getSorting(); - - if (ascending != null) { - sorting.setAscending(); - } - if (descending != null) { - sorting.setDescending(); - } - if (function != null) { - sorting.setFunction(function); - } - if (strength != null) { - sorting.setStrength(strength); - } - if (locale != null) { - sorting.setLocale(locale); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java deleted file mode 100644 index 6ee137ff38c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; - -/** - * @author Einar M R Rosenvinge - */ -public class StemmingOperation implements FieldOperation { - - private String setting; - - public String getSetting() { - return setting; - } - - public void setSetting(String setting) { - this.setting = setting; - } - - public void apply(SDField field) { - field.setStemming(Stemming.get(setting)); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java deleted file mode 100644 index 285e1d97242..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; - -/** - * @author Einar M R Rosenvinge - */ -public class StructFieldOperation implements FieldOperation, FieldOperationContainer { - - private final String structFieldName; - private final List pendingOperations = new LinkedList<>(); - - public StructFieldOperation(String structFieldName) { - this.structFieldName = structFieldName; - } - - public void apply(SDField field) { - SDField structField = field.getStructField(structFieldName); - if (structField == null ) { - throw new IllegalArgumentException("Struct field '" + structFieldName + "' has not been defined in struct " + - "for field '" + field.getName() + "'."); - } - - applyOperations(structField); - } - - @Override - public void addOperation(FieldOperation op) { - pendingOperations.add(op); - } - - @Override - public void applyOperations(SDField field) { - if (pendingOperations.isEmpty()) return; - - Collections.sort(pendingOperations); - ListIterator ops = pendingOperations.listIterator(); - while (ops.hasNext()) { - FieldOperation op = ops.next(); - ops.remove(); - op.apply(field); - } - } - - @Override - public String getName() { - return structFieldName; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java deleted file mode 100644 index f776f7b1cec..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -/** - * @author Einar M R Rosenvinge - */ -public class SummaryInFieldLongOperation extends SummaryInFieldOperation { - - private DataType type; - private Boolean bold; - private Set destinations = new java.util.LinkedHashSet<>(); - - public SummaryInFieldLongOperation(String name) { - super(name); - } - - public SummaryInFieldLongOperation() { - super(null); - } - - public void setType(DataType type) { - this.type = type; - } - - public void setBold(Boolean bold) { - this.bold = bold; - } - - public void addDestination(String destination) { - destinations.add(destination); - } - - public Iterator destinationIterator() { - return destinations.iterator(); - } - - public void apply(SDField field) { - if (type == null) { - type = field.getDataType(); - } - SummaryField summary = new SummaryField(name, type); - applyToSummary(summary); - field.addSummaryField(summary); - } - - public void applyToSummary(SummaryField summary) { - if (transform != null) { - summary.setTransform(transform); - } - - if (bold != null) { - summary.setTransform(bold ? summary.getTransform().bold() : summary.getTransform().unbold()); - } - - for (SummaryField.Source source : sources) { - summary.addSource(source); - } - - for (String destination : destinations) { - summary.addDestination(destination); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java deleted file mode 100644 index 5bd99fb3c8f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; - -import java.util.Set; - -/** - * @author Einar M R Rosenvinge - */ -public abstract class SummaryInFieldOperation implements FieldOperation { - - protected String name; - protected SummaryTransform transform; - protected Set sources = new java.util.LinkedHashSet<>(); - - public SummaryInFieldOperation(String name) { - this.name = name; - } - - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public void setTransform(SummaryTransform transform) { - this.transform = transform; - } - - public SummaryTransform getTransform() { - return transform; - } - - public void addSource(String name) { - sources.add(new SummaryField.Source(name)); - } - - public void addSource(SummaryField.Source source) { - sources.add(source); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java deleted file mode 100644 index e5b96940ac7..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; - -/** - * @author Einar M R Rosenvinge - */ -public class SummaryInFieldShortOperation extends SummaryInFieldOperation { - - public SummaryInFieldShortOperation(String name) { - super(name); - } - - public void apply(SDField field) { - SummaryField ret = field.getSummaryField(name); - if (ret == null) { - ret = new SummaryField(name, field.getDataType()); - ret.addSource(field.getName()); - ret.addDestination("default"); - } - ret.setImplicit(false); - - ret.setTransform(transform); - for (SummaryField.Source source : sources) { - ret.addSource(source); - } - field.addSummaryField(ret); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java deleted file mode 100644 index 1378389c3e6..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; - -import java.util.Set; - -/** - * @author Einar M R Rosenvinge - */ -public class SummaryToOperation implements FieldOperation { - - private Set destinations = new java.util.LinkedHashSet<>(); - private String name; - - public void setName(String name) { - this.name = name; - } - - public void addDestination(String destination) { - destinations.add(destination); - } - - public void apply(SDField field) { - SummaryField summary; - summary = field.getSummaryField(name); - if (summary == null) { - summary = new SummaryField(field); - summary.addSource(field.getName()); - summary.addDestination("default"); - field.addSummaryField(summary); - } - summary.setImplicit(false); - - for (String destination : destinations) { - summary.addDestination(destination); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java deleted file mode 100644 index cc3a0ba687b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.SDField; - -/** - * @author Einar M R Rosenvinge - */ -public class WeightOperation implements FieldOperation { - - private int weight; - - public int getWeight() { - return weight; - } - - public void setWeight(int weight) { - this.weight = weight; - } - - public void apply(SDField field) { - field.setWeight(weight); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java deleted file mode 100644 index 11d731f814d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.document.WeightedSetDataType; - -/** - * @author Einar M R Rosenvinge - */ -public class WeightedSetOperation implements FieldOperation { - - private Boolean createIfNonExistent; - private Boolean removeIfZero; - - public Boolean getCreateIfNonExistent() { - return createIfNonExistent; - } - - public void setCreateIfNonExistent(Boolean createIfNonExistent) { - this.createIfNonExistent = createIfNonExistent; - } - - public Boolean getRemoveIfZero() { - return removeIfZero; - } - - public void setRemoveIfZero(Boolean removeIfZero) { - this.removeIfZero = removeIfZero; - } - - public void apply(SDField field) { - WeightedSetDataType ctype = (WeightedSetDataType) field.getDataType(); - - if (createIfNonExistent != null) { - field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), createIfNonExistent, - ctype.removeIfZero())); - } - - ctype = (WeightedSetDataType) field.getDataType(); - if (removeIfZero != null) { - field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), - ctype.createIfNonExistent(), removeIfZero)); - } - - ctype = (WeightedSetDataType) field.getDataType(); - for (Object o : field.getAttributes().values()) { - Attribute attribute = (Attribute) o; - attribute.setRemoveIfZero(ctype.removeIfZero()); - attribute.setCreateIfNonExistent(ctype.createIfNonExistent()); - } - } - - @Override - public int compareTo(FieldOperation other) { - // this operation should be executed first because it modifies the type of weighted sets, and other - // operation depends on the type of the weighted set - return -1; - } - - @Override - public String toString() { - return "WeightedSetOperation{" + - "createIfNonExistent=" + createIfNonExistent + - ", removeIfZero=" + removeIfZero + - '}'; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java deleted file mode 100644 index 0e79356abd2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.document.DataType; -import com.yahoo.document.DataTypeName; -import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; - -import java.util.Locale; - -/** - * Helper for converting ParsedField etc to SDField with settings - * - * @author arnej27959 - **/ -public class ConvertParsedFields { - - private final TypeResolver context; - - ConvertParsedFields(TypeResolver context) { - this.context = context; - } - - static void convertMatchSettings(SDField field, ParsedMatchSettings parsed) { - parsed.getMatchType().ifPresent(matchingType -> field.setMatchingType(matchingType)); - parsed.getMatchCase().ifPresent(casing -> field.setMatchingCase(casing)); - parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); - parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); - parsed.getMatchAlgorithm().ifPresent - (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); - parsed.getExactTerminator().ifPresent - (exactMatchTerminator -> field.getMatching().setExactMatchTerminator(exactMatchTerminator)); - } - - void convertSorting(SDField field, ParsedSorting parsed, String name) { - Attribute attribute = field.getAttributes().get(name); - if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); - field.addAttribute(attribute); - } - Sorting sorting = attribute.getSorting(); - if (parsed.getAscending()) { - sorting.setAscending(); - } else { - sorting.setDescending(); - } - parsed.getFunction().ifPresent(function -> sorting.setFunction(function)); - parsed.getStrength().ifPresent(strength -> sorting.setStrength(strength)); - parsed.getLocale().ifPresent(locale -> sorting.setLocale(locale)); - } - - void convertAttribute(SDField field, ParsedAttribute parsed) { - String name = parsed.name(); - String fieldName = field.getName(); - Attribute attribute = null; - if (fieldName.endsWith("." + name)) { - attribute = field.getAttributes().get(field.getName()); - } - if (attribute == null) { - attribute = field.getAttributes().get(name); - if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); - field.addAttribute(attribute); - } - } - attribute.setHuge(parsed.getHuge()); - attribute.setPaged(parsed.getPaged()); - attribute.setFastSearch(parsed.getFastSearch()); - if (parsed.getFastRank()) { - attribute.setFastRank(parsed.getFastRank()); - } - attribute.setFastAccess(parsed.getFastAccess()); - attribute.setMutable(parsed.getMutable()); - attribute.setEnableBitVectors(parsed.getEnableBitVectors()); - attribute.setEnableOnlyBitVector(parsed.getEnableOnlyBitVector()); - - // attribute.setTensorType(?) - - for (String alias : parsed.getAliases()) { - field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); - } - var distanceMetric = parsed.getDistanceMetric(); - if (distanceMetric.isPresent()) { - String upper = distanceMetric.get().toUpperCase(Locale.ENGLISH); - attribute.setDistanceMetric(Attribute.DistanceMetric.valueOf(upper)); - } - var sorting = parsed.getSorting(); - if (sorting.isPresent()) { - convertSorting(field, sorting.get(), name); - } - } - - private void convertRankType(SDField field, String indexName, String rankType) { - RankType type = RankType.fromString(rankType); - if (indexName == null || indexName.equals("")) { - field.setRankType(type); // Set default if the index is not specified. - } else { - Index index = field.getIndex(indexName); - if (index == null) { - index = new Index(indexName); - field.addIndex(index); - } - index.setRankType(type); - } - } - - private void convertNormalizing(SDField field, String setting) { - NormalizeLevel.Level level; - if ("none".equals(setting)) { - level = NormalizeLevel.Level.NONE; - } else if ("codepoint".equals(setting)) { - level = NormalizeLevel.Level.CODEPOINT; - } else if ("lowercase".equals(setting)) { - level = NormalizeLevel.Level.LOWERCASE; - } else if ("accent".equals(setting)) { - level = NormalizeLevel.Level.ACCENT; - } else if ("all".equals(setting)) { - level = NormalizeLevel.Level.ACCENT; - } else { - throw new IllegalArgumentException("invalid normalizing setting: " + setting); - } - field.setNormalizing(new NormalizeLevel(level, true)); - } - - // from grammar, things that can be inside struct-field block - private void convertCommonFieldSettings(SDField field, ParsedField parsed) { - convertMatchSettings(field, parsed.matchSettings()); - var indexing = parsed.getIndexing(); - if (indexing.isPresent()) { - field.setIndexingScript(indexing.get().script()); - } - parsed.getWeight().ifPresent(value -> field.setWeight(value)); - parsed.getStemming().ifPresent(value -> field.setStemming(value)); - parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); - for (var attribute : parsed.getAttributes()) { - convertAttribute(field, attribute); - } - for (var summaryField : parsed.getSummaryFields()) { - var dataType = field.getDataType(); - var otherType = summaryField.getType(); - if (otherType != null) { - dataType = context.resolveType(otherType); - } - convertSummaryField(field, summaryField, dataType); - } - for (String command : parsed.getQueryCommands()) { - field.addQueryCommand(command); - } - for (var structField : parsed.getStructFields()) { - convertStructField(field, structField); - } - if (parsed.hasLiteral()) { - field.getRanking().setLiteral(true); - } - if (parsed.hasFilter()) { - field.getRanking().setFilter(true); - } - if (parsed.hasNormal()) { - field.getRanking().setNormal(true); - } - } - - private void convertStructField(SDField field, ParsedField parsed) { - SDField structField = field.getStructField(parsed.name()); - if (structField == null ) { - throw new IllegalArgumentException("Struct field '" + parsed.name() + "' has not been defined in struct " + - "for field '" + field.getName() + "'."); - } - convertCommonFieldSettings(structField, parsed); - } - - private void convertExtraFieldSettings(SDField field, ParsedField parsed) { - String name = parsed.name(); - for (var dictOp : parsed.getDictionaryOptions()) { - var dictionary = field.getOrSetDictionary(); - switch (dictOp) { - case HASH: dictionary.updateType(Dictionary.Type.HASH); break; - case BTREE: dictionary.updateType(Dictionary.Type.BTREE); break; - case CASED: dictionary.updateMatch(Case.CASED); break; - case UNCASED: dictionary.updateMatch(Case.UNCASED); break; - } - } - for (var index : parsed.getIndexes()) { - convertIndex(field, index); - } - for (var alias : parsed.getAliases()) { - field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); - } - parsed.getRankTypes().forEach((indexName, rankType) -> convertRankType(field, indexName, rankType)); - parsed.getSorting().ifPresent(sortInfo -> convertSorting(field, sortInfo, name)); - if (parsed.hasBolding()) { - // TODO must it be so ugly: - SummaryField summaryField = field.getSummaryField(name, true); - summaryField.addSource(name); - summaryField.addDestination("default"); - summaryField.setTransform(summaryField.getTransform().bold()); - } - } - - static void convertSummaryFieldSettings(SummaryField summary, ParsedSummaryField parsed) { - var transform = SummaryTransform.NONE; - if (parsed.getMatchedElementsOnly()) { - transform = SummaryTransform.MATCHED_ELEMENTS_FILTER; - } else if (parsed.getDynamic()) { - transform = SummaryTransform.DYNAMICTEASER; - } - if (parsed.getBolded()) { - transform = transform.bold(); - } - summary.setTransform(transform); - for (String source : parsed.getSources()) { - summary.addSource(source); - } - for (String destination : parsed.getDestinations()) { - summary.addDestination(destination); - } - summary.setImplicit(false); - } - - private void convertSummaryField(SDField field, ParsedSummaryField parsed, DataType type) { - var summary = new SummaryField(parsed.name(), type); - convertSummaryFieldSettings(summary, parsed); - summary.addDestination("default"); - if (parsed.getSources().isEmpty()) { - summary.addSource(field.getName()); - } - field.addSummaryField(summary); - } - - private void convertIndex(SDField field, ParsedIndex parsed) { - String indexName = parsed.name(); - Index index = field.getIndex(indexName); - if (index == null) { - index = new Index(indexName); - field.addIndex(index); - } - convertIndexSettings(index, parsed); - } - - private void convertIndexSettings(Index index, ParsedIndex parsed) { - parsed.getPrefix().ifPresent(prefix -> index.setPrefix(prefix)); - for (String alias : parsed.getAliases()) { - index.addAlias(alias); - } - parsed.getStemming().ifPresent(stemming -> index.setStemming(stemming)); - var arity = parsed.getArity(); - var lowerBound = parsed.getLowerBound(); - var upperBound = parsed.getUpperBound(); - var densePostingListThreshold = parsed.getDensePostingListThreshold(); - if (arity.isPresent() || - lowerBound.isPresent() || - upperBound.isPresent() || - densePostingListThreshold.isPresent()) - { - var bid = new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold); - index.setBooleanIndexDefiniton(bid); - } - parsed.getEnableBm25().ifPresent(enableBm25 -> index.setInterleavedFeatures(enableBm25)); - parsed.getHnswIndexParams().ifPresent - (hnswIndexParams -> index.setHnswIndexParams(hnswIndexParams)); - } - - SDField convertDocumentField(Schema schema, SDDocumentType document, ParsedField parsed) { - String name = parsed.name(); - DataType dataType = context.resolveType(parsed.getType()); - var field = new SDField(document, name, dataType); - convertCommonFieldSettings(field, parsed); - convertExtraFieldSettings(field, parsed); - document.addField(field); - return field; - } - - void convertExtraField(Schema schema, ParsedField parsed) { - String name = parsed.name(); - DataType dataType = context.resolveType(parsed.getType()); - var field = new SDField(schema.getDocument(), name, dataType); - convertCommonFieldSettings(field, parsed); - convertExtraFieldSettings(field, parsed); - schema.addExtraField(field); - } - - void convertExtraIndex(Schema schema, ParsedIndex parsed) { - Index index = new Index(parsed.name()); - convertIndexSettings(index, parsed); - schema.addIndex(index); - } - - SDDocumentType convertStructDeclaration(Schema schema, SDDocumentType document, ParsedStruct parsed) { - // TODO - can we cleanup this mess - var structProxy = new SDDocumentType(parsed.name(), schema); - for (var parsedField : parsed.getFields()) { - var fieldType = context.resolveType(parsedField.getType()); - var field = new SDField(document, parsedField.name(), fieldType); - convertCommonFieldSettings(field, parsedField); - structProxy.addField(field); - if (parsedField.hasIdOverride()) { - structProxy.setFieldId(field, parsedField.idOverride()); - } - } - for (String inherit : parsed.getInherited()) { - structProxy.inherit(new DataTypeName(inherit)); - } - structProxy.setStruct(context.resolveStruct(parsed)); - return structProxy; - } - - void convertAnnotation(Schema schema, SDDocumentType document, ParsedAnnotation parsed) { - SDAnnotationType annType = context.resolveAnnotation(parsed.name()); - var withStruct = parsed.getStruct(); - if (withStruct.isPresent()) { - ParsedStruct parsedStruct = withStruct.get(); - SDDocumentType structProxy = convertStructDeclaration(schema, document, parsedStruct); - structProxy.setStruct(context.resolveStruct(parsedStruct)); - annType.setSdDocType(structProxy); - } - document.addAnnotation(annType); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java deleted file mode 100644 index 47d770f609e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; - -import java.util.List; - -/** - * Helper for converting ParsedRankProfile etc to RankProfile with settings - * - * @author arnej27959 - */ -public class ConvertParsedRanking { - - private final RankProfileRegistry rankProfileRegistry; - - // for unit test - ConvertParsedRanking() { - this(new RankProfileRegistry()); - } - - public ConvertParsedRanking(RankProfileRegistry rankProfileRegistry) { - this.rankProfileRegistry = rankProfileRegistry; - } - - private RankProfile makeRankProfile(Schema schema, String name) { - if (name.equals("default")) { - return rankProfileRegistry.get(schema, "default"); - } - return new RankProfile(name, schema, rankProfileRegistry); - } - - void convertRankProfile(Schema schema, ParsedRankProfile parsed) { - RankProfile profile = makeRankProfile(schema, parsed.name()); - for (String name : parsed.getInherited()) - profile.inherit(name); - - parsed.isStrict().ifPresent(value -> profile.setStrict(value)); - - for (var constant : parsed.getConstants().values()) - profile.add(constant); - - for (var onnxModel : parsed.getOnnxModels()) - profile.add(onnxModel); - - for (var input : parsed.getInputs().entrySet()) - profile.addInput(input.getKey(), input.getValue()); - - for (var func : parsed.getFunctions()) { - String name = func.name(); - List parameters = func.getParameters(); - String expression = func.getExpression(); - boolean inline = func.getInline(); - profile.addFunction(name, parameters, expression, inline); - } - - parsed.getRankScoreDropLimit().ifPresent - (value -> profile.setRankScoreDropLimit(value)); - parsed.getTermwiseLimit().ifPresent - (value -> profile.setTermwiseLimit(value)); - parsed.getPostFilterThreshold().ifPresent - (value -> profile.setPostFilterThreshold(value)); - parsed.getApproximateThreshold().ifPresent - (value -> profile.setApproximateThreshold(value)); - parsed.getKeepRankCount().ifPresent - (value -> profile.setKeepRankCount(value)); - parsed.getMinHitsPerThread().ifPresent - (value -> profile.setMinHitsPerThread(value)); - parsed.getNumSearchPartitions().ifPresent - (value -> profile.setNumSearchPartitions(value)); - parsed.getNumThreadsPerSearch().ifPresent - (value -> profile.setNumThreadsPerSearch(value)); - parsed.getReRankCount().ifPresent - (value -> profile.setRerankCount(value)); - - parsed.getMatchPhaseSettings().ifPresent - (value -> profile.setMatchPhaseSettings(value)); - - parsed.getFirstPhaseExpression().ifPresent - (value -> profile.setFirstPhaseRanking(value)); - parsed.getSecondPhaseExpression().ifPresent - (value -> profile.setSecondPhaseRanking(value)); - - for (var value : parsed.getMatchFeatures()) { - profile.addMatchFeatures(value); - } - for (var value : parsed.getRankFeatures()) { - profile.addRankFeatures(value); - } - for (var value : parsed.getSummaryFeatures()) { - profile.addSummaryFeatures(value); - } - - parsed.getInheritedMatchFeatures().ifPresent - (value -> profile.setInheritedMatchFeatures(value)); - parsed.getInheritedSummaryFeatures().ifPresent - (value -> profile.setInheritedSummaryFeatures(value)); - if (parsed.getIgnoreDefaultRankFeatures()) { - profile.setIgnoreDefaultRankFeatures(true); - } - - for (var mutateOp : parsed.getMutateOperations()) { - profile.addMutateOperation(mutateOp); - } - parsed.getFieldsWithRankFilter().forEach - ((fieldName, isFilter) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.PREFERBITVECTOR, isFilter)); - - parsed.getFieldsWithRankWeight().forEach - ((fieldName, weight) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.WEIGHT, weight)); - - parsed.getFieldsWithRankType().forEach - ((fieldName, rankType) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.fromString(rankType))); - - parsed.getRankProperties().forEach - ((key, values) -> {for (String value : values) profile.addRankProperty(key, value);}); - - // always? - rankProfileRegistry.add(profile); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java deleted file mode 100644 index 4c32f11c20d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.searchdefinition.DefaultRankProfile; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.UnrankedRankProfile; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; -import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * Class converting a collection of schemas from the intermediate format. - * - * @author arnej27959 - **/ -public class ConvertParsedSchemas { - - private final List orderedInput; - private final DocumentTypeManager docMan; - private final ApplicationPackage applicationPackage; - private final FileRegistry fileRegistry; - private final DeployLogger deployLogger; - private final ModelContext.Properties properties; - private final RankProfileRegistry rankProfileRegistry; - private final boolean documentsOnly; - private final ConvertParsedTypes typeConverter; - - // for unit test - ConvertParsedSchemas(List orderedInput, - DocumentTypeManager documentTypeManager) - { - this(orderedInput, documentTypeManager, - MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - new BaseDeployLogger(), - new TestProperties(), - new RankProfileRegistry(), - true); - } - - public ConvertParsedSchemas(List orderedInput, - DocumentTypeManager documentTypeManager, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - boolean documentsOnly) - { - this.orderedInput = orderedInput; - this.docMan = documentTypeManager; - this.applicationPackage = applicationPackage; - this.fileRegistry = fileRegistry; - this.deployLogger = deployLogger; - this.properties = properties; - this.rankProfileRegistry = rankProfileRegistry; - this.documentsOnly = documentsOnly; - this.typeConverter = new ConvertParsedTypes(orderedInput, docMan); - } - - private final Map convertedDocuments = new LinkedHashMap<>(); - - public List convertToSchemas() { - typeConverter.convert(false); - var resultList = new ArrayList(); - for (var parsed : orderedInput) { - Optional inherited; - var inheritList = parsed.getInherited(); - if (inheritList.size() == 0) { - inherited = Optional.empty(); - } else if (inheritList.size() == 1) { - inherited = Optional.of(inheritList.get(0)); - } else { - throw new IllegalArgumentException("schema " + parsed.name() + "cannot inherit more than once"); - } - Schema schema = parsed.getDocumentWithoutSchema() - ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) - : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); - convertSchema(schema, parsed); - resultList.add(schema); - } - return resultList; - } - - private void convertDocument(Schema schema, ParsedDocument parsed, - ConvertParsedFields fieldConverter) - { - SDDocumentType document = new SDDocumentType(parsed.name()); - for (var struct : parsed.getStructs()) { - var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); - document.addType(structProxy); - } - for (String inherit : parsed.getInherited()) { - var parent = convertedDocuments.get(inherit); - assert(parent != null); - document.inherit(parent); - } - for (var annotation : parsed.getAnnotations()) { - fieldConverter.convertAnnotation(schema, document, annotation); - } - for (var field : parsed.getFields()) { - var sdf = fieldConverter.convertDocumentField(schema, document, field); - if (field.hasIdOverride()) { - document.setFieldId(sdf, field.idOverride()); - } - } - convertedDocuments.put(parsed.name(), document); - schema.addDocument(document); - } - - private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { - var docsum = new DocumentSummary(parsed.name(), schema); - var inheritList = parsed.getInherited(); - if (inheritList.size() == 1) { - docsum.setInherited(inheritList.get(0)); - } else if (inheritList.size() != 0) { - throw new IllegalArgumentException("document-summary "+parsed.name()+" cannot inherit more than once"); - } - if (parsed.getFromDisk()) { - docsum.setFromDisk(true); - } - if (parsed.getOmitSummaryFeatures()) { - docsum.setOmitSummaryFeatures(true); - } - for (var parsedField : parsed.getSummaryFields()) { - DataType dataType = typeContext.resolveType(parsedField.getType()); - var summaryField = new SummaryField(parsedField.name(), dataType); - // XXX does not belong here: - summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); - ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); - docsum.add(summaryField); - } - schema.addSummary(docsum); - } - - private void convertImportField(Schema schema, ParsedSchema.ImportedField f) { - // needs rethinking - var importedFields = schema.temporaryImportedFields().get(); - if (importedFields.hasField(f.asFieldName)) { - throw new IllegalArgumentException("For schema '" + schema.getName() + - "', import field as '" + f.asFieldName + - "': Field already imported"); - } - importedFields.add(new TemporaryImportedField(f.asFieldName, f.refFieldName, f.foreignFieldName)); - } - - private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { - String setName = parsed.name(); - for (String field : parsed.getFieldNames()) { - schema.fieldSets().addUserFieldSetItem(setName, field); - } - for (String command : parsed.getQueryCommands()) { - schema.fieldSets().userFieldSets().get(setName).queryCommands().add(command); - } - if (parsed.getMatchSettings().isPresent()) { - // same ugliness as SDParser.jj used to have: - var tmp = new SDField(setName, DataType.STRING); - ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); - schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); - } - } - - private void convertSchema(Schema schema, ParsedSchema parsed) { - if (parsed.hasStemming()) { - schema.setStemming(parsed.getStemming()); - } - parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); - var typeContext = typeConverter.makeContext(parsed.getDocument()); - var fieldConverter = new ConvertParsedFields(typeContext); - convertDocument(schema, parsed.getDocument(), fieldConverter); - for (var field : parsed.getFields()) { - fieldConverter.convertExtraField(schema, field); - } - for (var index : parsed.getIndexes()) { - fieldConverter.convertExtraIndex(schema, index); - } - for (var docsum : parsed.getDocumentSummaries()) { - convertDocumentSummary(schema, docsum, typeContext); - } - for (var importedField : parsed.getImportedFields()) { - convertImportField(schema, importedField); - } - for (var fieldSet : parsed.getFieldSets()) { - convertFieldSet(schema, fieldSet); - } - if (documentsOnly) { - return; // skip ranking-only content, not used for document type generation - } - for (var constant : parsed.getConstants()) - schema.add(constant); - for (var onnxModel : parsed.getOnnxModels()) - schema.add(onnxModel); - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); - var rankConverter = new ConvertParsedRanking(rankProfileRegistry); - for (var rankProfile : parsed.getRankProfiles()) { - rankConverter.convertRankProfile(schema, rankProfile); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java deleted file mode 100644 index 7311c8a4216..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.PositionDataType; -import com.yahoo.document.StructDataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.documentmodel.OwnedStructDataType; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Helper class for converting ParsedType instances to DataType - * - * @author arnej27959 - **/ -public class ConvertParsedTypes { - - private final List orderedInput; - private final DocumentTypeManager docMan; - - ConvertParsedTypes(List input) { - this.orderedInput = input; - this.docMan = new DocumentTypeManager(); - } - - public ConvertParsedTypes(List input, DocumentTypeManager docMan) { - this.orderedInput = input; - this.docMan = docMan; - } - - public void convert(boolean andRegister) { - startDataTypes(); - fillDataTypes(); - if (andRegister) { - registerDataTypes(); - } - } - - private Map documentsFromSchemas = new HashMap<>(); - private Map structsFromSchemas = new HashMap<>(); - private Map annotationsFromSchemas = new HashMap<>(); - - private void startDataTypes() { - for (var schema : orderedInput) { - String name = schema.getDocument().name(); - documentsFromSchemas.put(name, new DocumentType(name)); - } - for (var schema : orderedInput) { - var doc = schema.getDocument(); - for (var struct : doc.getStructs()) { - String structId = doc.name() + "->" + struct.name(); - var dt = new OwnedStructDataType(struct.name(), doc.name()); - structsFromSchemas.put(structId, dt); - } - for (var annotation : doc.getAnnotations()) { - String annId = doc.name() + "->" + annotation.name(); - var at = new SDAnnotationType(annotation.name()); - annotationsFromSchemas.put(annId, at); - for (String inherit : annotation.getInherited()) { - at.inherit(inherit); - } - var withStruct = annotation.getStruct(); - if (withStruct.isPresent()) { - ParsedStruct struct = withStruct.get(); - String structId = doc.name() + "->" + struct.name(); - var old = structsFromSchemas.put(structId, new OwnedStructDataType(struct.name(), doc.name())); - assert(old == null); - } - } - } - } - - void fillAnnotationStruct(ParsedAnnotation annotation) { - var withStruct = annotation.getStruct(); - if (withStruct.isPresent()) { - var doc = annotation.getOwnerDoc(); - var toFill = findStructFromParsed(withStruct.get()); - for (ParsedField field : withStruct.get().getFields()) { - var t = resolveFromContext(field.getType(), doc); - var f = field.hasIdOverride() - ? new com.yahoo.document.Field(field.name(), field.idOverride(), t) - : new com.yahoo.document.Field(field.name(), t); - toFill.addField(f); - } - for (var parent : annotation.getResolvedInherits()) { - parent.getStruct().ifPresent - (ps -> toFill.inherit(findStructFromParsed(ps))); - } - var at = findAnnotationFromParsed(annotation); - at.setDataType(toFill); - } - } - - private void fillDataTypes() { - for (var schema : orderedInput) { - var doc = schema.getDocument(); - for (var annotation : doc.getAnnotations()) { - var at = findAnnotationFromParsed(annotation); - for (var parent : annotation.getResolvedInherits()) { - at.inherit(findAnnotationFromParsed(parent)); - } - fillAnnotationStruct(annotation); - } - for (var struct : doc.getStructs()) { - var toFill = findStructFromParsed(struct); - // evil ugliness - for (ParsedField field : struct.getFields()) { - if (! field.hasIdOverride()) { - var t = resolveFromContext(field.getType(), doc); - var f = new com.yahoo.document.Field(field.name(), t); - toFill.addField(f); - } - } - for (ParsedField field : struct.getFields()) { - if (field.hasIdOverride()) { - var t = resolveFromContext(field.getType(), doc); - var f = new com.yahoo.document.Field(field.name(), field.idOverride(), t); - toFill.addField(f); - } - } - for (var inherit : struct.getResolvedInherits()) { - var parent = findStructFromParsed(inherit); - // ensure a nice, compatible exception message - for (var field : toFill.getFields()) { - if (parent.hasField(field)) { - for (var base : parent.getInheritedTypes()) { - if (base.hasField(field)) { - parent = base; - } - } - throw new IllegalArgumentException - ("In document " + doc.name() + ": struct " + struct.name() + - " cannot inherit from " + parent.getName() + " and redeclare field " + field.getName()); - } - } - toFill.inherit(parent); - } - } - var docToFill = documentsFromSchemas.get(doc.name()); - Map> fieldSets = new HashMap<>(); - List inDocFields = new ArrayList<>(); - for (var docField : doc.getFields()) { - String name = docField.name(); - var t = resolveFromContext(docField.getType(), doc); - var f = new com.yahoo.document.Field(docField.name(), t); - docToFill.addField(f); - if (docField.hasIdOverride()) { - f.setId(docField.idOverride(), docToFill); - } - inDocFields.add(name); - } - fieldSets.put("[document]", inDocFields); - for (var extraField : schema.getFields()) { - String name = extraField.name(); - if (docToFill.hasField(name)) continue; - var t = resolveFromContext(extraField.getType(), doc); - var f = new com.yahoo.document.Field(name, t); - docToFill.addField(f); - } - for (var fieldset : schema.getFieldSets()) { - fieldSets.put(fieldset.name(), fieldset.getFieldNames()); - } - docToFill.addFieldSets(fieldSets); - for (String inherit : doc.getInherited()) { - docToFill.inherit(findDocFromSchemas(inherit)); - } - } - } - - private StructDataType findStructFromParsed(ParsedStruct resolved) { - String structId = resolved.getOwnerName() + "->" + resolved.name(); - var struct = structsFromSchemas.get(structId); - assert(struct != null); - return struct; - } - - private StructDataType findStructFromSchemas(String name, ParsedDocument context) { - var resolved = context.findParsedStruct(name); - if (resolved == null) { - throw new IllegalArgumentException("no struct named " + name + " in context " + context); - } - return findStructFromParsed(resolved); - } - - private SDAnnotationType findAnnotationFromSchemas(String name, ParsedDocument context) { - var resolved = context.findParsedAnnotation(name); - String annotationId = resolved.getOwnerName() + "->" + resolved.name(); - var annotation = annotationsFromSchemas.get(annotationId); - if (annotation == null) { - throw new IllegalArgumentException("no annotation named " + name + " in context " + context); - } - return annotation; - } - - private SDAnnotationType findAnnotationFromParsed(ParsedAnnotation resolved) { - String annotationId = resolved.getOwnerName() + "->" + resolved.name(); - var annotation = annotationsFromSchemas.get(annotationId); - if (annotation == null) { - throw new IllegalArgumentException("no annotation " + resolved.name() + " in " + resolved.getOwnerName()); - } - return annotation; - } - - private DataType createArray(ParsedType pType, ParsedDocument context) { - DataType nested = resolveFromContext(pType.nestedType(), context); - return DataType.getArray(nested); - } - - private DataType createWset(ParsedType pType, ParsedDocument context) { - DataType nested = resolveFromContext(pType.nestedType(), context); - boolean cine = pType.getCreateIfNonExistent(); - boolean riz = pType.getRemoveIfZero(); - return new WeightedSetDataType(nested, cine, riz); - } - - private DataType createMap(ParsedType pType, ParsedDocument context) { - DataType kt = resolveFromContext(pType.mapKeyType(), context); - DataType vt = resolveFromContext(pType.mapValueType(), context); - return DataType.getMap(kt, vt); - } - - private DocumentType findDocFromSchemas(String name) { - var dt = documentsFromSchemas.get(name); - if (dt == null) { - throw new IllegalArgumentException("missing document type for: " + name); - } - return dt; - } - - private DataType createAnnRef(ParsedType pType, ParsedDocument context) { - SDAnnotationType annotation = findAnnotationFromSchemas(pType.getNameOfReferencedAnnotation(), context); - return new AnnotationReferenceDataType(annotation); - } - - private DataType createDocRef(ParsedType pType) { - var ref = pType.getReferencedDocumentType(); - assert(ref.getVariant() == ParsedType.Variant.DOCUMENT); - return new NewDocumentReferenceDataType(findDocFromSchemas(ref.name())); - } - - private DataType getBuiltinType(String name) { - switch (name) { - case "bool": return DataType.BOOL; - case "byte": return DataType.BYTE; - case "int": return DataType.INT; - case "long": return DataType.LONG; - case "string": return DataType.STRING; - case "float": return DataType.FLOAT; - case "double": return DataType.DOUBLE; - case "uri": return DataType.URI; - case "predicate": return DataType.PREDICATE; - case "raw": return DataType.RAW; - case "tag": return DataType.TAG; - case "float16": return DataType.FLOAT16; - default: - throw new IllegalArgumentException("Unknown builtin type: "+name); - } - } - - private DataType resolveFromContext(ParsedType pType, ParsedDocument context) { - String name = pType.name(); - switch (pType.getVariant()) { - case NONE: return DataType.NONE; - case BUILTIN: return getBuiltinType(name); - case POSITION: return PositionDataType.INSTANCE; - case ARRAY: return createArray(pType, context); - case WSET: return createWset(pType, context); - case MAP: return createMap(pType, context); - case TENSOR: return DataType.getTensor(pType.getTensorType()); - case DOC_REFERENCE: return createDocRef(pType); - case ANN_REFERENCE: return createAnnRef(pType, context); - case DOCUMENT: return findDocFromSchemas(name); - case STRUCT: return findStructFromSchemas(name, context); - case UNKNOWN: - // fallthrough - } - // unknown is probably struct - var found = context.findParsedStruct(name); - if (found != null) { - pType.setVariant(ParsedType.Variant.STRUCT); - return findStructFromSchemas(name, context); - } - if (documentsFromSchemas.containsKey(name)) { - pType.setVariant(ParsedType.Variant.DOCUMENT); - return findDocFromSchemas(name); - } - throw new IllegalArgumentException("unknown type named '" + name + "' in context "+context); - } - - @SuppressWarnings("deprecation") - private void registerDataTypes() { - for (DataType t : structsFromSchemas.values()) { - docMan.register(t); - } - for (DocumentType t : documentsFromSchemas.values()) { - docMan.registerDocumentType(t); - } - for (SDAnnotationType t : annotationsFromSchemas.values()) { - docMan.getAnnotationTypeRegistry().register(t); - } - } - - public class TypeResolver { - private final ParsedDocument context; - public DataType resolveType(ParsedType parsed) { - return resolveFromContext(parsed, context); - } - public DataType resolveStruct(ParsedStruct parsed) { - String structId = context.name() + "->" + parsed.name(); - var r = structsFromSchemas.get(structId); - if (r == null) { - throw new IllegalArgumentException("no datatype found for struct: " + structId); - } - return r; - } - public SDAnnotationType resolveAnnotation(String name) { - return findAnnotationFromSchemas(name, context); - } - TypeResolver(ParsedDocument context) { - this.context = context; - } - } - - public TypeResolver makeContext(ParsedDocument doc) { - return new TypeResolver(doc); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java deleted file mode 100644 index e46a6882bc7..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; - -import java.util.ArrayList; -import java.util.List; - -/** - * Class converting a collection of schemas from the intermediate format. - * - * @author arnej27959 - **/ -public class ConvertSchemaCollection { - - private final IntermediateCollection input; - private final List orderedInput = new ArrayList<>(); - private final DocumentTypeManager docMan; - private final ApplicationPackage applicationPackage; - private final FileRegistry fileRegistry; - private final DeployLogger deployLogger; - private final ModelContext.Properties properties; - private final RankProfileRegistry rankProfileRegistry; - private final boolean documentsOnly; - - // for unit test - ConvertSchemaCollection(IntermediateCollection input, - DocumentTypeManager documentTypeManager) - { - this(input, documentTypeManager, - MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - new BaseDeployLogger(), - new TestProperties(), - new RankProfileRegistry(), - true); - } - - public ConvertSchemaCollection(IntermediateCollection input, - DocumentTypeManager documentTypeManager, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - boolean documentsOnly) - { - this.input = input; - this.docMan = documentTypeManager; - this.applicationPackage = applicationPackage; - this.fileRegistry = fileRegistry; - this.deployLogger = deployLogger; - this.properties = properties; - this.rankProfileRegistry = rankProfileRegistry; - this.documentsOnly = documentsOnly; - - input.resolveInternalConnections(); - order(); - pushTypesToDocuments(); - } - - void order() { - var map = input.getParsedSchemas(); - for (var schema : map.values()) { - findOrdering(schema); - } - } - - void findOrdering(ParsedSchema schema) { - if (orderedInput.contains(schema)) return; - for (var parent : schema.getAllResolvedInherits()) { - findOrdering(parent); - } - orderedInput.add(schema); - } - - void pushTypesToDocuments() { - for (var schema : orderedInput) { - for (var struct : schema.getStructs()) { - schema.getDocument().addStruct(struct); - } - for (var annotation : schema.getAnnotations()) { - schema.getDocument().addAnnotation(annotation); - } - } - } - - private ConvertParsedTypes typeConverter; - - public void convertTypes() { - typeConverter = new ConvertParsedTypes(orderedInput, docMan); - typeConverter.convert(true); - } - - public List convertToSchemas() { - resolveStructInheritance(); - resolveAnnotationInheritance(); - addMissingAnnotationStructs(); - var converter = new ConvertParsedSchemas(orderedInput, - docMan, - applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - documentsOnly); - return converter.convertToSchemas(); - } - - private void resolveStructInheritance() { - List all = new ArrayList<>(); - for (var schema : orderedInput) { - var doc = schema.getDocument(); - for (var struct : doc.getStructs()) { - for (String inherit : struct.getInherited()) { - var parent = doc.findParsedStruct(inherit); - if (parent == null) { - throw new IllegalArgumentException("Can not find parent for "+struct+" in "+doc); - } - struct.resolveInherit(inherit, parent); - } - all.add(struct); - } - } - List seen = new ArrayList<>(); - for (ParsedStruct struct : all) { - inheritanceCycleCheck(struct, seen); - } - } - - private void resolveAnnotationInheritance() { - List all = new ArrayList(); - for (var schema : orderedInput) { - var doc = schema.getDocument(); - for (var annotation : doc.getAnnotations()) { - for (String inherit : annotation.getInherited()) { - var parent = doc.findParsedAnnotation(inherit); - if (parent == null) { - throw new IllegalArgumentException("Can not find parent for "+annotation+" in "+doc); - } - annotation.resolveInherit(inherit, parent); - } - all.add(annotation); - } - } - List seen = new ArrayList<>(); - for (ParsedAnnotation annotation : all) { - inheritanceCycleCheck(annotation, seen); - } - } - - private void fixupAnnotationStruct(ParsedAnnotation parsed) { - for (var parent : parsed.getResolvedInherits()) { - fixupAnnotationStruct(parent); - parent.getStruct().ifPresent(ps -> { - var myStruct = parsed.ensureStruct(); - if (! myStruct.getInherited().contains(ps.name())) { - myStruct.inherit(ps.name()); - myStruct.resolveInherit(ps.name(), ps); - } - }); - } - } - - private void addMissingAnnotationStructs() { - for (var schema : orderedInput) { - var doc = schema.getDocument(); - for (var annotation : doc.getAnnotations()) { - fixupAnnotationStruct(annotation); - } - } - } - - private void inheritanceCycleCheck(ParsedStruct struct, List seen) { - String name = struct.name(); - if (seen.contains(name)) { - seen.add(name); - throw new IllegalArgumentException("Inheritance/reference cycle for structs: " + - String.join(" -> ", seen)); - } - seen.add(name); - for (ParsedStruct parent : struct.getResolvedInherits()) { - inheritanceCycleCheck(parent, seen); - } - seen.remove(name); - } - - private void inheritanceCycleCheck(ParsedAnnotation annotation, List seen) { - String name = annotation.name(); - if (seen.contains(name)) { - seen.add(name); - throw new IllegalArgumentException("Inheritance/reference cycle for annotations: " + - String.join(" -> ", seen)); - } - seen.add(name); - for (ParsedAnnotation parent : annotation.getResolvedInherits()) { - inheritanceCycleCheck(parent, seen); - } - seen.remove(name); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/DictionaryOption.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/DictionaryOption.java deleted file mode 100644 index 095add74dd2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/DictionaryOption.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.yahoo.searchdefinition.parser; - -public enum DictionaryOption { - HASH, BTREE, CASED, UNCASED -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java deleted file mode 100644 index dbdcd4057e9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Class resolving some inheritance relationships. - * - * @author arnej27959 - **/ -public class InheritanceResolver { - - private final Map parsedSchemas; - private final Map parsedDocs = new HashMap<>(); - private final Map schemaForDocs = new HashMap<>(); - - public InheritanceResolver(Map parsedSchemas) { - this.parsedSchemas = parsedSchemas; - } - - private void inheritanceCycleCheck(ParsedSchema schema, List seen) { - String name = schema.name(); - if (seen.contains(name)) { - seen.add(name); - throw new IllegalArgumentException("Inheritance/reference cycle for schemas: " + - String.join(" -> ", seen)); - } - seen.add(name); - for (ParsedSchema parent : schema.getAllResolvedInherits()) { - inheritanceCycleCheck(parent, seen); - } - seen.remove(name); - } - - private void resolveSchemaInheritance() { - for (ParsedSchema schema : parsedSchemas.values()) { - for (String inherit : schema.getInherited()) { - var parent = parsedSchemas.get(inherit); - if (parent == null) { - throw new IllegalArgumentException("schema '" + schema.name() + "' inherits '" + inherit + "', but this schema does not exist"); - } - schema.resolveInherit(inherit, parent); - } - } - } - - private void checkSchemaCycles() { - List seen = new ArrayList<>(); - for (ParsedSchema schema : parsedSchemas.values()) { - inheritanceCycleCheck(schema, seen); - } - } - - private void resolveDocumentInheritance() { - for (ParsedSchema schema : parsedSchemas.values()) { - if (! schema.hasDocument()) { - throw new IllegalArgumentException("For schema '" + schema.name() + - "': A search specification must have an equally named document inside of it."); - } - ParsedDocument doc = schema.getDocument(); - var old = parsedDocs.put(doc.name(), doc); - if (old != null) { - throw new IllegalArgumentException("duplicate document declaration for " + doc.name()); - } - schemaForDocs.put(doc.name(), schema); - for (String docInherit : doc.getInherited()) { - schema.inheritByDocument(docInherit); - } - for (String docReferenced : doc.getReferencedDocuments()) { - schema.inheritByDocument(docReferenced); - } - } - for (ParsedDocument doc : parsedDocs.values()) { - for (String inherit : doc.getInherited()) { - var parentDoc = parsedDocs.get(inherit); - if (parentDoc == null) { - throw new IllegalArgumentException("document " + doc.name() + " inherits from unavailable document " + inherit); - } - doc.resolveInherit(inherit, parentDoc); - } - for (String docRefName : doc.getReferencedDocuments()) { - var refDoc = parsedDocs.get(docRefName); - if (refDoc == null) { - throw new IllegalArgumentException("document " + doc.name() + " references unavailable document " + docRefName); - } - doc.resolveReferenced(refDoc); - } - } - for (ParsedSchema schema : parsedSchemas.values()) { - for (String docName : schema.getInheritedByDocument()) { - var parent = schemaForDocs.get(docName); - assert(parent.hasDocument()); - assert(parent.getDocument().name().equals(docName)); - schema.resolveInheritByDocument(docName, parent); - } - } - } - - private void inheritanceCycleCheck(ParsedDocument document, List seen) { - String name = document.name(); - if (seen.contains(name)) { - seen.add(name); - throw new IllegalArgumentException("Inheritance/reference cycle for documents: " + - String.join(" -> ", seen)); - } - seen.add(name); - for (ParsedDocument parent : document.getAllResolvedParents()) { - inheritanceCycleCheck(parent, seen); - } - seen.remove(name); - } - - private void checkDocumentCycles() { - List seen = new ArrayList<>(); - for (ParsedDocument doc : parsedDocs.values()) { - inheritanceCycleCheck(doc, seen); - } - } - - public void resolveInheritance() { - resolveSchemaInheritance(); - resolveDocumentInheritance(); - checkDocumentCycles(); - checkSchemaCycles(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java deleted file mode 100644 index 8dd69085efb..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.yolean.Exceptions; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Class wrapping parsing of schema files and holding a collection of - * schemas in the intermediate format. - * - * @author arnej27959 - **/ -public class IntermediateCollection { - - private final DeployLogger deployLogger; - private final ModelContext.Properties modelProperties; - - private Map parsedSchemas = new LinkedHashMap<>(); - - IntermediateCollection() { - this.deployLogger = new BaseDeployLogger(); - this.modelProperties = new TestProperties(); - } - - public IntermediateCollection(DeployLogger logger, ModelContext.Properties properties) { - this.deployLogger = logger; - this.modelProperties = properties; - } - - public Map getParsedSchemas() { return Collections.unmodifiableMap(parsedSchemas); } - - public ParsedSchema getParsedSchema(String name) { return parsedSchemas.get(name); } - - public ParsedSchema addSchemaFromString(String input) throws ParseException { - var stream = new SimpleCharStream(input); - var parser = new SchemaParser(stream, deployLogger, modelProperties); - try { - var schema = parser.schema(); - if (parsedSchemas.containsKey(schema.name())) { - throw new IllegalArgumentException("Duplicate schemas named: " + schema.name()); - } - parsedSchemas.put(schema.name(), schema); - return schema; - } catch (TokenMgrException e) { - throw new ParseException("Unknown symbol: " + e.getMessage()); - } catch (ParseException pe) { - throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); - } - } - - private String addSchemaFromStringWithFileName(String input, String fileName) throws ParseException { - var parsed = addSchemaFromString(input); - String nameFromFile = baseName(fileName); - if (! parsed.name().equals(nameFromFile)) { - throw new IllegalArgumentException("The file containing schema '" - + parsed.name() + "' must be named '" - + parsed.name() + ApplicationPackage.SD_NAME_SUFFIX - + "', was '" + stripDirs(fileName) + "'"); - } - return parsed.name(); - } - - private String baseName(String filename) { - int pos = filename.lastIndexOf('/'); - if (pos != -1) { - filename = filename.substring(pos + 1); - } - pos = filename.lastIndexOf('.'); - if (pos != -1) { - filename = filename.substring(0, pos); - } - return filename; - } - - private String stripDirs(String filename) { - int pos = filename.lastIndexOf('/'); - if (pos != -1) { - return filename.substring(pos + 1); - } - return filename; - } - - /** - * parse a schema from the given reader and add result to collection - **/ - public String addSchemaFromReader(NamedReader reader) throws ParseException { - try { - var nameParsed = addSchemaFromStringWithFileName(IOUtils.readAll(reader.getReader()), reader.getName()); - reader.close(); - return nameParsed; - } catch (ParseException ex) { - throw new ParseException("Failed parsing schema from " + reader.getName() + ": " + ex.getMessage()); - } catch (java.io.IOException ex) { - throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); - } - } - - /** for unit tests */ - public String addSchemaFromFile(String fileName) throws ParseException { - try { - // return addSchemaFromStringWithFileName(IOUtils.readFile(new File(fileName)), fileName); - var parsed = addSchemaFromString(IOUtils.readFile(new File(fileName))); - return parsed.name(); - } catch (ParseException ex) { - throw new ParseException("Failed parsing schema from " + fileName + ": " + ex.getMessage()); - } catch (java.io.IOException ex) { - throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); - } - } - - /** - * parse a rank profile from the given reader and add to the schema identified by name. - * note: the named schema must have been parsed already. - **/ - public void addRankProfileFile(String schemaName, NamedReader reader) throws ParseException { - try { - ParsedSchema schema = parsedSchemas.get(schemaName); - if (schema == null) { - throw new IllegalArgumentException("No schema named: " + schemaName); - } - var stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); - var parser = new SchemaParser(stream, deployLogger, modelProperties); - try { - parser.rankProfile(schema); - } catch (ParseException pe) { - throw new ParseException("Failed parsing rank-profile from " + reader.getName() + ": " + - stream.formatException(Exceptions.toMessageString(pe))); - } - } catch (java.io.IOException ex) { - throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); - } - } - - // for unit test - void addRankProfileFile(String schemaName, String fileName) throws ParseException { - try { - var reader = IOUtils.createReader(fileName, "UTF-8"); - addRankProfileFile(schemaName, new NamedReader(fileName, reader)); - } catch (java.io.IOException ex) { - throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); - } - } - - void resolveInternalConnections() { - var resolver = new InheritanceResolver(parsedSchemas); - resolver.resolveInheritance(); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java deleted file mode 100644 index 3baac504135..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a - * "annotation" block, using simple data structures as far as - * possible. Do not put advanced logic here! - * @author arnej27959 - **/ -class ParsedAnnotation extends ParsedBlock { - - private ParsedStruct wrappedStruct = null; - private final List inherited = new ArrayList<>(); - private final List resolvedInherits = new ArrayList<>(); - private ParsedDocument ownedBy = null; - - ParsedAnnotation(String name) { - super(name, "annotation"); - } - - public List getInherited() { return List.copyOf(inherited); } - public List getResolvedInherits() { - assert(inherited.size() == resolvedInherits.size()); - return List.copyOf(resolvedInherits); - } - - - public Optional getStruct() { return Optional.ofNullable(wrappedStruct); } - public ParsedDocument getOwnerDoc() { return ownedBy; } - public String getOwnerName() { return ownedBy.name(); } - - public ParsedStruct ensureStruct() { - if (wrappedStruct == null) { - wrappedStruct = new ParsedStruct("annotation." + name()); - wrappedStruct.tagOwner(ownedBy); - } - return wrappedStruct; - } - void setStruct(ParsedStruct struct) { this.wrappedStruct = struct; } - - void inherit(String other) { inherited.add(other); } - - void tagOwner(ParsedDocument owner) { - verifyThat(ownedBy == null, "already owned by", ownedBy); - this.ownedBy = owner; - getStruct().ifPresent(s -> s.tagOwner(owner)); - } - - void resolveInherit(String name, ParsedAnnotation parsed) { - verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); - verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); - resolvedInherits.add(parsed); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java deleted file mode 100644 index b48bad89114..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a - * "attribute" block, using simple data structures as far as - * possible. Do not put advanced logic here! - * @author arnej27959 - **/ -class ParsedAttribute extends ParsedBlock { - - private boolean enableBitVectors = false; - private boolean enableOnlyBitVector = false; - private boolean enableFastAccess = false; - private boolean enableFastRank = false; - private boolean enableFastSearch = false; - private boolean enableHuge = false; - private boolean enableMutable = false; - private boolean enablePaged = false; - private final Map aliases = new LinkedHashMap<>(); - private ParsedSorting sortSettings = null; - private String distanceMetric = null; - - ParsedAttribute(String name) { - super(name, "attribute"); - } - - List getAliases() { return List.copyOf(aliases.keySet()); } - String lookupAliasedFrom(String alias) { return aliases.get(alias); } - Optional getDistanceMetric() { return Optional.ofNullable(distanceMetric); } - boolean getEnableBitVectors() { return this.enableBitVectors; } - boolean getEnableOnlyBitVector() { return this.enableOnlyBitVector; } - boolean getFastAccess() { return this.enableFastAccess; } - boolean getFastRank() { return this.enableFastRank; } - boolean getFastSearch() { return this.enableFastSearch; } - boolean getHuge() { return this.enableHuge; } - boolean getMutable() { return this.enableMutable; } - boolean getPaged() { return this.enablePaged; } - Optional getSorting() { return Optional.ofNullable(sortSettings); } - - void addAlias(String from, String to) { - verifyThat(! aliases.containsKey(to), "already has alias", to); - aliases.put(to, from); - } - - void setDistanceMetric(String value) { - verifyThat(distanceMetric == null, "already has distance-metric", distanceMetric); - this.distanceMetric = value; - } - - ParsedSorting sortInfo() { - if (sortSettings == null) sortSettings = new ParsedSorting(name(), "attribute.sorting"); - return this.sortSettings; - } - - void setEnableBitVectors(boolean value) { this.enableBitVectors = value; } - void setEnableOnlyBitVector(boolean value) { this.enableOnlyBitVector = value; } - void setFastAccess(boolean value) { this.enableFastAccess = true; } - void setFastRank(boolean value) { this.enableFastRank = true; } - void setFastSearch(boolean value) { this.enableFastSearch = true; } - void setHuge(boolean value) { this.enableHuge = true; } - void setMutable(boolean value) { this.enableMutable = true; } - void setPaged(boolean value) { this.enablePaged = true; } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java deleted file mode 100644 index 151da352f2f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -/** - * Common methods for various Parsed* classes. - * @author arnej27959 - **/ -public class ParsedBlock { - private final String name; - private final String blockType; - - public ParsedBlock(String name, String blockType) { - this.name = name; - this.blockType = blockType; - } - - public final String name() { return name; } - public final String blockType() { return blockType; } - - protected void verifyThat(boolean check, String msg, Object ... msgDetails) { - if (check) return; - var buf = new StringBuilder(); - buf.append(blockType).append(" '").append(name).append("' error: "); - buf.append(msg); - for (Object detail : msgDetails) { - buf.append(" "); - buf.append(detail.toString()); - } - throw new IllegalArgumentException(buf.toString()); - } - - public String toString() { - return blockType + " '" + name + "'"; - } -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java deleted file mode 100644 index ea138808289..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a - * "document" block in a schema (.sd) file, using simple data - * structures as far as possible. Do not put advanced logic here! - * @author arnej27959 - **/ -public class ParsedDocument extends ParsedBlock { - private final List inherited = new ArrayList<>(); - private final Map resolvedInherits = new LinkedHashMap(); - private final Map resolvedReferences = new LinkedHashMap(); - private final Map docFields = new LinkedHashMap<>(); - private final Map docStructs = new LinkedHashMap<>(); - private final Map docAnnotations = new LinkedHashMap<>(); - - public ParsedDocument(String name) { - super(name, "document"); - } - - List getInherited() { return List.copyOf(inherited); } - List getAnnotations() { return List.copyOf(docAnnotations.values()); } - List getResolvedInherits() { - assert(inherited.size() == resolvedInherits.size()); - return List.copyOf(resolvedInherits.values()); - } - List getResolvedReferences() { - return List.copyOf(resolvedReferences.values()); - } - List getAllResolvedParents() { - List all = new ArrayList<>(); - all.addAll(getResolvedInherits()); - all.addAll(getResolvedReferences()); - return all; - } - List getFields() { return List.copyOf(docFields.values()); } - List getStructs() { return List.copyOf(docStructs.values()); } - ParsedStruct getStruct(String name) { return docStructs.get(name); } - ParsedAnnotation getAnnotation(String name) { return docAnnotations.get(name); } - - List getReferencedDocuments() { - var result = new ArrayList(); - for (var field : docFields.values()) { - var type = field.getType(); - if (type.getVariant() == ParsedType.Variant.DOC_REFERENCE) { - var docType = type.getReferencedDocumentType(); - assert(docType.getVariant() == ParsedType.Variant.DOCUMENT); - result.add(docType.name()); - } - } - return result; - } - - void inherit(String other) { inherited.add(other); } - - void addField(ParsedField field) { - String fieldName = field.name().toLowerCase(); - verifyThat(! docFields.containsKey(fieldName), - "Duplicate (case insensitively) " + field + " in document type '" + this.name() + "'"); - docFields.put(fieldName, field); - } - - void addStruct(ParsedStruct struct) { - String sName = struct.name(); - verifyThat(! docStructs.containsKey(sName), "already has struct", sName); - docStructs.put(sName, struct); - struct.tagOwner(this); - } - - void addAnnotation(ParsedAnnotation annotation) { - String annName = annotation.name(); - verifyThat(! docAnnotations.containsKey(annName), "already has annotation", annName); - docAnnotations.put(annName, annotation); - annotation.tagOwner(this); - } - - void resolveInherit(String name, ParsedDocument parsed) { - verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); - verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); - verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); - resolvedInherits.put(name, parsed); - } - - void resolveReferenced(ParsedDocument parsed) { - var old = resolvedReferences.put(parsed.name(), parsed); - assert(old == null || old == parsed); - } - - ParsedStruct findParsedStruct(String name) { - ParsedStruct found = getStruct(name); - if (found != null) return found; - for (var parent : getAllResolvedParents()) { - var fromParent = parent.findParsedStruct(name); - if (fromParent == null) continue; - if (fromParent == found) continue; - if (found == null) { - found = fromParent; - } else { - throw new IllegalArgumentException("conflicting values for struct " + name + " in " +this); - } - } - return found; - } - - ParsedAnnotation findParsedAnnotation(String name) { - ParsedAnnotation found = docAnnotations.get(name); - if (found != null) return found; - for (var parent : getResolvedInherits()) { - var fromParent = parent.findParsedAnnotation(name); - if (fromParent == null) continue; - if (fromParent == found) continue; - if (found == null) { - found = fromParent; - } else { - throw new IllegalArgumentException("conflicting values for annotation " + name + " in " +this); - } - } - return found; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java deleted file mode 100644 index 25adc6f134f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java +++ /dev/null @@ -1,49 +0,0 @@ - -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * This class holds the extracted information after parsing a - * "document-summary" block, using simple data structures as far as - * possible. Do not put advanced logic here! - * @author arnej27959 - **/ -class ParsedDocumentSummary extends ParsedBlock { - - private boolean omitSummaryFeatures; - private boolean fromDisk; - private final List inherited = new ArrayList<>(); - private final Map fields = new LinkedHashMap<>(); - - ParsedDocumentSummary(String name) { - super(name, "document-summary"); - } - - boolean getOmitSummaryFeatures() { return omitSummaryFeatures; } - boolean getFromDisk() { return fromDisk; } - List getSummaryFields() { return List.copyOf(fields.values()); } - List getInherited() { return List.copyOf(inherited); } - - ParsedSummaryField addField(ParsedSummaryField field) { - String fieldName = field.name(); - // TODO disallow this on Vespa 8 - // verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); - return fields.put(fieldName, field); - } - - void setFromDisk(boolean value) { - this.fromDisk = value; - } - - void setOmitSummaryFeatures(boolean value) { - this.omitSummaryFeatures = value; - } - - void inherit(String other) { - inherited.add(other); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java deleted file mode 100644 index fcbb62b1229..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.document.Stemming; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a "field" - * block, using simple data structures as far as possible. Do not put - * advanced logic here! - * @author arnej27959 - **/ -class ParsedField extends ParsedBlock { - - private ParsedType type; - private boolean hasBolding = false; - private boolean isFilter = false; - private int overrideId = 0; - private boolean isLiteral = false; - private boolean isNormal = false; - private Integer weight; - private String normalizing = null; - private final ParsedMatchSettings matchInfo = new ParsedMatchSettings(); - private Stemming stemming = null; - private ParsedIndexingOp indexingOp = null; - private ParsedSorting sortSettings = null; - private final Map attributes = new LinkedHashMap<>(); - private final Map fieldIndexes = new LinkedHashMap<>(); - private final Map aliases = new LinkedHashMap<>(); - private final Map rankTypes = new LinkedHashMap<>(); - private final Map structFields = new LinkedHashMap<>(); - private final Map summaryFields = new LinkedHashMap<>(); - private final List dictionaryOptions = new ArrayList<>(); - private final List queryCommands = new ArrayList<>(); - - ParsedField(String name, ParsedType type) { - super(name, "field"); - this.type = type; - } - - ParsedType getType() { return this.type; } - boolean hasBolding() { return this.hasBolding; } - boolean hasFilter() { return this.isFilter; } - boolean hasLiteral() { return this.isLiteral; } - boolean hasNormal() { return this.isNormal; } - boolean hasIdOverride() { return overrideId != 0; } - int idOverride() { return overrideId; } - List getDictionaryOptions() { return List.copyOf(dictionaryOptions); } - List getAttributes() { return List.copyOf(attributes.values()); } - List getIndexes() { return List.copyOf(fieldIndexes.values()); } - List getSummaryFields() { return List.copyOf(summaryFields.values()); } - List getStructFields() { return List.copyOf(structFields.values()); } - List getAliases() { return List.copyOf(aliases.keySet()); } - List getQueryCommands() { return List.copyOf(queryCommands); } - String lookupAliasedFrom(String alias) { return aliases.get(alias); } - ParsedMatchSettings matchSettings() { return this.matchInfo; } - Optional getWeight() { return Optional.ofNullable(weight); } - Optional getStemming() { return Optional.ofNullable(stemming); } - Optional getNormalizing() { return Optional.ofNullable(normalizing); } - Optional getIndexing() { return Optional.ofNullable(indexingOp); } - Optional getSorting() { return Optional.ofNullable(sortSettings); } - Map getRankTypes() { return Collections.unmodifiableMap(rankTypes); } - - /** get an existing summary field for modification, or create it */ - ParsedSummaryField summaryFieldFor(String name) { - if (summaryFields.containsKey(name)) { - return summaryFields.get(name); - } - var sf = new ParsedSummaryField(name, getType()); - summaryFields.put(name, sf); - return sf; - } - - /** get an existing summary field for modification, or create it */ - ParsedSummaryField summaryFieldFor(String name, ParsedType type) { - if (summaryFields.containsKey(name)) { - var sf = summaryFields.get(name); - if (sf.getType() == null) { - sf.setType(type); - } else { - // TODO check that types are properly equal here - String oldName = sf.getType().name(); - String newName = type.name(); - verifyThat(newName.equals(oldName), "type mismatch for summary field", name, ":", oldName, "/", newName); - } - return sf; - } - var sf = new ParsedSummaryField(name, type); - summaryFields.put(name, sf); - return sf; - } - - void addAlias(String from, String to) { - verifyThat(! aliases.containsKey(to), "already has alias", to); - aliases.put(to, from); - } - - void addIndex(ParsedIndex index) { - String idxName = index.name(); - verifyThat(! fieldIndexes.containsKey(idxName), "already has index", idxName); - fieldIndexes.put(idxName, index); - } - - void addRankType(String index, String rankType) { - rankTypes.put(index, rankType); - } - - void dictionary(DictionaryOption option) { - dictionaryOptions.add(option); - } - - void setBolding(boolean value) { this.hasBolding = value; } - void setFilter(boolean value) { this.isFilter = value; } - void setId(int id) { this.overrideId = id; } - void setLiteral(boolean value) { this.isLiteral = value; } - void setNormal(boolean value) { this.isNormal = value; } - void setNormalizing(String value) { this.normalizing = value; } - void setStemming(Stemming stemming) { this.stemming = stemming; } - void setWeight(int weight) { this.weight = weight; } - - ParsedAttribute attributeFor(String attrName) { - return attributes.computeIfAbsent(attrName, n -> new ParsedAttribute(n)); - } - - void setIndexingOperation(ParsedIndexingOp idxOp) { - verifyThat(indexingOp == null, "already has indexing"); - indexingOp = idxOp; - } - - ParsedSorting sortInfo() { - if (sortSettings == null) sortSettings = new ParsedSorting(name(), "field.sorting"); - return this.sortSettings; - } - - void addQueryCommand(String command) { - queryCommands.add(command); - } - - void addStructField(ParsedField structField) { - String fieldName = structField.name(); - verifyThat(! structFields.containsKey(fieldName), "already has struct-field", fieldName); - structFields.put(fieldName, structField); - } - - void addSummaryField(ParsedSummaryField summaryField) { - String fieldName = summaryField.name(); - verifyThat(! summaryFields.containsKey(fieldName), "already has summary field", fieldName); - if (summaryField.getType() == null) { - summaryField.setType(getType()); - } - summaryFields.put(fieldName, summaryField); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java deleted file mode 100644 index 9103ed46631..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a "fieldset" - * block, using simple data structures as far as possible. Do not put - * advanced logic here! - * @author arnej27959 - **/ -class ParsedFieldSet extends ParsedBlock { - - private final List fields = new ArrayList<>(); - private final List queryCommands = new ArrayList<>(); - private ParsedMatchSettings matchInfo = null; - - ParsedFieldSet(String name) { - super(name, "fieldset"); - } - - ParsedMatchSettings matchSettings() { - if (matchInfo == null) matchInfo = new ParsedMatchSettings(); - return this.matchInfo; - } - - List getQueryCommands() { return List.copyOf(queryCommands); } - List getFieldNames() { return List.copyOf(fields); } - Optional getMatchSettings() { - return Optional.ofNullable(this.matchInfo); - } - - void addField(String field) { fields.add(field); } - void addQueryCommand(String command) { queryCommands.add(command); } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java deleted file mode 100644 index a3504cd2bf7..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.Stemming; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing an "index" - * block, using simple data structures as far as possible. Do not put - * advanced logic here! - * @author arnej27959 - **/ -class ParsedIndex extends ParsedBlock { - - private Boolean enableBm25 = null; - private Boolean isPrefix = null; - private HnswIndexParams hnswParams = null; - private final List aliases = new ArrayList<>(); - private Stemming stemming = null; - private Integer arity = null; - private Long lowerBound = null; - private Long upperBound = null; - private Double densePLT = null; - - ParsedIndex(String name) { - super(name, "index"); - } - - Optional getEnableBm25() { return Optional.ofNullable(this.enableBm25); } - Optional getPrefix() { return Optional.ofNullable(this.isPrefix); } - Optional getHnswIndexParams() { return Optional.ofNullable(this.hnswParams); } - List getAliases() { return List.copyOf(aliases); } - boolean hasStemming() { return stemming != null; } - Optional getStemming() { return Optional.ofNullable(stemming); } - Optional getArity() { return Optional.ofNullable(this.arity); } - Optional getLowerBound() { return Optional.ofNullable(this.lowerBound); } - Optional getUpperBound() { return Optional.ofNullable(this.upperBound); } - Optional getDensePostingListThreshold() { return Optional.ofNullable(this.densePLT); } - - void addAlias(String alias) { - aliases.add(alias); - } - - void setArity(int arity) { - this.arity = arity; - } - - void setDensePostingListThreshold(double threshold) { - this.densePLT = threshold; - } - - void setEnableBm25(boolean value) { - this.enableBm25 = value; - } - - void setHnswIndexParams(HnswIndexParams params) { - this.hnswParams = params; - } - - void setLowerBound(long value) { - this.lowerBound = value; - } - - void setPrefix(boolean value) { - this.isPrefix = value; - } - - void setStemming(Stemming stemming) { - this.stemming = stemming; - } - - void setUpperBound(long value) { - this.upperBound = value; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndexingOp.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndexingOp.java deleted file mode 100644 index 4178de904de..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndexingOp.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; -import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; - -/** - * This class wraps an indexing script expression, with some helper - * methods for extracting information from it - * @author arnej27959 - **/ -class ParsedIndexingOp { - - private final ScriptExpression script; - - ParsedIndexingOp(ScriptExpression script) { - this.script = script; - } - - ScriptExpression script() { return this.script; } - - public boolean doesAttributing() { return containsExpression(AttributeExpression.class); } - public boolean doesIndexing() { return containsExpression(IndexExpression.class); } - public boolean doesLowerCasing() { return containsExpression(LowerCaseExpression.class); } - public boolean doesSummarying() { return containsExpression(SummaryExpression.class); } - - private boolean containsExpression(Class searchFor) { - var searcher = new ExpressionSearcher<>(searchFor); - var expr = searcher.searchIn(script); - return (expr != null); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java deleted file mode 100644 index 9b51521ad2b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.MatchAlgorithm; - -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a "match" - * block, using simple data structures as far as possible. Do not put - * advanced logic here! - * @author arnej27959 - **/ -public class ParsedMatchSettings { - - private MatchType matchType = null; - private Case matchCase = null; - private MatchAlgorithm matchAlgorithm = null; - private String exactTerminator = null; - private Integer gramSize = null; - private Integer maxLength = null; - - Optional getMatchType() { return Optional.ofNullable(matchType); } - Optional getMatchCase() { return Optional.ofNullable(matchCase); } - Optional getMatchAlgorithm() { return Optional.ofNullable(matchAlgorithm); } - Optional getExactTerminator() { return Optional.ofNullable(exactTerminator); } - Optional getGramSize() { return Optional.ofNullable(gramSize); } - Optional getMaxLength() { return Optional.ofNullable(maxLength); } - - // TODO - consider allowing each set only once: - void setType(MatchType value) { this.matchType = value; } - void setCase(Case value) { this.matchCase = value; } - void setAlgorithm(MatchAlgorithm value) { this.matchAlgorithm = value; } - void setExactTerminator(String value) { this.exactTerminator = value; } - void setGramSize(int value) { this.gramSize = value; } - void setMaxLength(int value) { this.maxLength = value; } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java deleted file mode 100644 index a8ee9a24e69..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.List; - -/** - * This class holds the extracted information after parsing a - * "function" block in a rank-profile, using simple data structures as - * far as possible. Do not put advanced logic here! - * @author arnej27959 - **/ -class ParsedRankFunction extends ParsedBlock { - - private boolean inline; - private String expression; - private final List parameters = new ArrayList<>(); - - ParsedRankFunction(String name) { - super(name, "function"); - } - - boolean getInline() { return this.inline; } - String getExpression() { return this.expression; } - List getParameters() { return List.copyOf(parameters); } - - void addParameter(String param) { - verifyThat(! parameters.contains(param), "cannot have parameter", param, "twice"); - parameters.add(param); - } - - void setInline(boolean value) { - this.inline = value; - } - - void setExpression(String value) { - this.expression = value; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java deleted file mode 100644 index 8f0f92c4027..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; -import com.yahoo.searchdefinition.RankProfile.MutateOperation; -import com.yahoo.searchlib.rankingexpression.FeatureList; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.tensor.TensorType; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a - * rank-profile block in a schema (.sd) file, using simple data - * structures as far as possible. Do not put advanced logic here! - * - * @author arnej27959 - */ -class ParsedRankProfile extends ParsedBlock { - - private boolean ignoreDefaultRankFeatures = false; - private Double rankScoreDropLimit = null; - private Double termwiseLimit = null; - private Double postFilterThreshold = null; - private Double approximateThreshold = null; - private final List matchFeatures = new ArrayList<>(); - private final List rankFeatures = new ArrayList<>(); - private final List summaryFeatures = new ArrayList<>(); - private Integer keepRankCount = null; - private Integer minHitsPerThread = null; - private Integer numSearchPartitions = null; - private Integer numThreadsPerSearch = null; - private Integer reRankCount = null; - private MatchPhaseSettings matchPhaseSettings = null; - private String firstPhaseExpression = null; - private String inheritedSummaryFeatures = null; - private String inheritedMatchFeatures = null; - private String secondPhaseExpression = null; - private Boolean strict = null; - private final List mutateOperations = new ArrayList<>(); - private final List inherited = new ArrayList<>(); - private final Map fieldsRankFilter = new LinkedHashMap<>(); - private final Map fieldsRankWeight = new LinkedHashMap<>(); - private final Map functions = new LinkedHashMap<>(); - private final Map fieldsRankType = new LinkedHashMap<>(); - private final Map> rankProperties = new LinkedHashMap<>(); - private final Map constants = new LinkedHashMap<>(); - private final Map inputs = new LinkedHashMap<>(); - private final List onnxModels = new ArrayList<>(); - - ParsedRankProfile(String name) { - super(name, "rank-profile"); - } - - boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } - Optional getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } - Optional getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } - Optional getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } - Optional getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } - List getMatchFeatures() { return List.copyOf(this.matchFeatures); } - List getRankFeatures() { return List.copyOf(this.rankFeatures); } - List getSummaryFeatures() { return List.copyOf(this.summaryFeatures); } - Optional getKeepRankCount() { return Optional.ofNullable(this.keepRankCount); } - Optional getMinHitsPerThread() { return Optional.ofNullable(this.minHitsPerThread); } - Optional getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } - Optional getNumThreadsPerSearch() { return Optional.ofNullable(this.numThreadsPerSearch); } - Optional getReRankCount() { return Optional.ofNullable(this.reRankCount); } - Optional getMatchPhaseSettings() { return Optional.ofNullable(this.matchPhaseSettings); } - Optional getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } - Optional getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } - List getFunctions() { return List.copyOf(functions.values()); } - List getMutateOperations() { return List.copyOf(mutateOperations); } - List getInherited() { return List.copyOf(inherited); } - - Map getFieldsWithRankFilter() { return Collections.unmodifiableMap(fieldsRankFilter); } - Map getFieldsWithRankWeight() { return Collections.unmodifiableMap(fieldsRankWeight); } - Map getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } - Map> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } - Map getConstants() { return Collections.unmodifiableMap(constants); } - Map getInputs() { return Collections.unmodifiableMap(inputs); } - List getOnnxModels() { return List.copyOf(onnxModels); } - - Optional getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } - Optional getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } - Optional isStrict() { return Optional.ofNullable(this.strict); } - - void addSummaryFeatures(FeatureList features) { this.summaryFeatures.add(features); } - void addMatchFeatures(FeatureList features) { this.matchFeatures.add(features); } - void addRankFeatures(FeatureList features) { this.rankFeatures.add(features); } - - void inherit(String other) { inherited.add(other); } - - void setInheritedSummaryFeatures(String other) { - verifyThat(inheritedSummaryFeatures == null, "already inherits summary-features"); - this.inheritedSummaryFeatures = other; - } - - void add(RankProfile.Constant constant) { - verifyThat(! constants.containsKey(constant.name()), "already has constant", constant.name()); - constants.put(constant.name(), constant); - } - - void addInput(Reference name, RankProfile.Input input) { - verifyThat(! inputs.containsKey(name), "already has input", name); - inputs.put(name, input); - } - - void add(OnnxModel model) { - onnxModels.add(model); - } - - void addFieldRankFilter(String field, boolean filter) { - fieldsRankFilter.put(field, filter); - } - - void addFieldRankType(String field, String type) { - verifyThat(! fieldsRankType.containsKey(field), "already has rank type for field", field); - fieldsRankType.put(field, type); - } - - void addFieldRankWeight(String field, int weight) { - verifyThat(! fieldsRankType.containsKey(field), "already has weight for field", field); - fieldsRankWeight.put(field, weight); - } - - ParsedRankFunction addOrReplaceFunction(ParsedRankFunction func) { - // allowed with warning - // verifyThat(! functions.containsKey(func.name()), "already has function", func.name()); - return functions.put(func.name(), func); - } - - void addMutateOperation(MutateOperation.Phase phase, String attrName, String operation) { - mutateOperations.add(new MutateOperation(phase, attrName, operation)); - } - - void addRankProperty(String key, String value) { - List values = rankProperties.computeIfAbsent(key, k -> new ArrayList()); - values.add(value); - } - - void setFirstPhaseRanking(String expression) { - verifyThat(firstPhaseExpression == null, "already has first-phase expression"); - this.firstPhaseExpression = expression; - } - - void setIgnoreDefaultRankFeatures(boolean value) { - this.ignoreDefaultRankFeatures = value; - } - - void setInheritedMatchFeatures(String other) { - this.inheritedMatchFeatures = other; - } - - void setKeepRankCount(int count) { - verifyThat(keepRankCount == null, "already has rerank-count"); - this.keepRankCount = count; - } - - void setMatchPhaseSettings(MatchPhaseSettings settings) { - verifyThat(matchPhaseSettings == null, "already has match-phase"); - this.matchPhaseSettings = settings; - } - - void setMinHitsPerThread(int minHits) { - verifyThat(minHitsPerThread == null, "already has min-hits-per-thread"); - this.minHitsPerThread = minHits; - } - - void setNumSearchPartitions(int numParts) { - verifyThat(numSearchPartitions == null, "already has num-search-partitions"); - this.numSearchPartitions = numParts; - } - - void setNumThreadsPerSearch(int threads) { - verifyThat(numThreadsPerSearch == null, "already has num-threads-per-search"); - this.numThreadsPerSearch = threads; - } - - void setRankScoreDropLimit(double limit) { - verifyThat(rankScoreDropLimit == null, "already has rank-score-drop-limit"); - this.rankScoreDropLimit = limit; - } - - void setRerankCount(int count) { - verifyThat(reRankCount == null, "already has rerank-count"); - this.reRankCount = count; - } - - void setSecondPhaseRanking(String expression) { - verifyThat(secondPhaseExpression == null, "already has second-phase expression"); - this.secondPhaseExpression = expression; - } - - void setStrict(boolean strict) { - verifyThat(this.strict == null, "already has strict"); - this.strict = strict; - } - - void setTermwiseLimit(double limit) { - verifyThat(termwiseLimit == null, "already has termwise-limit"); - this.termwiseLimit = limit; - } - - void setPostFilterThreshold(double threshold) { - verifyThat(postFilterThreshold == null, "already has post-filter-threshold"); - this.postFilterThreshold = threshold; - } - - void setApproximateThreshold(double threshold) { - verifyThat(approximateThreshold == null, "already has approximate-threshold"); - this.approximateThreshold = threshold; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java deleted file mode 100644 index 4c102594479..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchlib.rankingexpression.Reference; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * This class holds the extracted information after parsing - * one schema (.sd) file, using simple data structures - * as far as possible. - * - * Do not put complicated logic here! - * - * @author arnej27959 - */ -public class ParsedSchema extends ParsedBlock { - - public static class ImportedField { - public final String asFieldName; - public final String refFieldName; - public final String foreignFieldName; - public ImportedField(String asField, String refField, String foreignField) { - this.asFieldName = asField; - this.refFieldName = refField; - this.foreignFieldName = foreignField; - } - } - - private boolean documentWithoutSchema = false; - private Boolean rawAsBase64 = null; - private ParsedDocument myDocument = null; - private Stemming defaultStemming = null; - private final List importedFields = new ArrayList<>(); - private final List onnxModels = new ArrayList<>(); - private final Map constants = new LinkedHashMap<>(); - private final List inherited = new ArrayList<>(); - private final List inheritedByDocument = new ArrayList<>(); - private final Map resolvedInherits = new LinkedHashMap<>(); - private final Map allResolvedInherits = new LinkedHashMap<>(); - private final Map extraAnnotations = new LinkedHashMap<>(); - private final Map docSums = new LinkedHashMap<>(); - private final Map extraFields = new LinkedHashMap<>(); - private final Map fieldSets = new LinkedHashMap<>(); - private final Map extraIndexes = new LinkedHashMap<>(); - private final Map rankProfiles = new LinkedHashMap<>(); - private final Map extraStructs = new LinkedHashMap<>(); - - public ParsedSchema(String name) { - super(name, "schema"); - } - - boolean getDocumentWithoutSchema() { return documentWithoutSchema; } - Optional getRawAsBase64() { return Optional.ofNullable(rawAsBase64); } - boolean hasDocument() { return myDocument != null; } - ParsedDocument getDocument() { return myDocument; } - boolean hasStemming() { return defaultStemming != null; } - Stemming getStemming() { return defaultStemming; } - List getImportedFields() { return List.copyOf(importedFields); } - List getOnnxModels() { return List.copyOf(onnxModels); } - List getAnnotations() { return List.copyOf(extraAnnotations.values()); } - List getDocumentSummaries() { return List.copyOf(docSums.values()); } - List getFields() { return List.copyOf(extraFields.values()); } - List getFieldSets() { return List.copyOf(fieldSets.values()); } - List getIndexes() { return List.copyOf(extraIndexes.values()); } - List getStructs() { return List.copyOf(extraStructs.values()); } - List getInherited() { return List.copyOf(inherited); } - List getInheritedByDocument() { return List.copyOf(inheritedByDocument); } - List getRankProfiles() { return List.copyOf(rankProfiles.values()); } - List getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } - List getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); } - List getConstants() { return List.copyOf(constants.values()); } - - void addAnnotation(ParsedAnnotation annotation) { - String annName = annotation.name(); - verifyThat(! extraAnnotations.containsKey(annName), "already has annotation", annName); - extraAnnotations.put(annName, annotation); - } - - void addDocument(ParsedDocument document) { - verifyThat(myDocument == null, - "already has", myDocument, "so cannot add", document); - // TODO - disallow? - // verifyThat(name().equals(document.name()), - // "schema " + name() + " can only contain document named " + name() + ", was: "+ document.name()); - this.myDocument = document; - } - - void setDocumentWithoutSchema() { this.documentWithoutSchema = true; } - - void addDocumentSummary(ParsedDocumentSummary docsum) { - String dsName = docsum.name(); - verifyThat(! docSums.containsKey(dsName), "already has document-summary", dsName); - docSums.put(dsName, docsum); - } - - void addField(ParsedField field) { - String fieldName = field.name(); - verifyThat(! extraFields.containsKey(fieldName), "already has field", fieldName); - extraFields.put(fieldName, field); - } - - void addFieldSet(ParsedFieldSet fieldSet) { - String fsName = fieldSet.name(); - verifyThat(! fieldSets.containsKey(fsName), "already has fieldset", fsName); - fieldSets.put(fsName, fieldSet); - } - - void addImportedField(String asFieldName, String refFieldName, String foregnFieldName) { - importedFields.add(new ImportedField(asFieldName, refFieldName, foregnFieldName)); - } - - void addIndex(ParsedIndex index) { - String idxName = index.name(); - verifyThat(! extraIndexes.containsKey(idxName), "already has index", idxName); - extraIndexes.put(idxName, index); - } - - void add(OnnxModel model) { - onnxModels.add(model); - } - - void addRankProfile(ParsedRankProfile profile) { - String rpName = profile.name(); - verifyThat(! rankProfiles.containsKey(rpName), "already has rank-profile", rpName); - rankProfiles.put(rpName, profile); - } - - void add(RankProfile.Constant constant) { - constants.put(constant.name(), constant); - } - - void addStruct(ParsedStruct struct) { - String sName = struct.name(); - verifyThat(! extraStructs.containsKey(sName), "already has struct", sName); - extraStructs.put(sName, struct); - } - - void enableRawAsBase64(boolean value) { - this.rawAsBase64 = value; - } - - void inherit(String other) { inherited.add(other); } - - void inheritByDocument(String other) { inheritedByDocument.add(other); } - - void setStemming(Stemming value) { - verifyThat((defaultStemming == null) || (defaultStemming == value), - "already has stemming", defaultStemming, "cannot also set", value); - defaultStemming = value; - } - - void resolveInherit(String name, ParsedSchema parsed) { - verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); - verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); - verifyThat(! resolvedInherits.containsKey(name), "double resolveInherit for", name); - resolvedInherits.put(name, parsed); - var old = allResolvedInherits.put("schema " + name, parsed); - verifyThat(old == null || old == parsed, "conflicting resolveInherit for", name); - } - - void resolveInheritByDocument(String name, ParsedSchema parsed) { - verifyThat(inheritedByDocument.contains(name), - "resolveInheritByDocument for non-inherited name", name); - var old = allResolvedInherits.put("document " + name, parsed); - verifyThat(old == null || old == parsed, "conflicting resolveInheritByDocument for", name); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java deleted file mode 100644 index 32c822bf25b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java +++ /dev/null @@ -1,48 +0,0 @@ - -package com.yahoo.searchdefinition.parser; - -import com.yahoo.searchdefinition.document.Sorting.Function; -import com.yahoo.searchdefinition.document.Sorting.Strength; - -import java.util.Optional; - -/** - * This class holds the extracted information after parsing a "sorting" - * block, using simple data structures as far as possible. Do not put - * advanced logic here! - * @author arnej27959 - **/ -class ParsedSorting extends ParsedBlock { - - private boolean ascending = true; - private Function sortFunction = null; - private Strength sortStrength = null; - private String sortLocale = null; - - ParsedSorting(String blockName, String blockType) { - super(blockName, blockType); - } - - boolean getAscending() { return this.ascending; } - boolean getDescending() { return ! this.ascending; } - Optional getFunction() { return Optional.ofNullable(sortFunction); } - Optional getStrength() { return Optional.ofNullable(sortStrength); } - Optional getLocale() { return Optional.ofNullable(sortLocale); } - - void setAscending() { this.ascending = true; } - - void setDescending() { this.ascending = false; } - - void setLocale(String value) { - verifyThat(sortLocale == null, "sorting already has locale", sortLocale); - this.sortLocale = value; - } - void setFunction(Function value) { - verifyThat(sortFunction == null, "sorting already has function", sortFunction); - this.sortFunction = value; - } - void setStrength(Strength value) { - verifyThat(sortStrength == null, "sorting already has strength", sortStrength); - this.sortStrength = value; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java deleted file mode 100644 index 376fd592c9d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * This class holds the extracted information after parsing a "struct" - * block, using simple data structures as far as possible. Do not put - * advanced logic here! - * @author arnej27959 - **/ -public class ParsedStruct extends ParsedBlock { - private final List inherited = new ArrayList<>(); - private final List resolvedInherits = new ArrayList<>(); - private final Map fields = new LinkedHashMap<>(); - private final ParsedType asParsedType; - private ParsedDocument ownedBy = null; - - public ParsedStruct(String name) { - super(name, "struct"); - this.asParsedType = ParsedType.fromName(name); - asParsedType.setVariant(ParsedType.Variant.STRUCT); - } - - List getFields() { return List.copyOf(fields.values()); } - List getInherited() { return List.copyOf(inherited); } - ParsedDocument getOwnerDoc() { return ownedBy; } - String getOwnerName() { return ownedBy.name(); } - List getResolvedInherits() { - assert(inherited.size() == resolvedInherits.size()); - return List.copyOf(resolvedInherits); - } - - void addField(ParsedField field) { - String fieldName = field.name(); - verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); - fields.put(fieldName, field); - } - - void inherit(String other) { - verifyThat(! name().equals(other), "cannot inherit from itself"); - inherited.add(other); - } - - void tagOwner(ParsedDocument document) { - verifyThat(ownedBy == null, "already owned by document "+ownedBy); - this.ownedBy = document; - } - - void resolveInherit(String name, ParsedStruct parsed) { - verifyThat(inherited.contains(name), "resolveInherit for non-inherited name", name); - verifyThat(name.equals(parsed.name()), "resolveInherit name mismatch for", name); - resolvedInherits.add(parsed); - } - -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSummaryField.java deleted file mode 100644 index 40e95045afd..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSummaryField.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import java.util.ArrayList; -import java.util.List; - -/** - * This class holds the extracted information after parsing a summary - * field declaration, either from "field" inside "document-summary" or - * "summary" inside "field". Using simple data structures as far as - * possible. Do not put advanced logic here! - * @author arnej27959 - **/ -class ParsedSummaryField extends ParsedBlock { - - private ParsedType type; - private boolean isDyn = false; - private boolean isMEO = false; - private boolean isFull = false; - private boolean isBold = false; - private final List sources = new ArrayList<>(); - private final List destinations = new ArrayList<>(); - - ParsedSummaryField(String name) { - this(name, null); - } - - ParsedSummaryField(String name, ParsedType type) { - super(name, "summary field"); - this.type = type; - } - - ParsedType getType() { return type; } - List getDestinations() { return List.copyOf(destinations); } - List getSources() { return List.copyOf(sources); } - boolean getBolded() { return isBold; } - boolean getDynamic() { return isDyn; } - boolean getFull() { return isFull; } - boolean getMatchedElementsOnly() { return isMEO; } - - void addDestination(String dst) { destinations.add(dst); } - void addSource(String src) { sources.add(src); } - void setBold(boolean value) { this.isBold = value; } - void setDynamic() { this.isDyn = true; } - void setFull() { this.isFull = true; } - void setMatchedElementsOnly() { this.isMEO = true; } - void setType(ParsedType value) { - verifyThat(type == null, "Cannot change type from ", type, "to", value); - this.type = value; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java deleted file mode 100644 index bcf8d8c9172..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.tensor.TensorType; - -/** - * This class holds the extracted information after parsing a type - * declaration (typically for a field). Since types can be complex, - * struct names (known or unknown), or even document names, this class - * is somewhat complicated. - * @author arnej27959 - **/ -class ParsedType { - public enum Variant { - NONE, - BUILTIN, - POSITION, - TENSOR, - ARRAY, WSET, MAP, - DOC_REFERENCE, - ANN_REFERENCE, - STRUCT, - DOCUMENT, - UNKNOWN - } - - private final String name; - private final ParsedType keyType; - private final ParsedType valType; - private final TensorType tensorType; - private Variant variant; - private boolean createIfNonExistent = false; - private boolean removeIfZero = false; - - public String toString() { - var buf = new StringBuilder(); - buf.append("[type ").append(variant).append("] {"); - switch (variant) { - case NONE: - break; - case BUILTIN: - buf.append(name); - break; - case POSITION: - buf.append(name); - break; - case TENSOR: - buf.append(tensorType.toString()); - break; - case ARRAY: buf - .append(" array<") - .append(valType.toString()) - .append("> "); - break; - case WSET: buf - .append(" weightedset<") - .append(valType.toString()) - .append(">"); - if (createIfNonExistent) buf.append(",createIfNonExistent"); - if (removeIfZero) buf.append(",removeIfZero"); - buf.append(" "); - break; - case MAP: buf - .append(" map<") - .append(keyType.toString()) - .append(",") - .append(valType.toString()) - .append("> "); - break; - case DOC_REFERENCE: buf - .append(" reference<") - .append(valType.toString()) - .append("> "); - break; - case ANN_REFERENCE: buf - .append(" ") - .append(toString()) - .append(" "); - break; - case STRUCT: - case DOCUMENT: - case UNKNOWN: - buf.append(" ").append(name).append(" "); - break; - } - buf.append("}"); - return buf.toString(); - } - - private static Variant guessVariant(String name) { - switch (name) { - case "bool": return Variant.BUILTIN; - case "byte": return Variant.BUILTIN; - case "int": return Variant.BUILTIN; - case "long": return Variant.BUILTIN; - case "string": return Variant.BUILTIN; - case "float": return Variant.BUILTIN; - case "double": return Variant.BUILTIN; - case "uri": return Variant.BUILTIN; - case "predicate": return Variant.BUILTIN; - case "raw": return Variant.BUILTIN; - case "tag": return Variant.BUILTIN; - case "position": return Variant.POSITION; - case "float16": return Variant.BUILTIN; - } - return Variant.UNKNOWN; - } - - public String name() { return name; } - public Variant getVariant() { return variant; } - public ParsedType mapKeyType() { assert(variant == Variant.MAP); return keyType; } - public ParsedType mapValueType() { assert(variant == Variant.MAP); return valType; } - public ParsedType nestedType() { assert(variant == Variant.ARRAY || variant == Variant.WSET); assert(valType != null); return valType; } - public boolean getCreateIfNonExistent() { assert(variant == Variant.WSET); return this.createIfNonExistent; } - public boolean getRemoveIfZero() { assert(variant == Variant.WSET); return this.removeIfZero; } - public ParsedType getReferencedDocumentType() { assert(variant == Variant.DOC_REFERENCE); return valType; } - public TensorType getTensorType() { assert(variant == Variant.TENSOR); return tensorType; } - - public String getNameOfReferencedAnnotation() { - assert(variant == Variant.ANN_REFERENCE); - String prefix = "annotationreference<"; - int fromPos = prefix.length(); - int toPos = name.length() - 1; - return name.substring(fromPos, toPos); - } - - private ParsedType(String name, Variant variant) { - this(name, variant, null, null, null); - } - private ParsedType(String name, Variant variant, ParsedType vt) { - this(name, variant, null, vt, null); - } - private ParsedType(String name, Variant variant, ParsedType kt, ParsedType vt) { - this(name, variant, kt, vt, null); - } - private ParsedType(String name, Variant variant, ParsedType kt, ParsedType vt, TensorType tType) { - this.name = name; - this.variant = variant; - this.keyType = kt; - this.valType = vt; - this.tensorType = tType; - } - - static ParsedType mapType(ParsedType kt, ParsedType vt) { - assert(kt != null); - assert(vt != null); - String name = "map<" + kt.name() + "," + vt.name() + ">"; - return new ParsedType(name, Variant.MAP, kt, vt); - } - static ParsedType arrayOf(ParsedType vt) { - assert(vt != null); - return new ParsedType("array<" + vt.name() + ">", Variant.ARRAY, vt); - } - static ParsedType wsetOf(ParsedType vt) { - assert(vt != null); - if (vt.getVariant() != Variant.BUILTIN) { - throw new IllegalArgumentException("weightedset of complex type '" + vt + "' is not supported"); - } - switch (vt.name()) { - // allowed types: - case "bool": - case "byte": - case "int": - case "long": - case "string": - case "uri": - break; - case "predicate": - case "raw": - case "tag": - throw new IllegalArgumentException("weightedset of complex type '" + vt + "' is not supported"); - case "float16": - case "float": - case "double": - /* TODO Vespa 8: - throw new IllegalArgumentException("weightedset of inexact type '" + vt + "' is not supported"); - */ - break; - default: - throw new IllegalArgumentException("weightedset of unknown type '" + vt + "' is not supported"); - } - return new ParsedType("weightedset<" + vt.name() + ">", Variant.WSET, vt); - } - static ParsedType documentRef(ParsedType docType) { - assert(docType != null); - return new ParsedType("reference<" + docType.name + ">", Variant.DOC_REFERENCE, docType); - } - static ParsedType annotationRef(String name) { - return new ParsedType("annotationreference<" + name + ">", Variant.ANN_REFERENCE); - } - static ParsedType tensorType(TensorType tType) { - assert(tType != null); - return new ParsedType(tType.toString(), Variant.TENSOR, null, null, tType); - } - static ParsedType fromName(String name) { - return new ParsedType(name, guessVariant(name)); - } - static ParsedType documentType(String name) { - return new ParsedType(name, Variant.DOCUMENT); - } - - void setCreateIfNonExistent(boolean value) { - if (variant != Variant.WSET) { - throw new IllegalArgumentException("CreateIfNonExistent only valid for weightedset, not " + variant); - } - this.createIfNonExistent = value; - } - - void setRemoveIfZero(boolean value) { - if (variant != Variant.WSET) { - throw new IllegalArgumentException("RemoveIfZero only valid for weightedset, not " + variant); - } - this.removeIfZero = value; - } - - void setVariant(Variant value) { - if (variant == value) return; // already OK - if (variant != Variant.UNKNOWN) { - throw new IllegalArgumentException("setVariant(" + value + ") only valid for UNKNOWN, not: " + variant); - } - // maybe even more checking would be useful - this.variant = value; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java deleted file mode 100644 index 0b275c6a722..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.javacc.FastCharStream; - -/** - * @author Simon Thoresen Hult - */ -public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream, - com.yahoo.vespa.indexinglanguage.parser.CharStream { - - public SimpleCharStream(String input) { - super(input); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java deleted file mode 100644 index e732bdad19c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -/** - * @author bjorncs - */ -class Utils { - - private Utils() {} - - // Separate class since javacc does not accept Java code using lambdas - static int count(String str, char ch) { - return (int) str.chars().filter(c -> c == ch).count(); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java deleted file mode 100644 index 47ded54dcb6..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableImportedComplexSDField; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.stream.Stream; - -/** - * Adds the attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported - * field as source. - * - * @author bjorncs - */ -public class AddAttributeTransformToSummaryOfImportedFields extends Processor { - - public AddAttributeTransformToSummaryOfImportedFields(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - schema.allImportedFields() - .forEach(field -> setTransform(field)); - } - - private Stream getSummaryFieldsForImportedField(ImmutableSDField importedField) { - return schema.getSummaryFields(importedField).stream(); - } - - private void setTransform(ImmutableSDField field) { - if (field instanceof ImmutableImportedComplexSDField) { - getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); - } else { - getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); - } - } - - private static void setAttributeTransform(SummaryField summaryField) { - if (summaryField.getTransform() == SummaryTransform.NONE) { - summaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - } - - private static void setAttributeCombinerTransform(SummaryField summaryField) { - if (summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER) { - summaryField.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - } else { - summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java deleted file mode 100644 index 0be48d1fd25..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * This processor creates a {@link com.yahoo.searchdefinition.document.SDDocumentType} for each {@link Schema} - * object which holds all the data that search - * associates with a document described in a search definition file. This includes all extra fields, summary fields and - * implicit fields. All non-indexed and non-summary fields are discarded. - */ -public class AddExtraFieldsToDocument extends Processor { - - AddExtraFieldsToDocument(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - SDDocumentType document = schema.getDocument(); - if (document != null) { - for (SDField field : schema.extraFieldList()) { - addSdField(schema, document, field, validate); - } - for (var docsum : schema.getSummaries().values()) { - for (var summaryField : docsum.getSummaryFields().values()) { - switch (summaryField.getTransform()) { - case NONE: - case BOLDED: - case DYNAMICBOLDED: - case DYNAMICTEASER: - case TEXTEXTRACTOR: - addSummaryField(schema, document, summaryField, validate); - break; - default: - // skip: generated from attribute or similar, - // so does not need to be included as an extra - // field in the document type - } - } - } - } - } - - private void addSdField(Schema schema, SDDocumentType document, SDField field, boolean validate) { - if (! field.hasIndex() && field.getAttributes().isEmpty()) { - return; - } - for (Attribute atr : field.getAttributes().values()) { - if (!atr.getName().equals(field.getName())) { - addField(schema, document, new SDField(document, atr.getName(), atr.getDataType()), validate); - } - } - addField(schema, document, field, validate); - } - - private void addSummaryField(Schema schema, SDDocumentType document, SummaryField field, boolean validate) { - Field docField = document.getField(field.getName()); - if (docField == null) { - ImmutableSDField existingField = schema.getField(field.getName()); - if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType()); - newField.setIsExtraField(true); - document.addField(newField); - } else if (!existingField.isImportedField()) { - document.addField(existingField.asField()); - } - } else if (!docField.getDataType().equals(field.getDataType())) { - if (validate) - throw newProcessException(schema, field, "Summary field has conflicting type."); - } - } - - private void addField(Schema schema, SDDocumentType document, Field field, boolean validate) { - if (document.getField(field.getName()) != null && !(document.getField(field.getName()) == field)) { - if (validate) - throw newProcessException(schema, field, "Field shadows another."); - } - document.addField(field); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java deleted file mode 100644 index 254b3743f52..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryField.Source; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/* - * Adjusts position summary fields by adding derived summary fields (.distance and .position) and setting summary - * transform and source. - */ -public class AdjustPositionSummaryFields extends Processor { - - public AdjustPositionSummaryFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - private boolean useV8GeoPositions = false; - - @Override - public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { - this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); - process(validate, documentsOnly); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (DocumentSummary summary : schema.getSummaries().values()) { - scanSummary(summary); - } - } - - private void scanSummary(DocumentSummary summary) { - for (SummaryField summaryField : summary.getSummaryFields().values()) { - if ( ! GeoPos.isAnyPos(summaryField.getDataType())) continue; - - String originalSource = summaryField.getSingleSource(); - if (originalSource.indexOf('.') == -1) { // Eliminate summary fields with pos.x or pos.y as source - ImmutableSDField sourceField = schema.getField(originalSource); - if (sourceField != null) { - String zCurve = null; - if (sourceField.getDataType().equals(summaryField.getDataType())) { - zCurve = PositionDataType.getZCurveFieldName(originalSource); - } else if (sourceField.getDataType().equals(makeZCurveDataType(summaryField.getDataType())) && - hasZCurveSuffix(originalSource)) { - zCurve = originalSource; - } - if (zCurve != null) { - if (hasPositionAttribute(zCurve)) { - Source source = new Source(zCurve); - adjustPositionField(summary, summaryField, source); - } else if (sourceField.isImportedField() || !summaryField.getName().equals(originalSource)) { - fail(summaryField, "No position attribute '" + zCurve + "'"); - } - } - } - } - } - } - - private void adjustPositionField(DocumentSummary summary, SummaryField summaryField, Source source) { - summaryField.setTransform(SummaryTransform.GEOPOS); - summaryField.getSources().clear(); - summaryField.addSource(source); - ensureSummaryField(summary, - PositionDataType.getPositionSummaryFieldName(summaryField.getName()), - DataType.getArray(DataType.STRING), - source, - SummaryTransform.POSITIONS); - ensureSummaryField(summary, - PositionDataType.getDistanceSummaryFieldName(summaryField.getName()), - DataType.INT, - source, - SummaryTransform.DISTANCE); - } - - private void ensureSummaryField(DocumentSummary summary, String fieldName, DataType dataType, Source source, SummaryTransform transform) { - SummaryField oldField = schema.getSummaryField(fieldName); - if (oldField == null) { - if (useV8GeoPositions) return; - SummaryField newField = new SummaryField(fieldName, dataType, transform); - newField.addSource(source); - summary.add(newField); - return; - } - if (!oldField.getDataType().equals(dataType)) { - fail(oldField, "exists with type '" + oldField.getDataType().toString() + "', should be of type '" + dataType.toString() + "'"); - } - if (oldField.getTransform() != transform) { - fail(oldField, "has summary transform '" + oldField.getTransform().toString() + "', should have transform '" + transform.toString() + "'"); - } - if (oldField.getSourceCount() != 1 || !oldField.getSingleSource().equals(source.getName())) { - fail(oldField, "has source '" + oldField.getSources().toString() + "', should have source '" + source + "'"); - } - if (useV8GeoPositions) return; - summary.add(oldField); - } - - private boolean hasPositionAttribute(String name) { - Attribute attribute = schema.getAttribute(name); - if (attribute == null) { - ImmutableSDField field = schema.getField(name); - if (field != null && field.isImportedField()) { - attribute = field.getAttribute(); - } - } - return attribute != null && attribute.isPosition(); - } - - private static boolean hasZCurveSuffix(String name) { - String suffix = PositionDataType.getZCurveFieldName(""); - return name.length() > suffix.length() && name.substring(name.length() - suffix.length()).equals(suffix); - } - - private static DataType makeZCurveDataType(DataType dataType) { - return dataType instanceof ArrayDataType ? DataType.getArray(DataType.LONG) : DataType.LONG; - } - - private void fail(SummaryField summaryField, String msg) { - throw newProcessException(schema.getName(), summaryField.getName(), msg); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java deleted file mode 100644 index 42e14a2c1ee..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Checks that attribute properties only are set for attributes that have data (are created by an indexing statement). - * - * @author hmusum - */ -public class AttributeProperties extends Processor { - - public AttributeProperties(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (ImmutableSDField field : schema.allConcreteFields()) { - String fieldName = field.getName(); - - // For each attribute, check if the attribute has been created - // by an indexing statement. - for (Attribute attribute : field.getAttributes().values()) { - if (attributeCreated(field, attribute.getName())) { - continue; - } - // Check other fields or statements that may have created this attribute. - boolean created = false; - for (SDField f : schema.allConcreteFields()) { - // Checking against the field we are looking at - if (!f.getName().equals(fieldName)) { - if (attributeCreated(f, attribute.getName())) { - created = true; - break; - } - } - } - if (validate && !created) { - throw new IllegalArgumentException("Attribute '" + attribute.getName() + "' in field '" + - field.getName() + "' is not created by the indexing statement"); - } - } - } - } - - /** - * Checks if the attribute has been created bye an indexing statement in this field. - * - * @param field a searchdefinition field - * @param attributeName name of the attribute - * @return true if the attribute has been created by this field, else false - */ - static boolean attributeCreated(ImmutableSDField field, String attributeName) { - if ( ! field.doesAttributing()) { - return false; - } - for (Attribute attribute : field.getAttributes().values()) { - if (attribute.getName().equals(attributeName)) { - return true; - } - } - return false; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java deleted file mode 100644 index 3c10ccb92dc..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.document.NumericDataType; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Fields that derive to attribute(s) and no indices should use the WORD indexing form, - * in a feeble attempt to match the most peoples expectations as closely as possible. - * - * @author Vegard Havdal - */ -public class AttributesImplicitWord extends Processor { - - public AttributesImplicitWord(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (ImmutableSDField field : schema.allConcreteFields()) { - processFieldRecursive(field); - } - } - - private void processFieldRecursive(ImmutableSDField field) { - processField(field); - for (ImmutableSDField structField : field.getStructFields()) { - processFieldRecursive(structField); - } - } - - private void processField(ImmutableSDField field) { - if (fieldImplicitlyWordMatch(field)) { - field.getMatching().setType(MatchType.WORD); - } - } - - private boolean fieldImplicitlyWordMatch(ImmutableSDField field) { - // numeric types should not trigger exact-match query parsing - DataType dt = field.getDataType().getPrimitiveType(); - if (dt != null && dt instanceof NumericDataType) { - return false; - } - return (! field.hasIndex() - && !field.getAttributes().isEmpty() - && field.getIndices().isEmpty() - && !field.getMatching().isTypeUserSet()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java deleted file mode 100644 index 575a037be91..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Checks that bolding or dynamic summary is turned on only for text fields. Throws exception if it is turned on for any - * other fields (otherwise will cause indexing failure) - * - * @author hmusum - */ -public class Bolding extends Processor { - - public Bolding(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - for (ImmutableSDField field : schema.allConcreteFields()) { - for (SummaryField summary : field.getSummaryFields().values()) { - if (summary.getTransform().isBolded() && - !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) - { - throw new IllegalArgumentException("'bolding: on' for non-text field " + - "'" + field.getName() + "'" + - " (" + summary.getDataType() + ")" + - " is not allowed"); - } else if (summary.getTransform().isDynamic() && - !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) - { - throw new IllegalArgumentException("'summary: dynamic' for non-text field " + - "'" + field.getName() + "'" + - " (" + summary.getDataType() + ")" + - " is not allowed"); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java deleted file mode 100644 index 6676e20c341..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Validates attribute fields using bool type, ensuring the collection type is supported. - * - * Currently, only the single value bool type is supported. - * - * @author geirst - */ -public class BoolAttributeValidator extends Processor { - - public BoolAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (var field : schema.allConcreteFields()) { - var attribute = field.getAttribute(); - if (attribute == null) { - continue; - } - if (attribute.getType().equals(Attribute.Type.BOOL) && - !attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE)) { - fail(schema, field, "Only single value bool attribute fields are supported"); - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java deleted file mode 100644 index ce93c1a5c3f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DocumentType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document - * - * @author Vegard Havdal - */ -public class BuiltInFieldSets extends Processor { - - public static final String SEARCH_FIELDSET_NAME = "[search]"; // Public due to oddities in position handling. - public static final String INTERNAL_FIELDSET_NAME = "[internal]"; // This one populated from misc places - - public BuiltInFieldSets(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - addDocumentFieldSet(); - addSearchFieldSet(); - // "Hook" the field sets on search onto the document types, since we will include them - // on the document configs - schema.getDocument().setFieldSets(schema.fieldSets()); - } - - private void addSearchFieldSet() { - for (SDField searchField : schema.extraFieldList()) { - schema.fieldSets().addBuiltInFieldSetItem(SEARCH_FIELDSET_NAME, searchField.getName()); - } - } - - private void addDocumentFieldSet() { - for (Field docField : schema.getDocument().fieldSet()) { - if (docField instanceof SDField && ((SDField) docField).isExtraField()) { - continue; // skip - } - schema.fieldSets().addBuiltInFieldSetItem(DocumentType.DOCUMENT, docField.getName()); - } - } - - - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java deleted file mode 100644 index d7882c7f8fb..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.logging.Level; - -/** - * Adds a "fieldName_zcurve" long attribute and "fieldName.distance" and "FieldName.position" summary fields to all position type fields. - * - * @author bratseth - */ -public class CreatePositionZCurve extends Processor { - - private final SDDocumentType repo; - - public CreatePositionZCurve(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - this.repo = schema.getDocument(); - } - - private boolean useV8GeoPositions = false; - - @Override - public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { - this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions(); - process(validate, documentsOnly); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - DataType fieldType = field.getDataType(); - if ( ! isSupportedPositionType(fieldType)) continue; - - if (validate && field.doesIndexing()) { - fail(schema, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " + - "replace 'index' statement with 'attribute'."); - } - - if ( ! field.doesAttributing()) continue; - - boolean doesSummary = field.doesSummarying(); - - String fieldName = field.getName(); - field.getAttributes().remove(fieldName); - - String zName = PositionDataType.getZCurveFieldName(fieldName); - SDField zCurveField = createZCurveField(field, zName, validate); - schema.addExtraField(zCurveField); - schema.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName()); - - // configure summary - Collection summaryTo = removeSummaryTo(field); - if (! useV8GeoPositions) { - ensureCompatibleSummary(field, zName, - PositionDataType.getPositionSummaryFieldName(fieldName), - DataType.getArray(DataType.STRING), // will become "xmlstring" - SummaryTransform.POSITIONS, summaryTo, validate); - ensureCompatibleSummary(field, zName, - PositionDataType.getDistanceSummaryFieldName(fieldName), - DataType.INT, - SummaryTransform.DISTANCE, summaryTo, validate); - } - // clear indexing script - field.setIndexingScript(null); - SDField posX = field.getStructField(PositionDataType.FIELD_X); - if (posX != null) { - posX.setIndexingScript(null); - } - SDField posY = field.getStructField(PositionDataType.FIELD_Y); - if (posY != null) { - posY.setIndexingScript(null); - } - if (doesSummary) ensureCompatibleSummary(field, zName, - field.getName(), - field.getDataType(), - SummaryTransform.GEOPOS, summaryTo, validate); - } - } - - private SDField createZCurveField(SDField inputField, String fieldName, boolean validate) { - if (validate && schema.getConcreteField(fieldName) != null || schema.getAttribute(fieldName) != null) { - throw newProcessException(schema, null, "Incompatible position attribute '" + fieldName + - "' already created."); - } - boolean isArray = inputField.getDataType() instanceof ArrayDataType; - SDField field = new SDField(repo, fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); - Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY : - Attribute.CollectionType.SINGLE); - attribute.setPosition(true); - attribute.setFastSearch(true); - field.addAttribute(attribute); - - ScriptExpression script = inputField.getIndexingScript(); - script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script); - script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script); - field.setIndexingScript(script); - return field; - } - - private void ensureCompatibleSummary(SDField field, String sourceName, String summaryName, DataType summaryType, - SummaryTransform summaryTransform, Collection summaryTo, boolean validate) { - SummaryField summary = schema.getSummaryField(summaryName); - if (summary == null) { - summary = new SummaryField(summaryName, summaryType, summaryTransform); - summary.addDestination("default"); - summary.addDestinations(summaryTo); - field.addSummaryField(summary); - } else if (!summary.getDataType().equals(summaryType)) { - if (validate) - fail(schema, field, "Incompatible summary field '" + summaryName + "' type " + summary.getDataType() + " already created."); - } else if (summary.getTransform() == SummaryTransform.NONE) { - summary.setTransform(summaryTransform); - summary.addDestination("default"); - summary.addDestinations(summaryTo); - } else if (summary.getTransform() != summaryTransform) { - deployLogger.logApplicationPackage(Level.WARNING, "Summary field " + summaryName + " has wrong transform: " + summary.getTransform()); - return; - } - SummaryField.Source source = new SummaryField.Source(sourceName); - summary.getSources().clear(); - summary.addSource(source); - } - - private Set removeSummaryTo(SDField field) { - Set summaryTo = new HashSet<>(); - Collection summaryFields = field.getSummaryFields().values(); - for (SummaryField summary : summaryFields) { - summaryTo.addAll(summary.getDestinations()); - } - field.removeSummaryFields(); - return summaryTo; - } - - private static boolean isSupportedPositionType(DataType dataType) { - return GeoPos.isAnyPos(dataType); - } - - private static class RemoveSummary extends ExpressionConverter { - - final String find; - - RemoveSummary(String find) { - this.find = find; - } - - @Override - protected boolean shouldConvert(Expression exp) { - if (!(exp instanceof SummaryExpression)) { - return false; - } - String fieldName = ((SummaryExpression)exp).getFieldName(); - return fieldName == null || fieldName.equals(find); - } - - @Override - protected Expression doConvert(Expression exp) { - return null; - } - } - - private static class PerformZCurve extends ExpressionConverter { - - final String find; - final String replace; - final boolean isArray; - - PerformZCurve(SDField find, String replace) { - this.find = find.getName(); - this.replace = replace; - this.isArray = find.getDataType() instanceof ArrayDataType; - } - - @Override - protected boolean shouldConvert(Expression exp) { - if (!(exp instanceof AttributeExpression)) { - return false; - } - String fieldName = ((AttributeExpression)exp).getFieldName(); - return fieldName == null || fieldName.equals(find); - } - - @Override - protected Expression doConvert(Expression exp) { - return new StatementExpression( - isArray ? new ForEachExpression(new ZCurveExpression()) : - new ZCurveExpression(), new AttributeExpression(replace)); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java deleted file mode 100644 index dc620e38e96..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.NumericDataType; -import com.yahoo.document.PrimitiveDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Propagates dictionary settings from field level to attribute level. - * Only applies to numeric fields with fast-search enabled. - * - * @author baldersheim - */ -public class DictionaryProcessor extends Processor { - public DictionaryProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - Attribute attribute = field.getAttribute(); - if (attribute == null) continue; - attribute.setCase(field.getMatching().getCase()); - Dictionary dictionary = field.getDictionary(); - if (dictionary == null) continue; - if (attribute.getDataType().getPrimitiveType() instanceof NumericDataType ) { - if (attribute.isFastSearch()) { - attribute.setDictionary(dictionary); - } else { - fail(schema, field, "You must specify 'attribute:fast-search' to allow dictionary control"); - } - } else if (attribute.getDataType().getPrimitiveType() == PrimitiveDataType.STRING) { - attribute.setDictionary(dictionary); - if (dictionary.getType() == Dictionary.Type.HASH) { - if (dictionary.getMatch() != Case.CASED) { - fail(schema, field, "hash dictionary require cased match"); - } - } - if (! dictionary.getMatch().equals(attribute.getCase())) { - fail(schema, field, "Dictionary casing '" + dictionary.getMatch() + "' does not match field match casing '" + attribute.getCase() + "'"); - } - } else { - fail(schema, field, "You can only specify 'dictionary:' for numeric or string fields"); - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java deleted file mode 100644 index 6ffe00458a8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.document.MapDataType; -import com.yahoo.document.PrimitiveDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Non-primitive key types for map and weighted set forbidden (though OK in document model) - * - * @author Vegard Havdal - */ -public class DisallowComplexMapAndWsetKeyTypes extends Processor { - - public DisallowComplexMapAndWsetKeyTypes(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - // TODO also traverse struct types to search for bad map or wset types. - // Do this after document manager is fixed, do not start using the static stuff on SDDocumentTypes any more. - for (SDField field : schema.allConcreteFields()) { - checkFieldType(field, field.getDataType()); - } - } - - private void checkFieldType(Field field, DataType dataType) { - if (dataType instanceof ArrayDataType) { - DataType nestedType = ((ArrayDataType) dataType).getNestedType(); - checkFieldType(field, nestedType); - } else if (dataType instanceof WeightedSetDataType) { - DataType nestedType = ((WeightedSetDataType) dataType).getNestedType(); - if ( ! (nestedType instanceof PrimitiveDataType)) { - fail(schema, field, "Weighted set must have a primitive key type."); - } - } else if (dataType instanceof MapDataType) { - DataType keyType = ((MapDataType) dataType).getKeyType(); - if ( ! (keyType instanceof PrimitiveDataType)) { - fail(schema, field, "Map key type must be a primitive type."); - } - checkFieldType(field, ((MapDataType) dataType).getValueType()); - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java deleted file mode 100644 index 3759fc453df..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * @author baldersheim - */ -public class DiversitySettingsValidator extends Processor { - - public DiversitySettingsValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - if (documentsOnly) return; - - for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { - validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); - } - } - } - private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { - String attributeName = settings.getAttribute(); - new AttributeValidator(schema.getName(), rankProfile.name(), - schema.getAttribute(attributeName), attributeName).validate(); - } - - private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { - - public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { - super(searchName, rankProfileName, attribute, attributeName); - } - - protected void validateThatAttributeIsSingleAndNotPredicate() { - if ( ! attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || - attribute.getType().equals(Attribute.Type.PREDICATE)) - { - failValidation("must be single value numeric, or enumerated attribute, but it is '" - + attribute.getDataType().getName() + "'"); - } - } - - @Override - public void validate() { - validateThatAttributeExists(); - validateThatAttributeIsSingleAndNotPredicate(); - } - - @Override - public String getValidationType() { - return "diversity"; - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java deleted file mode 100644 index ca473a2029f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; -import com.yahoo.vespa.indexinglanguage.expressions.ExactExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * The implementation of exact matching - * - * @author bratseth - */ -public class ExactMatch extends Processor { - - public static final String DEFAULT_EXACT_TERMINATOR = "@@"; - - ExactMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - processField(field, schema); - } - } - - private void processField(SDField field, Schema schema) { - MatchType matching = field.getMatching().getType(); - if (matching.equals(MatchType.EXACT) || matching.equals(MatchType.WORD)) { - implementExactMatch(field, schema); - } else if (field.getMatching().getExactMatchTerminator() != null) { - warn(schema, field, "exact-terminator requires 'exact' matching to have any effect."); - } - for (var structField : field.getStructFields()) { - processField(structField, schema); - } - } - - private void implementExactMatch(SDField field, Schema schema) { - field.setStemming(Stemming.NONE); - field.getNormalizing().inferLowercase(); - - if (field.getMatching().getType().equals(MatchType.WORD)) { - field.addQueryCommand("word"); - } else { // exact - String exactTerminator = DEFAULT_EXACT_TERMINATOR; - if (field.getMatching().getExactMatchTerminator() != null - && ! field.getMatching().getExactMatchTerminator().equals("")) { - exactTerminator = field.getMatching().getExactMatchTerminator(); - } else { - info(schema, field, - "With 'exact' matching, an exact-terminator is needed," + - " using default value '" + exactTerminator +"' as terminator"); - } - field.addQueryCommand("exact " + exactTerminator); - - // The following part illustrates how nice it would have been with canonical representation of indices - if (field.doesIndexing()) { - exactMatchSettingsForField(field); - } - } - ScriptExpression script = field.getIndexingScript(); - if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript((ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); - } - } - - private void exactMatchSettingsForField(SDField field) { - field.getRanking().setFilter(true); - } - - private static class MyProvider extends TypedTransformProvider { - - MyProvider(Schema schema) { - super(ExactExpression.class, schema); - } - - @Override - protected boolean requiresTransform(Expression exp, DataType fieldType) { - return exp instanceof OutputExpression; - } - - @Override - protected Expression newTransform(DataType fieldType) { - Expression exp = new ExactExpression(); - if (fieldType instanceof CollectionDataType) { - exp = new ForEachExpression(exp); - } - return exp; - } - - } - -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java deleted file mode 100644 index 60cc5c1cbb4..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.stream.Collectors; - -/** - * Validates the use of the fast-access property. - * - * @author bjorncs - */ -public class FastAccessValidator extends Processor { - - public FastAccessValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - String invalidAttributes = schema.allFields() - .flatMap(field -> field.getAttributes().values().stream()) - .filter(FastAccessValidator::isIncompatibleAttribute) - .map(Attribute::getName) - .collect(Collectors.joining(", ")); - if ( ! invalidAttributes.isEmpty()) { - throw new IllegalArgumentException( - "For " + schema + ": The following attributes have a type that is incompatible with fast-access: " + - invalidAttributes + ". Predicate, tensor and reference attributes are incompatible with fast-access."); - } - } - - private static boolean isIncompatibleAttribute(Attribute attribute) { - return attribute.isFastAccess() && isTypeIncompatibleWithFastAccess(attribute.getType()); - } - - private static boolean isTypeIncompatibleWithFastAccess(Attribute.Type type) { - switch (type) { - case PREDICATE: - case TENSOR: - case REFERENCE: - return true; - default: - return false; - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java deleted file mode 100644 index 1c50f78b539..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Computes the right "index commands" for each fieldset in a search definition. - * - * @author vegardh - * @author bratseth - */ -// See also IndexInfo.addFieldSetCommands, which does more of this in a complicated way. -// That should be moved here, and done in the way the match setting is done below -// (this requires adding normalizing and stemming settings to FieldSet). -public class FieldSetSettings extends Processor { - - public FieldSetSettings(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (FieldSet fieldSet : schema.fieldSets().userFieldSets().values()) { - if (validate) - checkFieldNames(schema, fieldSet); - checkMatching(schema, fieldSet); - checkNormalization(schema, fieldSet); - checkStemming(schema, fieldSet); - } - } - - private void checkFieldNames(Schema schema, FieldSet fieldSet) { - for (String field : fieldSet.getFieldNames()) { - if (schema.getField(field) == null) - throw new IllegalArgumentException("For " + schema + ": Field '" + field + "' in " + - fieldSet + " does not exist."); - } - } - - private void checkMatching(Schema schema, FieldSet fieldSet) { - Matching matching = fieldSet.getMatching(); - for (String fieldName : fieldSet.getFieldNames()) { - ImmutableSDField field = schema.getField(fieldName); - Matching fieldMatching = field.getMatching(); - if (matching == null) { - matching = fieldMatching; - } else { - if ( ! matching.equals(fieldMatching)) { - warn(schema, field.asField(), - "The matching settings for the fields in " + fieldSet + " are inconsistent " + - "(explicitly or because of field type). This may lead to recall and ranking issues."); - return; - } - } - } - fieldSet.setMatching(matching); // Assign the uniquely determined matching to the field set - } - - private void checkNormalization(Schema schema, FieldSet fieldSet) { - NormalizeLevel.Level normalizing = null; - for (String fieldName : fieldSet.getFieldNames()) { - ImmutableSDField field = schema.getField(fieldName); - NormalizeLevel.Level fieldNorm = field.getNormalizing().getLevel(); - if (normalizing == null) { - normalizing = fieldNorm; - } else { - if ( ! normalizing.equals(fieldNorm)) { - warn(schema, field.asField(), - "The normalization settings for the fields in " + fieldSet + " are inconsistent " + - "(explicitly or because of field type). This may lead to recall and ranking issues."); - return; - } - } - } - } - - private void checkStemming(Schema schema, FieldSet fieldSet) { - Stemming stemming = null; - for (String fieldName : fieldSet.getFieldNames()) { - ImmutableSDField field = schema.getField(fieldName); - Stemming fieldStemming = field.getStemming(); - if (stemming == null) { - stemming = fieldStemming; - } else { - if ( ! stemming.equals(fieldStemming)) { - warn(schema, field.asField(), - "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+ - "' are inconsistent (explicitly or because of field type). " + - "This may lead to recall and ranking issues."); - return; - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java deleted file mode 100644 index 3f97bf83565..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.Set; -import java.util.logging.Level; - -/** - * Takes the fields and indexes that are of type rank filter, and stores those names on all rank profiles - * - * @author Vegard Havdal - */ -public class FilterFieldNames extends Processor { - - public FilterFieldNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (documentsOnly) return; - - for (SDField f : schema.allConcreteFields()) { - if (f.getRanking().isFilter()) { - filterField(f.getName()); - } - } - - for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - Set filterFields = new LinkedHashSet<>(); - findFilterFields(schema, profile, filterFields); - for (Iterator itr = filterFields.iterator(); itr.hasNext(); ) { - String fieldName = itr.next(); - profile.filterFields().add(fieldName); - profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.EMPTY); - } - } - } - - private void filterField(String f) { - for (RankProfile rp : rankProfileRegistry.rankProfilesOf(schema)) { - rp.filterFields().add(f); - } - } - - private void findFilterFields(Schema schema, RankProfile profile, Set filterFields) { - for (Iterator itr = profile.declaredRankSettingIterator(); itr.hasNext(); ) { - RankProfile.RankSetting setting = itr.next(); - if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) && ((Boolean)setting.getValue())) - { - String fieldName = setting.getFieldName(); - if (schema.getConcreteField(fieldName) != null) { - if ( ! profile.filterFields().contains(fieldName)) { - filterFields.add(fieldName); - } - } else { - deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.name() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java deleted file mode 100644 index 0db6f4f05ba..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import java.util.logging.Level; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; - -/** - * Makes implicitly defined summaries into explicit summaries - * - * @author bratseth - */ -public class ImplicitSummaries extends Processor { - - public ImplicitSummaries(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - DocumentSummary defaultSummary = schema.getSummariesInThis().get("default"); - if (defaultSummary == null) { - defaultSummary = new DocumentSummary("default", schema); - defaultSummary.setFromDisk(true); // As we add documentid to this - schema.addSummary(defaultSummary); - } - - for (SDField field : schema.allConcreteFields()) { - collectSummaries(field, schema, validate); - } - for (DocumentSummary documentSummary : schema.getSummaries().values()) { - documentSummary.purgeImplicits(); - } - } - - private void addSummaryFieldSources(SummaryField summaryField, SDField sdField) { - sdField.addSummaryFieldSources(summaryField); - } - - private void collectSummaries(SDField field, Schema schema, boolean validate) { - SummaryField addedSummaryField = null; - - // Implicit - String fieldName = field.getName(); - SummaryField fieldSummaryField = field.getSummaryField(fieldName); - if (fieldSummaryField == null && field.doesSummarying()) { - fieldSummaryField = new SummaryField(fieldName, field.getDataType()); - fieldSummaryField.setImplicit(true); - addSummaryFieldSources(fieldSummaryField, field); - fieldSummaryField.addDestination("default"); - field.addSummaryField(fieldSummaryField); - addedSummaryField = fieldSummaryField; - } - if (fieldSummaryField != null) { - for (String dest : fieldSummaryField.getDestinations()) { - DocumentSummary summary = schema.getSummariesInThis().get(dest); - if (summary != null) { - summary.add(fieldSummaryField); - } - } - } - - // Attribute prefetch - for (Attribute attribute : field.getAttributes().values()) { - if (attribute.getName().equals(fieldName)) { - if (addedSummaryField != null) { - addedSummaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - if (attribute.isPrefetch()) { - addPrefetchAttribute(attribute, field, schema); - } - } - } - - if (addedSummaryField != null && isComplexFieldWithOnlyStructFieldAttributes(field)) { - addedSummaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); - } - - // Position attributes - if (field.doesSummarying()) { - for (Attribute attribute : field.getAttributes().values()) { - if ( ! attribute.isPosition()) continue; - var distField = field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)); - if (distField != null) { - DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); - attributePrefetchSummary.add(distField); - } - var posField = field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)); - if (posField != null) { - DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); - attributePrefetchSummary.add(posField); - } - } - } - - // Explicits - for (SummaryField summaryField : field.getSummaryFields().values()) { - // Make sure we fetch from attribute here too - Attribute attribute = field.getAttributes().get(fieldName); - if (attribute != null && summaryField.getTransform() == SummaryTransform.NONE) { - summaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - if (isValid(summaryField, schema, validate)) { - addToDestinations(summaryField, schema); - } - } - - } - - private DocumentSummary getOrCreateAttributePrefetchSummary(Schema schema) { - DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); - if (summary == null) { - summary = new DocumentSummary("attributeprefetch", schema); - schema.addSummary(summary); - } - return summary; - } - - - private void addPrefetchAttribute(Attribute attribute, SDField field, Schema schema) { - if (attribute.getPrefetchValue() == null) { // Prefetch by default - unless any summary makes this dynamic - // Check if there is an implicit dynamic definition - SummaryField fieldSummaryField = field.getSummaryField(attribute.getName()); - if (fieldSummaryField != null && fieldSummaryField.getTransform().isDynamic()) return; - - // Check if an explicit class makes it dynamic (first is enough, as all must be the same, checked later) - SummaryField explicitSummaryField = schema.getExplicitSummaryField(attribute.getName()); - if (explicitSummaryField != null && explicitSummaryField.getTransform().isDynamic()) return; - } - - DocumentSummary summary = getOrCreateAttributePrefetchSummary(schema); - SummaryField attributeSummaryField = new SummaryField(attribute.getName(), attribute.getDataType()); - attributeSummaryField.addSource(attribute.getName()); - attributeSummaryField.addDestination("attributeprefetch"); - attributeSummaryField.setTransform(SummaryTransform.ATTRIBUTE); - summary.add(attributeSummaryField); - } - - // Returns whether this is valid. Warns if invalid and ignorable. Throws if not ignorable. - private boolean isValid(SummaryField summaryField, Schema schema, boolean validate) { - if (summaryField.getTransform() == SummaryTransform.DISTANCE || - summaryField.getTransform() == SummaryTransform.POSITIONS) { - int sourceCount = summaryField.getSourceCount(); - if (validate && sourceCount != 1) { - throw newProcessException(schema.getName(), summaryField.getName(), - "Expected 1 source field, got " + sourceCount + "."); - } - String sourceName = summaryField.getSingleSource(); - if (validate && schema.getAttribute(sourceName) == null) { - throw newProcessException(schema.getName(), summaryField.getName(), - "Summary source attribute '" + sourceName + "' not found."); - } - return true; - } - - String fieldName = summaryField.getSourceField(); - SDField sourceField = schema.getConcreteField(fieldName); - if (validate && sourceField == null) { - throw newProcessException(schema, summaryField, "Source field '" + fieldName + "' does not exist."); - } - if (! sourceField.doesSummarying() && - summaryField.getTransform() != SummaryTransform.ATTRIBUTE && - summaryField.getTransform() != SummaryTransform.GEOPOS) - { - // Summary transform attribute may indicate that the ilscript was rewritten to remove summary - // by another search that uses this same field in inheritance. - deployLogger.logApplicationPackage(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField + - " is not creating a summary value in its indexing statement"); - return false; - } - - if (summaryField.getTransform().isDynamic() - && summaryField.getName().equals(sourceField.getName()) - && sourceField.doesAttributing()) { - Attribute attribute = sourceField.getAttributes().get(sourceField.getName()); - if (attribute != null) { - String destinations = "document summary 'default'"; - if (summaryField.getDestinations().size() >0) { - destinations = "document summaries " + summaryField.getDestinations(); - } - deployLogger.logApplicationPackage(Level.WARNING, - "Will fetch the disk summary value of " + sourceField + " in " + destinations + - " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " + - "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " + - "to this attribute, remove any bolding and dynamic snippeting from this field"); - // Note: The dynamic setting has already overridden the attribute map setting, - // so we do not need to actually do attribute.setSummary(false) here - // Also, we can not do this, since it makes it impossible to fetch this attribute - // in another summary - } - } - - return true; - } - - private void addToDestinations(SummaryField summaryField, Schema schema) { - if (summaryField.getDestinations().size() == 0) { - addToDestination("default", summaryField, schema); - } - else { - for (String destinationName : summaryField.getDestinations()) { - addToDestination(destinationName, summaryField, schema); - } - } - } - - private void addToDestination(String destinationName, SummaryField summaryField, Schema schema) { - DocumentSummary destination = schema.getSummariesInThis().get(destinationName); - if (destination == null) { - destination = new DocumentSummary(destinationName, schema); - schema.addSummary(destination); - destination.add(summaryField); - } - else { - SummaryField existingField= destination.getSummaryField(summaryField.getName()); - SummaryField merged = summaryField.mergeWith(existingField); - destination.add(merged); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java deleted file mode 100644 index df103dcb096..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * This processor adds all implicit summary fields to all registered document summaries. If another field has already - * been registered with one of the implicit names, this processor will throw an {@link IllegalStateException}. - */ -public class ImplicitSummaryFields extends Processor { - - public ImplicitSummaryFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (DocumentSummary docsum : schema.getSummariesInThis().values()) { - if (docsum.inherited().isPresent()) continue; // Implicit fields are added to inheriting summaries through their parent - addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); - addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); - } - } - - private void addField(DocumentSummary docsum, SummaryField field, boolean validate) { - if (validate && docsum.getSummaryField(field.getName()) != null) { - throw new IllegalArgumentException("Summary class '" + docsum.getName() + "' uses reserved field name '" + - field.getName() + "'."); - } - docsum.add(field); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java deleted file mode 100644 index e836caac10d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedComplexField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.ImportedSimpleField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; - -/** - * Iterates all imported fields from schema parsing and validates and resolves them into concrete fields from referenced document types. - * - * @author geirst - */ -public class ImportedFieldsResolver extends Processor { - - private final Map importedFields = new LinkedHashMap<>(); - private final Optional references; - - public ImportedFieldsResolver(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - references = schema.getDocument().getDocumentReferences(); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - schema.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); - schema.setImportedFields(new ImportedFields(importedFields)); - } - - private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { - DocumentReference reference = validateDocumentReference(importedField); - ImmutableSDField targetField = getTargetField(importedField, reference); - if (GeoPos.isAnyPos(targetField)) { - resolveImportedPositionField(importedField, reference, targetField, validate); - } else if (isArrayOfSimpleStruct(targetField)) { - resolveImportedArrayOfStructField(importedField, reference, targetField, validate); - } else if (isMapOfSimpleStruct(targetField)) { - resolveImportedMapOfStructField(importedField, reference, targetField, validate); - } else if (isMapOfPrimitiveType(targetField)) { - resolveImportedMapOfPrimitiveField(importedField, reference, targetField, validate); - } else { - resolveImportedNormalField(importedField, reference, targetField, validate); - } - } - - private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { - TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), - reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); - ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); - resolveImportedNormalField(importedZCurveField, reference, targetZCurveField, validate); - ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName(), reference, targetField); - registerImportedField(importedField, null, importedStructField); - } - - private void resolveImportedArrayOfStructField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { - ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName(), reference, targetField); - resolveImportedNestedStructField(importedField, reference, importedStructField, targetField, validate); - registerImportedField(importedField, null, importedStructField); - } - - private void resolveImportedMapOfStructField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { - ImportedComplexField importedMapField = new ImportedComplexField(importedField.fieldName(), reference, targetField); - ImportedComplexField importedStructField = new ImportedComplexField(importedField.fieldName() + ".value", reference, targetField.getStructField("value")); - importedMapField.addNestedField(importedStructField); - resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("key"), validate); - resolveImportedNestedStructField(importedField, reference, importedStructField, importedStructField.targetField(), validate); - registerImportedField(importedField, null, importedMapField); - } - - private void makeImportedNormalField(TemporaryImportedField importedField, ImportedComplexField owner, String name, DocumentReference reference, ImmutableSDField targetField) { - ImportedField importedSimpleField = new ImportedSimpleField(name, reference, targetField); - registerImportedField(importedField, owner, importedSimpleField); - } - - private void registerImportedField(TemporaryImportedField temporaryImportedField, ImportedComplexField owner, ImportedField importedField) { - if (owner != null) { - owner.addNestedField(importedField); - } else { - if (importedFields.get(importedField.fieldName()) != null) { - fail(temporaryImportedField, importedField.fieldName(), targetFieldAsString(importedField.targetField().getName(), importedField.reference()) + ": Field already imported"); - } - importedFields.put(importedField.fieldName(), importedField); - } - } - - private static String makeImportedNestedFieldName(TemporaryImportedField importedField, ImmutableSDField targetNestedField) { - return importedField.fieldName() + targetNestedField.getName().substring(importedField.targetFieldName().length()); - } - - private boolean resolveImportedNestedField(TemporaryImportedField importedField, DocumentReference reference, - ImportedComplexField owner, ImmutableSDField targetNestedField, boolean requireAttribute) { - Attribute attribute = targetNestedField.getAttribute(); - String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); - if (attribute != null) { - makeImportedNormalField(importedField, owner, importedNestedFieldName, reference, targetNestedField); - } else if (requireAttribute) { - fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + - ": Is not an attribute field. Only attribute fields supported"); - } - return attribute != null; - } - - private void resolveImportedNestedStructField(TemporaryImportedField importedField, DocumentReference reference, - ImportedComplexField ownerField, ImmutableSDField targetNestedField, boolean validate) { - boolean foundAttribute = false; - for (ImmutableSDField targetStructField : targetNestedField.getStructFields()) { - if (resolveImportedNestedField(importedField, reference, ownerField, targetStructField, false)) { - foundAttribute = true; - }; - } - if (validate && !foundAttribute) { - String importedNestedFieldName = makeImportedNestedFieldName(importedField, targetNestedField); - fail(importedField, importedNestedFieldName, targetFieldAsString(targetNestedField.getName(), reference) + - ": Is not a struct containing an attribute field."); - } - } - - private void resolveImportedMapOfPrimitiveField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { - ImportedComplexField importedMapField = new ImportedComplexField(importedField.fieldName(), reference, targetField); - resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("key"), validate); - resolveImportedNestedField(importedField, reference, importedMapField, targetField.getStructField("value"), validate); - registerImportedField(importedField, null, importedMapField); - } - - private void resolveImportedNormalField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { - if (validate) { - validateTargetField(importedField, targetField, reference); - } - makeImportedNormalField(importedField, null, importedField.fieldName(), reference, targetField); - } - - private DocumentReference validateDocumentReference(TemporaryImportedField importedField) { - String referenceFieldName = importedField.referenceFieldName(); - DocumentReference reference = references.get().referenceMap().get(referenceFieldName); - if (reference == null) { - fail(importedField, "Reference field '" + referenceFieldName + "' not found"); - } - return reference; - } - - private ImmutableSDField getTargetField(TemporaryImportedField importedField, - DocumentReference reference) { - String targetFieldName = importedField.targetFieldName(); - Schema targetSchema = reference.targetSearch(); - ImmutableSDField targetField = targetSchema.getField(targetFieldName); - if (targetField == null) { - fail(importedField, targetFieldAsString(targetFieldName, reference) + ": Not found"); - } - return targetField; - } - - private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { - if (!targetField.doesAttributing()) { - fail(importedField, targetFieldAsString(targetField.getName(), reference) + - ": Is not an attribute field. Only attribute fields supported"); - } else if (targetField.doesIndexing()) { - fail(importedField, targetFieldAsString(targetField.getName(), reference) + - ": Is an index field. Not supported"); - } else if (targetField.getDataType().equals(DataType.PREDICATE)) { - fail(importedField, targetFieldAsString(targetField.getName(), reference) + - ": Is of type 'predicate'. Not supported"); - } - } - - private static String targetFieldAsString(String targetFieldName, DocumentReference reference) { - return "Field '" + targetFieldName + "' via reference field '" + reference.referenceField().getName() + "'"; - } - - private void fail(TemporaryImportedField importedField, String msg) { - throw new IllegalArgumentException("For " + schema + ", import field '" + - importedField.fieldName() + "': " + msg); - } - - private void fail(TemporaryImportedField importedField, String importedNestedFieldName, String msg) { - if (importedField.fieldName().equals(importedNestedFieldName)) { - fail(importedField, msg); - } - throw new IllegalArgumentException("For " + schema + ", import field '" + - importedField.fieldName() + "' (nested to '" + importedNestedFieldName + "'): " + msg); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java deleted file mode 100644 index 76e97ff0f2e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Because of the way the parser works (allowing any token as identifier), - * it is not practical to limit the syntax of field names there, do it here. - * Important to disallow dash, has semantic in IL. - * - * @author Vehard Havdal - */ -public class IndexFieldNames extends Processor { - - private static final String FIELD_NAME_REGEXP = "[a-zA-Z]\\w*"; - - public IndexFieldNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : schema.allConcreteFields()) { - if ( ! field.getName().matches(FIELD_NAME_REGEXP) && ! legalDottedPositionField(field)) { - fail(schema, field, " Not a legal field name. Legal expression: " + FIELD_NAME_REGEXP); - } - } - } - - /** - * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. - * TODO Vespa 8: Rename to _position and _distance and delete this method. - * - * @param field an {@link com.yahoo.searchdefinition.document.SDField} - * @return true if allowed - */ - private boolean legalDottedPositionField(SDField field) { - return field.getName().endsWith(".position") || field.getName().endsWith(".distance"); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java deleted file mode 100644 index e589bd8e91a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * This processor modifies all indexing scripts so that they input the value of the owning field by default. It also - * ensures that all fields used as input exist. - * - * @author Simon Thoresen Hult - */ -public class IndexingInputs extends Processor { - - public IndexingInputs(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - ScriptExpression script = field.getIndexingScript(); - if (script == null) continue; - - String fieldName = field.getName(); - script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script); - script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script); - if (validate) - new VerifyInputExpression(schema, field).visit(script); - - field.setIndexingScript(script); - } - } - - private static class DefaultToCurrentField extends ExpressionConverter { - - final String fieldName; - - DefaultToCurrentField(String fieldName) { - this.fieldName = fieldName; - } - - @Override - protected boolean shouldConvert(Expression exp) { - return exp instanceof InputExpression && ((InputExpression)exp).getFieldName() == null; - } - - @Override - protected Expression doConvert(Expression exp) { - return new InputExpression(fieldName); - } - } - - private static class EnsureInputExpression extends ExpressionConverter { - - final String fieldName; - - EnsureInputExpression(String fieldName) { - this.fieldName = fieldName; - } - - @Override - protected boolean shouldConvert(Expression exp) { - return exp instanceof StatementExpression; - } - - @Override - protected Expression doConvert(Expression exp) { - if (exp.requiredInputType() != null) { - return new StatementExpression(new InputExpression(fieldName), exp); - } else { - return exp; - } - } - } - - private class VerifyInputExpression extends ExpressionVisitor { - - private final Schema schema; - private final SDField field; - - public VerifyInputExpression(Schema schema, SDField field) { - this.schema = schema; - this.field = field; - } - - @Override - protected void doVisit(Expression exp) { - if ( ! (exp instanceof InputExpression)) return; - String inputField = ((InputExpression)exp).getFieldName(); - if (schema.getField(inputField).hasFullIndexingDocprocRights()) return; - - fail(schema, field, "Indexing script refers to field '" + inputField + "' which does not exist " + - "in document type '" + schema.getDocument().getName() + "', and is not a mutable attribute."); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java deleted file mode 100644 index b10d839535d..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.*; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.*; - -/** - * This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents - * any output expression from writing to any field except for the owning field. Finally, for SummaryExpression, - * this processor expands to write all appropriate summary fields. - * - * @author Simon Thoresen Hult - */ -public class IndexingOutputs extends Processor { - - public IndexingOutputs(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - ScriptExpression script = field.getIndexingScript(); - if (script == null) continue; - - Set summaryFields = new TreeSet<>(); - findSummaryTo(schema, field, summaryFields, summaryFields); - MyConverter converter = new MyConverter(schema, field, summaryFields, validate); - field.setIndexingScript((ScriptExpression)converter.convert(script)); - } - } - - public void findSummaryTo(Schema schema, SDField field, Set dynamicSummary, Set staticSummary) { - var summaryFields = schema.getSummaryFields(field); - if (summaryFields.isEmpty()) { - fillSummaryToFromField(field, dynamicSummary, staticSummary); - } else { - fillSummaryToFromSearch(schema, field, summaryFields, dynamicSummary, staticSummary); - } - } - - private void fillSummaryToFromSearch(Schema schema, SDField field, List summaryFields, - Set dynamicSummary, Set staticSummary) { - for (SummaryField summaryField : summaryFields) { - fillSummaryToFromSummaryField(schema, field, summaryField, dynamicSummary, staticSummary); - } - } - - private void fillSummaryToFromSummaryField(Schema schema, SDField field, SummaryField summaryField, - Set dynamicSummary, Set staticSummary) { - SummaryTransform summaryTransform = summaryField.getTransform(); - String summaryName = summaryField.getName(); - if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) { - // Avoid writing to summary fields that have more than a single input field, as that is handled by the - // summary rewriter in the search core. - return; - } - if (summaryTransform.isDynamic()) { - DataType fieldType = field.getDataType(); - if (fieldType != DataType.URI && fieldType != DataType.STRING) { - warn(schema, field, "Dynamic summaries are only supported for fields of type " + - "string, ignoring summary field '" + summaryField.getName() + - "' for sd field '" + field.getName() + "' of type " + - fieldType.getName() + "."); - return; - } - dynamicSummary.add(summaryName); - } else if (summaryTransform != SummaryTransform.ATTRIBUTE) { - staticSummary.add(summaryName); - } - } - - private static void fillSummaryToFromField(SDField field, Set dynamicSummary, Set staticSummary) { - for (SummaryField summaryField : field.getSummaryFields().values()) { - String summaryName = summaryField.getName(); - if (summaryField.getTransform().isDynamic()) { - dynamicSummary.add(summaryName); - } else { - staticSummary.add(summaryName); - } - } - } - - private class MyConverter extends ExpressionConverter { - - final Schema schema; - final Field field; - final Set summaryFields; - final boolean validate; - - MyConverter(Schema schema, Field field, Set summaryFields, boolean validate) { - this.schema = schema; - this.field = field; - this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields; - this.validate = validate; - } - - @Override - protected boolean shouldConvert(Expression exp) { - if ( ! (exp instanceof OutputExpression)) { - return false; - } - String fieldName = ((OutputExpression)exp).getFieldName(); - if (fieldName == null) { - return true; // inject appropriate field name - } - if ( validate && ! fieldName.equals(field.getName())) { - fail(schema, field, "Indexing expression '" + exp + "' attempts to write to a field other than '" + - field.getName() + "'."); - } - return false; - } - - @Override - protected Expression doConvert(Expression exp) { - List ret = new LinkedList<>(); - if (exp instanceof AttributeExpression) { - ret.add(new AttributeExpression(field.getName())); - } else if (exp instanceof IndexExpression) { - ret.add(new IndexExpression(field.getName())); - } else if (exp instanceof SummaryExpression) { - for (String fieldName : summaryFields) { - ret.add(new SummaryExpression(fieldName)); - } - } else { - throw new UnsupportedOperationException(exp.getClass().getName()); - } - return new StatementExpression(ret); - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java deleted file mode 100644 index 242f5dab308..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.MapDataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.FieldTypeAdapter; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; -import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; -import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashSet; -import java.util.Set; - -/** - * @author Simon Thoresen Hult - */ -public class IndexingValidation extends Processor { - - IndexingValidation(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - VerificationContext context = new VerificationContext(new MyAdapter(schema)); - for (SDField field : schema.allConcreteFields()) { - ScriptExpression script = field.getIndexingScript(); - try { - script.verify(context); - MyConverter converter = new MyConverter(); - for (StatementExpression exp : script) { - converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch - } - } catch (VerificationException e) { - fail(schema, field, "For expression '" + e.getExpression() + "': " + e.getMessage()); - } - } - } - - private static class MyConverter extends ExpressionConverter { - - final Set outputs = new HashSet<>(); - final Set prevNames = new HashSet<>(); - - @Override - protected ExpressionConverter branch() { - MyConverter ret = new MyConverter(); - ret.outputs.addAll(outputs); - ret.prevNames.addAll(prevNames); - return ret; - } - - @Override - protected boolean shouldConvert(Expression exp) { - if (exp instanceof OutputExpression) { - String fieldName = ((OutputExpression)exp).getFieldName(); - if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) { - throw new VerificationException(exp, "Attempting to assign conflicting values to field '" + - fieldName + "'."); - } - outputs.add(fieldName); - prevNames.add(fieldName); - } - if (exp.createdOutputType() != null) { - prevNames.clear(); - } - return false; - } - - @Override - protected Expression doConvert(Expression exp) { - throw new UnsupportedOperationException(); - } - } - - private static class MyAdapter implements FieldTypeAdapter { - - final Schema schema; - - MyAdapter(Schema schema) { - this.schema = schema; - } - - @Override - public DataType getInputType(Expression exp, String fieldName) { - SDField field = schema.getDocumentField(fieldName); - if (field == null) { - throw new VerificationException(exp, "Input field '" + fieldName + "' not found."); - } - return field.getDataType(); - } - - @Override - public void tryOutputType(Expression exp, String fieldName, DataType valueType) { - String fieldDesc; - DataType fieldType; - if (exp instanceof AttributeExpression) { - Attribute attribute = schema.getAttribute(fieldName); - if (attribute == null) { - throw new VerificationException(exp, "Attribute '" + fieldName + "' not found."); - } - fieldDesc = "attribute"; - fieldType = attribute.getDataType(); - } else if (exp instanceof IndexExpression) { - SDField field = schema.getConcreteField(fieldName); - if (field == null) { - throw new VerificationException(exp, "Index field '" + fieldName + "' not found."); - } - fieldDesc = "index field"; - fieldType = field.getDataType(); - } else if (exp instanceof SummaryExpression) { - SummaryField field = schema.getSummaryField(fieldName); - if (field == null) { - throw new VerificationException(exp, "Summary field '" + fieldName + "' not found."); - } - fieldDesc = "summary field"; - fieldType = field.getDataType(); - } else { - throw new UnsupportedOperationException(); - } - if ( ! fieldType.isAssignableFrom(valueType) && - ! fieldType.isAssignableFrom(createCompatType(valueType))) { - throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + - " '" + fieldName + "' which is " + fieldType.getName() + "."); - } - } - - private static DataType createCompatType(DataType origType) { - if (origType instanceof ArrayDataType) { - return DataType.getArray(createCompatType(((ArrayDataType)origType).getNestedType())); - } else if (origType instanceof MapDataType) { - MapDataType mapType = (MapDataType)origType; - return DataType.getMap(createCompatType(mapType.getKeyType()), - createCompatType(mapType.getValueType())); - } else if (origType instanceof WeightedSetDataType) { - return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType())); - } else if (GeoPos.isPos(origType)) { - return DataType.LONG; - } else { - return origType; - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java deleted file mode 100644 index 815e5a42df4..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * @author Simon Thoresen Hult - */ -public class IndexingValues extends Processor { - - public IndexingValues(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (Field field : schema.getDocument().fieldSet()) { - SDField sdField = (SDField)field; - if ( ! sdField.isExtraField()) { - new RequireThatDocumentFieldsAreImmutable(field).convert(sdField.getIndexingScript()); - } - } - } - - private class RequireThatDocumentFieldsAreImmutable extends ExpressionConverter { - - final Field field; - Expression mutatedBy; - - RequireThatDocumentFieldsAreImmutable(Field field) { - this.field = field; - } - - @Override - public ExpressionConverter branch() { - return clone(); - } - - @Override - protected boolean shouldConvert(Expression exp) { - if (exp instanceof OutputExpression && mutatedBy != null) { - throw newProcessException(schema, field, - "Indexing expression '" + mutatedBy + "' attempts to modify the value of the " + - "document field '" + field.getName() + "'. Use a field outside the document " + - "block instead."); - } - if (exp instanceof InputExpression && ((InputExpression)exp).getFieldName().equals(field.getName())) { - mutatedBy = null; - } else if (exp.createdOutputType() != null) { - mutatedBy = exp; - } - return false; - } - - @Override - protected Expression doConvert(Expression exp) { - throw new UnsupportedOperationException(); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java deleted file mode 100644 index ff6224d8d19..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.NumericDataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashSet; -import java.util.Set; - -/** - * Replaces the 'index' statement of all numerical fields to 'attribute' because we no longer support numerical indexes. - * - * @author baldersheim - */ -public class IntegerIndex2Attribute extends Processor { - - public IntegerIndex2Attribute(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { - if (field.getIndex(field.getName()) != null - && ! (field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue; - ScriptExpression script = field.getIndexingScript(); - Set attributeNames = new HashSet<>(); - new MyVisitor(attributeNames).visit(script); - field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); - warn(schema, field, "Changed to attribute because numerical indexes (field has type " + - field.getDataType().getName() + ") is not currently supported." + - " Index-only settings may fail. Ignore this warning for streaming search."); - } - } - } - - private static class MyVisitor extends ExpressionVisitor { - - final Set attributeNames; - - public MyVisitor(Set attributeNames) { - this.attributeNames = attributeNames; - } - - @Override - protected void doVisit(Expression exp) { - if (exp instanceof AttributeExpression) { - attributeNames.add(((AttributeExpression)exp).getFieldName()); - } - } - } - - private static class MyConverter extends ExpressionConverter { - - final Set attributeNames; - - public MyConverter(Set attributeNames) { - this.attributeNames = attributeNames; - } - - @Override - protected boolean shouldConvert(Expression exp) { - return exp instanceof IndexExpression; - } - - @Override - protected Expression doConvert(Expression exp) { - String indexName = ((IndexExpression)exp).getFieldName(); - if (attributeNames.contains(indexName)) { - return null; - } - return new AttributeExpression(indexName); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java deleted file mode 100644 index afe004fd5f3..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Iterator; - -/** - * Expresses literal boosts in terms of extra indices with rank boost. - * One extra index named indexname_exact is added for each index having - * a fields with literal-boosts of zero or more (zero to support other - * rank profiles setting a literal boost). Complete boost values in to fields - * are translated to rank boosts to the implementation indices. - * These indices has no positional - * or phrase support and contains concatenated versions of each field value - * of complete-boosted fields indexed to indexname. A search for indexname - * will be rewritten to also search indexname_exaxt - * - * @author bratseth - */ -public class LiteralBoost extends Processor { - - public LiteralBoost(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - /** Adds extra search fields and indices to express literal boosts */ - @Override - public void process(boolean validate, boolean documentsOnly) { - checkRankModifierRankType(schema); - addLiteralBoostsToFields(schema); - reduceFieldLiteralBoosts(schema); - } - - /** Checks if literal boost is given using rank: , and set the actual literal boost accordingly. */ - private void checkRankModifierRankType(Schema schema) { - for (SDField field : schema.allConcreteFields()) { - if (field.getLiteralBoost() > -1) continue; // Let explicit value take precedence - if (field.getRanking().isLiteral()) - field.setLiteralBoost(100); - } - } - - /** - * Ensures there are field boosts for all literal boosts mentioned in rank profiles. - * This is required because boost indices will only be generated by looking - * at field boosts - */ - private void addLiteralBoostsToFields(Schema schema) { - Iterator i = matchingRankSettingsIterator(schema, RankProfile.RankSetting.Type.LITERALBOOST); - while (i.hasNext()) { - RankProfile.RankSetting setting = (RankProfile.RankSetting)i.next(); - SDField field = schema.getConcreteField(setting.getFieldName()); - if (field == null) continue; - if (field.getLiteralBoost() < 0) - field.setLiteralBoost(0); - } - } - - private void reduceFieldLiteralBoosts(Schema schema) { - for (SDField field : schema.allConcreteFields()) { - if (field.getLiteralBoost() < 0) continue; - reduceFieldLiteralBoost(field, schema); - } - } - - private void reduceFieldLiteralBoost(SDField field, Schema schema) { - SDField literalField = addField(schema, field, "literal", - "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }", - "literal-boost"); - literalField.setWeight(field.getWeight() + field.getLiteralBoost()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java deleted file mode 100644 index 14d75c6438e..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Takes the aliases set on field by parser and sets them on correct Index or Attribute - * - * @author vegardh - */ -public class MakeAliases extends Processor { - - public MakeAliases(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - List usedAliases = new ArrayList<>(); - for (SDField field : schema.allConcreteFields()) { - for (Map.Entry e : field.getAliasToName().entrySet()) { - String alias = e.getKey(); - String name = e.getValue(); - String errMsg = "For " + schema + ": alias '" + alias + "' "; - if (validate && schema.existsIndex(alias)) { - throw new IllegalArgumentException(errMsg + "is illegal since it is the name of an index."); - } - if (validate && schema.getAttribute(alias) != null) { - throw new IllegalArgumentException(errMsg + "is illegal since it is the name of an attribute."); - } - if (validate && usedAliases.contains(alias)) { - throw new IllegalArgumentException(errMsg + "specified more than once."); - } - usedAliases.add(alias); - - Index index = field.getIndex(name); - Attribute attribute = field.getAttributes().get(name); - if (index != null) { - index.addAlias(alias); // alias will be for index in this case, since it is the one used in a search - } else if (attribute != null && ! field.doesIndexing()) { - attribute.getAliases().add(alias); - } else { - index = new Index(name); - index.addAlias(alias); - field.addIndex(index); - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java deleted file mode 100644 index 49a56bafe2a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * All summary fields which are not attributes - * must currently be present in the default summary class, - * since the default summary class also defines the docsum.dat format. - * This processor adds any missing summaries to the default summary. - * When that is decoupled from the actual summaries returned, this - * processor can be removed. Note: the StreamingSummary also takes advantage of - * the fact that default is the superset. - * - * All other summary logic should work unchanged without this processing step - * except that IndexStructureValidator.validateSummaryFields must be changed to - * consider all summaries, not just the default, i.e change to - * if (search.getSummaryField(expr.getFieldName()) == null) - * - * This must be done after other summary processors. - * - * @author bratseth - */ -public class MakeDefaultSummaryTheSuperSet extends Processor { - - public MakeDefaultSummaryTheSuperSet(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - DocumentSummary defaultSummary= schema.getSummariesInThis().get("default"); - for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values() ) { - if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; - if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; - if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue; - if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue; - - defaultSummary.add(summaryField.clone()); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java deleted file mode 100644 index 6cb0a86b481..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashMap; -import java.util.Map; - -/** - * Warn on inconsistent match settings for any index - * - * @author vegardh - */ -public class MatchConsistency extends Processor { - - public MatchConsistency(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - Map types = new HashMap<>(); - for (SDField field : schema.allConcreteFields()) { - new MyVisitor(schema, field, types).visit(field.getIndexingScript()); - } - } - - private void checkMatching(Schema schema, SDField field, Map types, String indexTo) { - MatchType prevType = types.get(indexTo); - if (prevType == null) { - types.put(indexTo, field.getMatching().getType()); - } else if ( ! field.getMatching().getType().equals(prevType)) { - warn(schema, field, "The matching type for index '" + indexTo + "' (got " + field.getMatching().getType() + - ") is inconsistent with that given for the same index in a previous field (had " + - prevType + ")."); - } - } - - private class MyVisitor extends ExpressionVisitor { - - final Schema schema; - final SDField field; - final Map types; - - MyVisitor(Schema schema, SDField field, Map types) { - this.schema = schema; - this.field = field; - this.types = types; - } - - @Override - protected void doVisit(Expression exp) { - if (exp instanceof IndexExpression) { - checkMatching(schema, field, types, ((IndexExpression)exp).getFieldName()); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java deleted file mode 100644 index 4eae6b47833..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Validates the match phase settings for all registered rank profiles. - * - * @author geirst - */ -public class MatchPhaseSettingsValidator extends Processor { - - public MatchPhaseSettingsValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - if (documentsOnly) return; - - for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); - if (settings != null) { - validateMatchPhaseSettings(rankProfile, settings); - } - } - } - - private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { - String attributeName = settings.getAttribute(); - new AttributeValidator(schema.getName(), - rankProfile.name(), - schema.getAttribute(attributeName), attributeName).validate(); - } - - public static class AttributeValidator { - - private final String searchName; - private final String rankProfileName; - protected final Attribute attribute; - private final String attributeName; - - public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { - this.searchName = searchName; - this.rankProfileName = rankProfileName; - this.attribute = attribute; - this.attributeName = attributeName; - } - - public void validate() { - validateThatAttributeExists(); - validateThatAttributeIsSingleNumeric(); - validateThatAttributeIsFastSearch(); - } - - protected void validateThatAttributeExists() { - if (attribute == null) { - failValidation("does not exists"); - } - } - - protected void validateThatAttributeIsSingleNumeric() { - if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || - attribute.getType().equals(Attribute.Type.STRING) || - attribute.getType().equals(Attribute.Type.PREDICATE)) - { - failValidation("must be single value numeric, but it is '" + - attribute.getDataType().getName() + "'"); - } - } - - protected void validateThatAttributeIsFastSearch() { - if ( ! attribute.isFastSearch()) { - failValidation("must be fast-search, but it is not"); - } - } - - protected void failValidation(String what) { - throw new IllegalArgumentException(createMessagePrefix() + what); - } - - public String getValidationType() { return "match-phase"; } - - private String createMessagePrefix() { - return "In search definition '" + searchName + - "', rank-profile '" + rankProfileName + - "': " + getValidationType() + " attribute '" + attributeName + "' "; - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java deleted file mode 100644 index 299f28ca573..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField; - -/** - * Iterates all summary fields with 'matched-elements-only' and adjusts transform (if all struct-fields are attributes) - * and validates that the field type is supported. - * - * @author geirst - */ -public class MatchedElementsOnlyResolver extends Processor { - - public MatchedElementsOnlyResolver(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (var entry : schema.getSummaries().entrySet()) { - var summary = entry.getValue(); - for (var field : summary.getSummaryFields().values()) { - if (field.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER) { - processSummaryField(summary, field, validate); - } - } - } - } - - private void processSummaryField(DocumentSummary summary, SummaryField field, boolean validate) { - var sourceField = schema.getField(field.getSingleSource()); - if (sourceField != null) { - if (isSupportedComplexField(sourceField)) { - if (isComplexFieldWithOnlyStructFieldAttributes(sourceField)) { - field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - } - } else if (isSupportedMultiValueField(sourceField)) { - if (sourceField.doesAttributing()) { - field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - } - } else if (validate) { - fail(summary, field, "'matched-elements-only' is not supported for this field type. " + - "Supported field types are: array of primitive, weighted set of primitive, " + - "array of simple struct, map of primitive type to simple struct, " + - "and map of primitive type to primitive type"); - } - } - // else case is handled in SummaryFieldsMustHaveValidSource - } - - private boolean isSupportedMultiValueField(ImmutableSDField sourceField) { - var type = sourceField.getDataType(); - return (isArrayOfPrimitiveType(type) || isWeightedsetOfPrimitiveType(type)); - } - - private boolean isArrayOfPrimitiveType(DataType type) { - if (type instanceof ArrayDataType) { - var arrayType = (ArrayDataType) type; - return ComplexAttributeFieldUtils.isPrimitiveType(arrayType.getNestedType()); - } - return false; - } - - private boolean isWeightedsetOfPrimitiveType(DataType type) { - if (type instanceof WeightedSetDataType) { - var wsetType = (WeightedSetDataType) type; - return ComplexAttributeFieldUtils.isPrimitiveType(wsetType.getNestedType()); - } - return false; - } - - private void fail(DocumentSummary summary, SummaryField field, String msg) { - throw new IllegalArgumentException(formatError(schema, summary, field, msg)); - } - - private String formatError(Schema schema, DocumentSummary summary, SummaryField field, String msg) { - return "For " + schema + ", document summary '" + summary.getName() - + "', summary field '" + field.getName() + "': " + msg; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java deleted file mode 100644 index 39220b8cf8f..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.multifieldresolver.IndexCommandResolver; -import com.yahoo.searchdefinition.processing.multifieldresolver.RankTypeResolver; -import com.yahoo.searchdefinition.processing.multifieldresolver.StemmingResolver; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.List; -import java.util.Map; - -/** - * Ensures that there are no conflicting types or field settings - * in multifield indices, either by changing settings or by splitting - * conflicting fields in multiple ones with different settings. - * - * @author bratseth - */ -public class MultifieldIndexHarmonizer extends Processor { - - /** A map from index names to a List of fields going to that index */ - private Map> indexToFields=new java.util.HashMap<>(); - - public MultifieldIndexHarmonizer(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - populateIndexToFields(schema); - resolveAllConflicts(schema); - } - - private void populateIndexToFields(Schema schema) { - for (SDField field : schema.allConcreteFields() ) { - if ( ! field.doesIndexing()) continue; - addIndexField(field.getName(), field); - } - } - - private void addIndexField(String indexName,SDField field) { - List fields = indexToFields.get(indexName); - if (fields == null) { - fields = new java.util.ArrayList<>(); - indexToFields.put(indexName, fields); - } - fields.add(field); - } - - private void resolveAllConflicts(Schema schema) { - for (Map.Entry> entry : indexToFields.entrySet()) { - String indexName = entry.getKey(); - List fields = entry.getValue(); - if (fields.size() == 1) continue; // It takes two to make a conflict - resolveConflicts(indexName, fields, schema); - } - } - - /** - * Resolves all conflicts for one index - * - * @param indexName the name of the index in question - * @param fields all the fields indexed to this index - * @param schema the search definition having this - */ - private void resolveConflicts(String indexName, List fields, Schema schema) { - new StemmingResolver(indexName, fields, schema, deployLogger).resolve(); - new IndexCommandResolver(indexName, fields, schema, deployLogger).resolve(); - new RankTypeResolver(indexName, fields, schema, deployLogger).resolve(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java deleted file mode 100644 index 198f89caddb..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -public class MutableAttributes extends Processor { - - public MutableAttributes(Schema schema, DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) - { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) { - if (field.getAttributes().get(field.getName()).isMutable()) { - throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + schema.getDocument().getName() + - "' can not be marked mutable as it is inside the document clause."); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java deleted file mode 100644 index 2ba149925e9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.vespa.indexinglanguage.expressions.*; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * The implementation of "gram" matching - splitting the incoming text and the queries into - * n-grams for matching. This will also validate the gram settings. - * - * @author bratseth - */ -public class NGramMatch extends Processor { - - public static final int DEFAULT_GRAM_SIZE = 2; - - public NGramMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.getMatching().getType().equals(MatchType.GRAM)) - implementGramMatch(schema, field, validate); - else if (validate && field.getMatching().getGramSize() >= 0) - throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); - } - } - - private void implementGramMatch(Schema schema, SDField field, boolean validate) { - if (validate && field.doesAttributing() && ! field.doesIndexing()) - throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' in indexing"); - - int n = field.getMatching().getGramSize(); - if (n < 0) - n = DEFAULT_GRAM_SIZE; // not set - use default gram size - if (validate && n == 0) - throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); - field.getNormalizing().inferCodepoint(); - field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing - field.addQueryCommand("ngram " + n); - field.setIndexingScript((ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); - } - - private static class MyProvider extends TypedTransformProvider { - - final int ngram; - - MyProvider(Schema schema, int ngram) { - super(NGramExpression.class, schema); - this.ngram = ngram; - } - - @Override - protected boolean requiresTransform(Expression exp, DataType fieldType) { - return exp instanceof OutputExpression; - } - - @Override - protected Expression newTransform(DataType fieldType) { - Expression exp = new NGramExpression(null, ngram); - if (fieldType instanceof CollectionDataType) - exp = new ForEachExpression(exp); - return exp; - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelConfigGenerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelConfigGenerator.java deleted file mode 100644 index 70ce051bb21..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelConfigGenerator.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.vespa.model.ml.OnnxModelInfo; - -import java.util.Map; - -/** - * Processes ONNX ranking features of the form: - * - * onnx("files/model.onnx", "path/to/output:1") - * - * And generates an "onnx-model" configuration as if it was defined in the profile: - * - * onnx-model files_model_onnx { - * file: "files/model.onnx" - * } - * - * Inputs and outputs are resolved in OnnxModelTypeResolver, which must be - * processed after this. - * - * @author lesters - */ -public class OnnxModelConfigGenerator extends Processor { - - public OnnxModelConfigGenerator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (documentsOnly) return; - for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - if (profile.getFirstPhaseRanking() != null) { - process(profile.getFirstPhaseRanking().getRoot(), profile); - } - if (profile.getSecondPhaseRanking() != null) { - process(profile.getSecondPhaseRanking().getRoot(), profile); - } - for (Map.Entry function : profile.getFunctions().entrySet()) { - process(function.getValue().function().getBody().getRoot(), profile); - } - for (ReferenceNode feature : profile.getSummaryFeatures()) { - process(feature, profile); - } - } - } - - private void process(ExpressionNode node, RankProfile profile) { - if (node instanceof ReferenceNode) { - process((ReferenceNode)node, profile); - } else if (node instanceof CompositeNode) { - for (ExpressionNode child : ((CompositeNode) node).children()) { - process(child, profile); - } - } - } - - private void process(ReferenceNode feature, RankProfile profile) { - if (feature.getName().equals("onnxModel") || feature.getName().equals("onnx")) { - if (feature.getArguments().size() > 0) { - if (feature.getArguments().expressions().get(0) instanceof ConstantNode) { - ConstantNode node = (ConstantNode) feature.getArguments().expressions().get(0); - String path = OnnxModelTransformer.stripQuotes(node.toString()); - String modelConfigName = OnnxModelTransformer.asValidIdentifier(path); - - // Only add the configuration if the model can actually be found. - if ( ! OnnxModelInfo.modelExists(path, schema.applicationPackage())) { - path = ApplicationPackage.MODELS_DIR.append(path).toString(); - if ( ! OnnxModelInfo.modelExists(path, schema.applicationPackage())) { - return; - } - } - - OnnxModel onnxModel = profile.onnxModels().get(modelConfigName); - if (onnxModel == null) - profile.add(new OnnxModel(modelConfigName, path)); - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelTypeResolver.java deleted file mode 100644 index 83e0c367292..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelTypeResolver.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.vespa.model.ml.OnnxModelInfo; - -/** - * Processes every "onnx-model" element in the schema. Associates model type - * information by retrieving from either the ONNX model file directly or from - * preprocessed information in ZK. Adds missing input and output mappings - * (assigning default names). - * - * Must be processed before RankingExpressingTypeResolver. - * - * @author lesters - */ -public class OnnxModelTypeResolver extends Processor { - - public OnnxModelTypeResolver(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (documentsOnly) return; - for (OnnxModel onnxModel : schema.declaredOnnxModels().values()) - onnxModel.setModelInfo(OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage())); - for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - for (OnnxModel onnxModel : profile.declaredOnnxModels().values()) - onnxModel.setModelInfo(OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage())); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java deleted file mode 100644 index 3175862e900..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.indexinglanguage.ExpressionOptimizer; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Run ExpressionOptimizer on all scripts, to get rid of expressions that have no effect. - */ -public class OptimizeIlscript extends Processor { - - public OptimizeIlscript(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - ScriptExpression script = field.getIndexingScript(); - if (script == null) continue; - - field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); - if ( ! field.getIndexingScript().toString().equals(script.toString())) { - info(schema, field, "Rewrote ilscript from:\n" + script.toString() + - "\nto\n" + field.getIndexingScript().toString()); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java deleted file mode 100644 index 2ca4abae2c4..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Optional; - -/** - * Validates the 'paged' attribute setting and throws if specified on unsupported types. - * - * @author geirst - */ -public class PagedAttributeValidator extends Processor { - - public PagedAttributeValidator(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (!validate) { - return; - } - for (var field : schema.allConcreteFields()) { - for (var attribute : field.getAttributes().values()) { - if (attribute.isPaged()) { - validatePagedSetting(field, attribute); - } - } - } - } - - private void validatePagedSetting(Field field, Attribute attribute) { - if (!isSupportedType(attribute)) { - fail(schema, field, "The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types"); - } - } - - private boolean isSupportedType(Attribute attribute) { - var type = attribute.getType(); - return (type != Attribute.Type.PREDICATE) && - (type != Attribute.Type.REFERENCE) && - (isSupportedTensorType(attribute.tensorType())); - } - - private boolean isSupportedTensorType(Optional tensorType) { - if (tensorType.isPresent()) { - return isDenseTensorType(tensorType.get()); - } - return true; - } - - private boolean isDenseTensorType(TensorType type) { - return type.dimensions().stream().allMatch(d -> d.isIndexed()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java deleted file mode 100644 index d7ae4c33fe1..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.LongFieldValue; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; -import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.ArrayList; -import java.util.List; - -/** - * Validates the predicate fields. - * - * @author Lester Solbakken - */ -public class PredicateProcessor extends Processor { - - public PredicateProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.getDataType() == DataType.PREDICATE) { - if (validate && field.doesIndexing()) { - fail(schema, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - } - if (field.doesAttributing()) { - Attribute attribute = field.getAttributes().get(field.getName()); - for (Index index : field.getIndices().values()) { - BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton(); - if (validate && (booleanDefinition == null || ! booleanDefinition.hasArity())) { - fail(schema, field, "Missing arity value in predicate field."); - } - if (validate && (booleanDefinition.getArity() < 2)) { - fail(schema, field, "Invalid arity value in predicate field, must be greater than 1."); - } - double threshold = booleanDefinition.getDensePostingListThreshold(); - if (validate && (threshold <= 0 || threshold > 1)) { - fail(schema, field, "Invalid dense-posting-list-threshold value in predicate field. " + - "Value must be in range (0..1]."); - } - - attribute.setArity(booleanDefinition.getArity()); - attribute.setLowerBound(booleanDefinition.getLowerBound()); - attribute.setUpperBound(booleanDefinition.getUpperBound()); - - attribute.setDensePostingListThreshold(threshold); - addPredicateOptimizationIlScript(field, booleanDefinition); - } - DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); - if (summary != null) { - summary.remove(attribute.getName()); - } - for (SummaryField summaryField : schema.getSummaryFields(field)) { - summaryField.setTransform(SummaryTransform.NONE); - } - } - } else if (validate && field.getDataType().getPrimitiveType() == DataType.PREDICATE) { - fail(schema, field, "Collections of predicates are not allowed."); - } else if (validate && field.getDataType() == DataType.RAW && field.doesIndexing()) { - fail(schema, field, "Indexing of RAW fields is not supported."); - } else if (validate) { - // if field is not a predicate, disallow predicate-related index parameters - for (Index index : field.getIndices().values()) { - if (index.getBooleanIndexDefiniton() != null) { - BooleanIndexDefinition def = index.getBooleanIndexDefiniton(); - if (def.hasArity()) { - fail(schema, field, "Arity parameter is used only for predicate type fields."); - } else if (def.hasLowerBound() || def.hasUpperBound()) { - fail(schema, field, "Parameters lower-bound and upper-bound are used only for predicate type fields."); - } else if (def.hasDensePostingListThreshold()) { - fail(schema, field, "Parameter dense-posting-list-threshold is used only for predicate type fields."); - } - } - } - } - } - } - - private void addPredicateOptimizationIlScript(SDField field, BooleanIndexDefinition booleanIndexDefiniton) { - Expression script = field.getIndexingScript(); - if (script == null) return; - - script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); - - ExpressionConverter converter = new PredicateOutputTransformer(schema); - field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script))); - } - - private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { - List expressions = new ArrayList<>(); - expressions.add(new SetValueExpression(new IntegerFieldValue(options.getArity()))); - expressions.add(new SetVarExpression("arity")); - if (options.hasLowerBound()) { - expressions.add(new SetValueExpression(new LongFieldValue(options.getLowerBound()))); - expressions.add(new SetVarExpression("lower_bound")); - } - if (options.hasUpperBound()) { - expressions.add(new SetValueExpression(new LongFieldValue(options.getUpperBound()))); - expressions.add(new SetVarExpression("upper_bound")); - } - return new StatementExpression(expressions); - } - - private static class PredicateOutputTransformer extends TypedTransformProvider { - - PredicateOutputTransformer(Schema schema) { - super(OptimizePredicateExpression.class, schema); - } - - @Override - protected boolean requiresTransform(Expression exp, DataType fieldType) { - return exp instanceof OutputExpression && fieldType == DataType.PREDICATE; - } - - @Override - protected Expression newTransform(DataType fieldType) { - return new OptimizePredicateExpression(); - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java deleted file mode 100644 index c0ee0b5df53..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.deploy.TestProperties; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Set; - -/** - * Executor of processors. This defines the right order of processor execution. - * - * @author bratseth - * @author bjorncs - */ -public class Processing { - - private final ModelContext.Properties properties; - - public Processing() { this.properties = new TestProperties(); } - - public Processing(ModelContext.Properties properties) { this.properties = properties; } - - private Collection processors() { - return Arrays.asList( - SearchMustHaveDocument::new, - UrlFieldValidator::new, - BuiltInFieldSets::new, - ReservedDocumentNames::new, - IndexFieldNames::new, - IntegerIndex2Attribute::new, - MakeAliases::new, - UriHack::new, - LiteralBoost::new, - TagType::new, - ValidateFieldTypesDocumentsOnly::new, - IndexingInputs::new, - OptimizeIlscript::new, - ValidateFieldWithIndexSettingsCreatesIndex::new, - AttributesImplicitWord::new, - MutableAttributes::new, - CreatePositionZCurve::new, - DictionaryProcessor::new, - WordMatch::new, - ImportedFieldsResolver::new, - ImplicitSummaries::new, - ImplicitSummaryFields::new, - AdjustPositionSummaryFields::new, - SummaryConsistency::new, - SummaryNamesFieldCollisions::new, - SummaryFieldsMustHaveValidSource::new, - MatchedElementsOnlyResolver::new, - AddAttributeTransformToSummaryOfImportedFields::new, - MakeDefaultSummaryTheSuperSet::new, - Bolding::new, - AttributeProperties::new, - SetRankTypeEmptyOnFilters::new, - SummaryDynamicStructsArrays::new, - StringSettingsOnNonStringFields::new, - IndexingOutputs::new, - ExactMatch::new, - NGramMatch::new, - TextMatch::new, - MultifieldIndexHarmonizer::new, - FilterFieldNames::new, - MatchConsistency::new, - ValidateStructTypeInheritance::new, - ValidateFieldTypes::new, - SummaryDiskAccessValidator::new, - DisallowComplexMapAndWsetKeyTypes::new, - SortingSettings::new, - FieldSetSettings::new, - AddExtraFieldsToDocument::new, - PredicateProcessor::new, - MatchPhaseSettingsValidator::new, - DiversitySettingsValidator::new, - TensorFieldProcessor::new, - RankProfileTypeSettingsProcessor::new, - ReferenceFieldsProcessor::new, - FastAccessValidator::new, - ReservedFunctionNames::new, - OnnxModelConfigGenerator::new, - OnnxModelTypeResolver::new, - RankingExpressionTypeResolver::new, - BoolAttributeValidator::new, - PagedAttributeValidator::new, - // These should be last: - IndexingValidation::new, - IndexingValues::new); - } - - /** Processors of rank profiles only (those who tolerate and do something useful when the search field is null) */ - private Collection rankProfileProcessors() { - return Arrays.asList( - RankProfileTypeSettingsProcessor::new, - ReservedFunctionNames::new, - RankingExpressionTypeResolver::new); - } - - private void runProcessor(Processor processor, boolean validate, boolean documentsOnly) { - processor.process(validate, documentsOnly, properties); - } - - /** - * Runs all search processors on the given {@link Schema} object. These will modify the search object, possibly - * exchanging it with another, as well as its document types. - * - * @param schema the search to process - * @param deployLogger the log to log messages and warnings for application deployment to - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} - * @param queryProfiles the query profiles contained in the application this search is part of - * @param processorsToSkip a set of processor classes we should not invoke in this. Useful for testing. - */ - public void process(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, boolean validate, boolean documentsOnly, - Set> processorsToSkip) - { - Collection factories = processors(); - factories.stream() - .map(factory -> factory.create(schema, deployLogger, rankProfileRegistry, queryProfiles)) - .filter(processor -> ! processorsToSkip.contains(processor.getClass())) - .forEach(processor -> runProcessor(processor, validate, documentsOnly)); - } - - /** - * Runs rank profiles processors only. - * - * @param deployLogger the log to log messages and warnings for application deployment to - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} - * @param queryProfiles the query profiles contained in the application this search is part of - */ - public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { - Collection factories = rankProfileProcessors(); - factories.stream() - .map(factory -> factory.create(null, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(processor -> runProcessor(processor, validate, documentsOnly)); - } - - @FunctionalInterface - public interface ProcessorFactory { - Processor create(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java deleted file mode 100644 index 590681ffb86..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; - -/** - * Abstract superclass of all search definition processors. - * - * @author bratseth - */ -public abstract class Processor { - - protected final Schema schema; - protected final DeployLogger deployLogger; - protected final RankProfileRegistry rankProfileRegistry; - protected final QueryProfiles queryProfiles; - - /** - * Base constructor - * - * @param schema the search to process - * @param deployLogger Logger du use when logging deploy output. - * @param rankProfileRegistry Registry with all rank profiles, used for lookup and insertion. - * @param queryProfiles The query profiles contained in the application this search is part of. - */ - public Processor(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - this.schema = schema; - this.deployLogger = deployLogger; - this.rankProfileRegistry = rankProfileRegistry; - this.queryProfiles = queryProfiles; - } - - /** - * Processes the input search definition by modifying the input search and its documents, and returns the - * input search definition. - * - * @param validate true to throw exceptions on validation errors, false to make the best possible effort - * at completing processing without throwing an exception. - * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. - * @param documentsOnly true to skip processing (including validation, regardless of the validate setting) - * of aspects not relating to document definitions (e.g rank profiles) - */ - public abstract void process(boolean validate, boolean documentsOnly); - - /** - * As above, possibly with properties from a context. Override if needed. - **/ - public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) { - process(validate, documentsOnly); - } - - /** - * Convenience method for adding a no-strings-attached implementation field for a regular field - * - * @param schema the search definition in question - * @param field the field to add an implementation field for - * @param suffix the suffix of the added implementation field (without the underscore) - * @param indexing the indexing statement of the field - * @param queryCommand the query command of the original field, or null if none - * @return the implementation field which is added to the search - */ - protected SDField addField(Schema schema, SDField field, String suffix, String indexing, String queryCommand) { - SDField implementationField = schema.getConcreteField(field.getName() + "_" + suffix); - if (implementationField != null) { - deployLogger.logApplicationPackage(Level.WARNING, "Implementation field " + implementationField + " added twice"); - } else { - implementationField = new SDField(schema.getDocument(), field.getName() + "_" + suffix, DataType.STRING); - } - implementationField.setRankType(RankType.EMPTY); - implementationField.setStemming(Stemming.NONE); - implementationField.getNormalizing().inferCodepoint(); - implementationField.parseIndexingScript(indexing); - String indexName = field.getName(); - String implementationIndexName = indexName + "_" + suffix; - Index implementationIndex = new Index(implementationIndexName); - schema.addIndex(implementationIndex); - if (queryCommand != null) { - field.addQueryCommand(queryCommand); - } - schema.addExtraField(implementationField); - schema.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, implementationField.getName()); - return implementationField; - } - - /** - * Returns an iterator of all the rank settings with given type in all the rank profiles in this search - * definition. - */ - protected Iterator matchingRankSettingsIterator( - Schema schema, RankProfile.RankSetting.Type type) - { - List someRankSettings = new java.util.ArrayList<>(); - - for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) { - RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next(); - if (setting.getType().equals(type)) { - someRankSettings.add(setting); - } - } - } - return someRankSettings.iterator(); - } - - protected String formatError(String schemaName, String fieldName, String msg) { - return "For schema '" + schemaName + "', field '" + fieldName + "': " + msg; - } - - protected RuntimeException newProcessException(String schemaName, String fieldName, String msg) { - return new IllegalArgumentException(formatError(schemaName, fieldName, msg)); - } - - protected RuntimeException newProcessException(Schema schema, Field field, String msg) { - return newProcessException(schema.getName(), field.getName(), msg); - } - - public void fail(Schema schema, Field field, String msg) { - throw newProcessException(schema, field, msg); - } - - protected void warn(String schemaName, String fieldName, String message) { - String fullMsg = formatError(schemaName, fieldName, message); - deployLogger.logApplicationPackage(Level.WARNING, fullMsg); - } - - protected void warn(Schema schema, Field field, String message) { - warn(schema.getName(), field.getName(), message); - } - - protected void info(String schemaName, String fieldName, String message) { - String fullMsg = formatError(schemaName, fieldName, message); - deployLogger.logApplicationPackage(Level.INFO, fullMsg); - } - - protected void info(Schema schema, Field field, String message) { - info(schema.getName(), field.getName(), message); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java deleted file mode 100644 index fdf1dc187cf..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.MapEvaluationTypeContext; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; - -/** - * Resolves and assigns types to all functions in a ranking expression, and - * validates the types of all ranking expressions under a search instance: - * Some operators constrain the types of inputs, and first-and second-phase expressions - * must return scalar values. - * - * In addition, the existence of all referred attribute, query and constant - * features is ensured. - * - * @author bratseth - */ -public class RankingExpressionTypeResolver extends Processor { - - private final QueryProfileRegistry queryProfiles; - - public RankingExpressionTypeResolver(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - this.queryProfiles = queryProfiles.getRegistry(); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (documentsOnly) return; - - Set warnedAbout = new HashSet<>(); - for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - try { - resolveTypesIn(profile, validate, warnedAbout); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("In " + (schema != null ? schema + ", " : "") + profile, e); - } - } - } - - /** - * Resolves the types of all functions in the given profile - * - * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types - */ - private void resolveTypesIn(RankProfile profile, boolean validate, Set warnedAbout) { - MapEvaluationTypeContext context = profile.typeContext(queryProfiles); - for (Map.Entry function : profile.getFunctions().entrySet()) { - ExpressionFunction expressionFunction = function.getValue().function(); - if (hasUntypedArguments(expressionFunction)) continue; - - // Add any missing inputs for type resolution - for (String argument : expressionFunction.arguments()) { - Reference ref = Reference.fromIdentifier(argument); - if (context.getType(ref).equals(TensorType.empty)) { - context.setType(ref, expressionFunction.argumentTypes().get(argument)); - } - } - context.forgetResolvedTypes(); - - TensorType type = resolveType(expressionFunction.getBody(), "function '" + function.getKey() + "'", context); - function.getValue().setReturnType(type); - } - - if (validate) { - profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); - ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); - ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - if ( ( context.tensorsAreUsed() || profile.isStrict()) - && ! context.queryFeaturesNotDeclared().isEmpty() - && ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { - if (profile.isStrict()) - throw new IllegalArgumentException(profile + " is strict but is missing a query profile type " + - "declaration of features " + context.queryFeaturesNotDeclared()); - else - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + profile + - " are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + - context.queryFeaturesNotDeclared()); - warnedAbout.addAll(context.queryFeaturesNotDeclared()); - } - } - } - - private boolean hasUntypedArguments(ExpressionFunction function) { - return function.arguments().size() > function.argumentTypes().size(); - } - - private TensorType resolveType(RankingExpression expression, String expressionDescription, TypeContext context) { - if (expression == null) return null; - return resolveType(expression.getRoot(), expressionDescription, context); - } - - private TensorType resolveType(ExpressionNode expression, String expressionDescription, TypeContext context) { - TensorType type; - try { - type = expression.type(context); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("The " + expressionDescription + " is invalid", e); - } - if (type == null) // Not expected to happen - throw new IllegalStateException("Could not determine the type produced by " + expressionDescription); - return type; - } - - private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) { - if (expression == null) return; - TensorType type = resolveType(expression, expressionDescription, context); - if ( ! type.equals(TensorType.empty)) - throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " + - "(a tensor with no dimensions), but produces " + type); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java deleted file mode 100644 index 09fdf9c65f6..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Class that processes reference fields and removes attribute aspect of such fields from summary. - * - * A document summary for a reference field should always be fetched from the document instance in back-end - * as the attribute vector does not store the original document id string. - * - * @author geirst - */ -public class ReferenceFieldsProcessor extends Processor { - - public ReferenceFieldsProcessor(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - clearSummaryAttributeAspectForConcreteFields(); - clearSummaryAttributeAspectForExplicitSummaryFields(); - } - - private void clearSummaryAttributeAspectForExplicitSummaryFields() { - for (DocumentSummary docSum : schema.getSummaries().values()) { - docSum.getSummaryFields().values().stream() - .filter(summaryField -> summaryField.getDataType() instanceof NewDocumentReferenceDataType) - .forEach(summaryField -> summaryField.setTransform(SummaryTransform.NONE)); - } - } - - private void clearSummaryAttributeAspectForConcreteFields() { - for (SDField field : schema.allConcreteFields()) { - if (field.getDataType() instanceof NewDocumentReferenceDataType) { - removeFromAttributePrefetchSummaryClass(field); - clearSummaryTransformOnSummaryFields(field); - } - } - } - - private void removeFromAttributePrefetchSummaryClass(SDField field) { - DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); - if (summary != null) { - summary.remove(field.getName()); - } - } - - private void clearSummaryTransformOnSummaryFields(SDField field) { - schema.getSummaryFields(field).forEach(summaryField -> summaryField.setTransform(SummaryTransform.NONE)); - } - -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java deleted file mode 100644 index 4a39a52a005..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashSet; -import java.util.Set; - -/** - * @author Simon Thoresen Hult - */ -public class ReservedDocumentNames extends Processor { - - private static final Set RESERVED_NAMES = new HashSet<>(); - - static { - for (SDDocumentType dataType : SDDocumentType.VESPA_DOCUMENT.getTypes()) { - RESERVED_NAMES.add(dataType.getName()); - } - } - - public ReservedDocumentNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - String docName = schema.getDocument().getName(); - if (RESERVED_NAMES.contains(docName)) - throw new IllegalArgumentException("For " + schema + ": Document name '" + docName + "' is reserved."); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java deleted file mode 100644 index f4f920d9ec8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.google.common.collect.ImmutableSet; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Set; -import java.util.logging.Level; - -/** - * Issues a warning if some function has a reserved name. This is not necessarily - * an error, as a rank profile function can shadow a built-in function. - * - * @author lesters - */ -public class ReservedFunctionNames extends Processor { - - private static Set reservedNames = getReservedNames(); - - public ReservedFunctionNames(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - if (documentsOnly) return; - - for (RankProfile rp : rankProfileRegistry.all()) { - for (String functionName : rp.getFunctions().keySet()) { - if (reservedNames.contains(functionName)) { - deployLogger.logApplicationPackage(Level.WARNING, "Function '" + functionName + "' " + - "in rank profile '" + rp.name() + "' " + - "has a reserved name. This might mean that the function shadows " + - "the built-in function with the same name." - ); - } - } - } - } - - private static ImmutableSet getReservedNames() { - ImmutableSet.Builder names = ImmutableSet.builder(); - for (String token : RankingExpressionParserConstants.tokenImage) { - String tokenWithoutQuotes = token.substring(1, token.length()-1); - names.add(tokenWithoutQuotes); - } - return names.build(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java deleted file mode 100644 index 68717f1f06b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * A search must have a document definition of the same name inside of it, otherwise crashes may occur as late as - * during feeding - * - * @author Vegard Havdal - */ -public class SearchMustHaveDocument extends Processor { - - public SearchMustHaveDocument(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - if (schema.getDocument() == null) - throw new IllegalArgumentException("For " + schema + - ": A search specification must have an equally named document inside of it."); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java deleted file mode 100644 index 6631258813a..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * All rank: filter fields should have rank type empty. - * - * @author bratseth - */ -public class SetRankTypeEmptyOnFilters extends Processor { - - public SetRankTypeEmptyOnFilters(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.getRanking().isFilter()) { - field.setRankType(RankType.EMPTY); - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java deleted file mode 100644 index 33e9634323b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Validate conflicting settings for sorting - * - * @author Vegard Havdal - */ -public class SortingSettings extends Processor { - - public SortingSettings(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : schema.allConcreteFields()) { - for (Attribute attribute : field.getAttributes().values()) { - Sorting sorting = attribute.getSorting(); - if (sorting.getFunction() != Sorting.Function.UCA) { - if (sorting.getStrength()!=null && sorting.getStrength() != Sorting.Strength.PRIMARY) { - warn(schema, field, "Sort strength only works for sort function 'uca'."); - } - if (sorting.getLocale() != null && ! "".equals(sorting.getLocale())) { - warn(schema, field, "Sort locale only works for sort function 'uca'."); - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java deleted file mode 100644 index 8c6b07515aa..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.NumericDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -public class StringSettingsOnNonStringFields extends Processor { - - public StringSettingsOnNonStringFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : schema.allConcreteFields()) { - if ( ! doCheck(field)) continue; - if (field.getMatching().isTypeUserSet()) { - warn(schema, field, "Matching type " + field.getMatching().getType() + - " is only allowed for string fields."); - } - if (field.getRanking().isLiteral()) { - warn(schema, field, "Rank type literal only applies to string fields"); - } - } - } - - private boolean doCheck(SDField field) { - if (field.getDataType() instanceof NumericDataType) return true; - if (field.getDataType() instanceof CollectionDataType) { - if (((CollectionDataType)field.getDataType()).getNestedType() instanceof NumericDataType) { - return true; - } - } - return false; - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java deleted file mode 100644 index e2e0cf94bb8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; - -/** - * Ensure that summary field transforms for fields having the same name - * are consistent across summary classes - * - * @author bratseth - */ -public class SummaryConsistency extends Processor { - - public SummaryConsistency(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (DocumentSummary summary : schema.getSummaries().values()) { - if (summary.getName().equals("default")) continue; - - for (SummaryField summaryField : summary.getSummaryFields().values()) { - assertConsistency(summaryField, schema, validate); - makeAttributeTransformIfAppropriate(summaryField, schema); - makeAttributeCombinerTransformIfAppropriate(summaryField, schema); - } - } - } - - private void assertConsistency(SummaryField summaryField, Schema schema, boolean validate) { - // Compare to default: - SummaryField existingDefault = schema.getSummariesInThis().get("default").getSummaryField(summaryField.getName()); - if (existingDefault != null) { - if (validate) - assertConsistentTypes(existingDefault, summaryField); - makeConsistentWithDefaultOrThrow(existingDefault, summaryField); - } - else { - // If no default, compare to whichever definition of the field - SummaryField existing = schema.getExplicitSummaryField(summaryField.getName()); - if (existing == null) return; - if (validate) - assertConsistentTypes(existing, summaryField); - makeConsistentOrThrow(existing, summaryField, schema); - } - } - - /** If the source is an attribute, make this use the attribute transform */ - private void makeAttributeTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() != SummaryTransform.NONE) return; - Attribute attribute = schema.getAttribute(summaryField.getSingleSource()); - if (attribute == null) return; - summaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - - /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ - private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() == SummaryTransform.NONE) { - String source_field_name = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(source_field_name); - if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { - summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); - } - } - } - - private void assertConsistentTypes(SummaryField existing, SummaryField seen) { - if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && - ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) - return; // Disregard create-if-nonexistent and create-if-zero distinction - if ( ! compatibleTypes(seen.getDataType(), existing.getDataType())) - throw new IllegalArgumentException(existing.toLocateString() + " is inconsistent with " + - seen.toLocateString() + ": All declarations of the same summary field must have the same type"); - } - - private boolean compatibleTypes(DataType summaryType, DataType existingType) { - if (summaryType instanceof TensorDataType && existingType instanceof TensorDataType) { - return summaryType.isAssignableFrom(existingType); // TODO: Just do this for all types - } - return summaryType.equals(existingType); - } - - private void makeConsistentOrThrow(SummaryField field1, SummaryField field2, Schema schema) { - if (field2.getTransform() == SummaryTransform.ATTRIBUTE && field1.getTransform() == SummaryTransform.NONE) { - Attribute attribute = schema.getAttribute(field1.getName()); - if (attribute != null) { - field1.setTransform(SummaryTransform.ATTRIBUTE); - } - } - - if (field2.getTransform().equals(SummaryTransform.NONE)) { - field2.setTransform(field1.getTransform()); - } - else { // New field sets an explicit transform - must be the same - assertEqualTransform(field1,field2); - } - } - - private void makeConsistentWithDefaultOrThrow(SummaryField defaultField, SummaryField newField) { - if (newField.getTransform().equals(SummaryTransform.NONE)) { - newField.setTransform(defaultField.getTransform()); - } - else { // New field sets an explicit transform - must be the same - assertEqualTransform(defaultField,newField); - } - } - - private void assertEqualTransform(SummaryField field1, SummaryField field2) { - if ( ! field2.getTransform().equals(field1.getTransform())) { - throw new IllegalArgumentException("Conflicting summary transforms. " + field2 + " is already defined as " + - field1 + ". A field with the same name " + - "can not have different transforms in different summary classes"); - } - } - - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java deleted file mode 100644 index 0741b0fd6f1..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SummaryClass; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Optional; -import java.util.logging.Level; - -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; - -/** - * Emits a warning for summaries which accesses disk. - * - * @author bratseth - */ -public class SummaryDiskAccessValidator extends Processor { - - public SummaryDiskAccessValidator(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - if (documentsOnly) return; - - for (DocumentSummary summary : schema.getSummaries().values()) { - for (SummaryField summaryField : summary.getSummaryFields().values()) { - for (SummaryField.Source source : summaryField.getSources()) { - ImmutableSDField field = schema.getField(source.getName()); - if (field == null) - field = findFieldProducingSummaryField(source.getName(), schema).orElse(null); - if (field == null && ! source.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) - throw new IllegalArgumentException(summaryField + " in " + summary + " references " + - source + ", but this field does not exist"); - if ( ! isInMemory(field, summaryField) && ! summary.isFromDisk()) { - deployLogger.logApplicationPackage(Level.WARNING, summaryField + " in " + summary + " references " + - source + ", which is not an attribute: Using this " + - "summary will cause disk accesses. " + - "Set 'from-disk' on this summary class to silence this warning."); - } - } - } - } - } - - private boolean isInMemory(ImmutableSDField field, SummaryField summaryField) { - if (field == null) return false; // For DOCUMENT_ID_FIELD, which may be implicit, but is then not in memory - if (isComplexFieldWithOnlyStructFieldAttributes(field) && - (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER || - summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER)) { - return true; - } - return field.doesAttributing(); - } - - private Optional findFieldProducingSummaryField(String name, Schema schema) { - return schema.allFields().filter(field -> field.getSummaryFields().get(name) != null).findAny(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java deleted file mode 100644 index c350c6800ed..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.*; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Fail if: - * An SD field explicitly says summary:dynamic , but the field is wset, array or struct. - * If there is an explicitly defined summary class, saying dynamic in one of its summary - * fields is always legal. - * - * @author Vegard Havdal - */ -public class SummaryDynamicStructsArrays extends Processor { - - public SummaryDynamicStructsArrays(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : schema.allConcreteFields()) { - DataType type = field.getDataType(); - if (type instanceof ArrayDataType || type instanceof WeightedSetDataType || type instanceof StructDataType) { - for (SummaryField sField : field.getSummaryFields().values()) { - if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) { - throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " + - "for fields of type struct, array or weighted set. Use an " + - "explicit summary class with explicit summary fields sourcing" + - " from the array/struct/weighted set."); - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java deleted file mode 100644 index c6f5b35aaa8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SummaryClass; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Verifies that the source fields actually refers to a valid field. - * - * @author baldersheim - */ -public class SummaryFieldsMustHaveValidSource extends Processor { - - SummaryFieldsMustHaveValidSource(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (DocumentSummary summary : schema.getSummaries().values()) { - for (SummaryField summaryField : summary.getSummaryFields().values()) { - if (summaryField.getSources().isEmpty()) { - if ((summaryField.getTransform() != SummaryTransform.RANKFEATURES) && - (summaryField.getTransform() != SummaryTransform.SUMMARYFEATURES)) - { - verifySource(summaryField.getName(), summaryField, summary); - } - } else if (summaryField.getSourceCount() == 1) { - verifySource(summaryField.getSingleSource(), summaryField, summary); - } else { - for (SummaryField.Source source : summaryField.getSources()) { - if ( ! source.getName().equals(summaryField.getName()) ) { - verifySource(source.getName(), summaryField, summary); - } - } - } - } - } - - } - - private boolean isValid(String source, SummaryField summaryField, DocumentSummary summary) { - return isDocumentField(source) || - (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) || - (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) || - (SummaryClass.DOCUMENT_ID_FIELD.equals(source)); - } - - private void verifySource(String source, SummaryField summaryField, DocumentSummary summary) { - if ( ! isValid(source, summaryField, summary) ) { - throw new IllegalArgumentException("For " + schema + ", summary class '" + - summary.getName() + "'," + " summary field '" + summaryField.getName() + - "': there is no valid source '" + source + "'."); - } - } - - private static boolean isNotInThisSummaryClass(DocumentSummary summary, String name) { - return summary.getSummaryField(name) == null; - } - - private static boolean isInThisSummaryClass(DocumentSummary summary, String name) { - return summary.getSummaryField(name) != null; - } - - private boolean isDocumentField(String name) { - return schema.getField(name) != null; - } - - private boolean isSummaryField(String name) { - return schema.getSummaryField(name) != null; - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java deleted file mode 100644 index 2d0afe88281..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import java.util.HashMap; -import java.util.Map; - -import com.yahoo.collections.Pair; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryField.Source; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Verifies that equally named summary fields in different summary classes don't use different fields for source. - * The summarymap config doesn't model this. - * - * @author Vegard Havdal - */ -public class SummaryNamesFieldCollisions extends Processor { - - public SummaryNamesFieldCollisions(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - Map> fieldToClassAndSource = new HashMap<>(); - for (DocumentSummary summary : schema.getSummaries().values()) { - if ("default".equals(summary.getName())) continue; - for (SummaryField summaryField : summary.getSummaryFields().values()) { - if (summaryField.isImplicit()) continue; - Pair prevClassAndSource = fieldToClassAndSource.get(summaryField.getName()); - for (Source source : summaryField.getSources()) { - if (prevClassAndSource!=null) { - String prevClass = prevClassAndSource.getFirst(); - String prevSource = prevClassAndSource.getSecond(); - if ( ! prevClass.equals(summary.getName())) { - if ( ! prevSource.equals(source.getName())) { - throw new IllegalArgumentException("For " + schema + - ", summary class '" + summary.getName() + "'," + - " summary field '" + summaryField.getName() + "':" + - " Can not use source '" + source.getName() + - "' for this summary field, an equally named field in summary class '" + - prevClass + "' uses a different source: '" + prevSource + "'."); - } - } - } else { - fieldToClassAndSource.put(summaryField.getName(), new Pair<>(summary.getName(), source.getName())); - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java deleted file mode 100644 index 357575660ea..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.*; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * The implementation of the tag datatype - * - * @author bratseth - */ -public class TagType extends Processor { - - public TagType(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) - implementTagType(field); - } - } - - private void implementTagType(SDField field) { - field.setDataType(DataType.getWeightedSet(DataType.STRING, true, true)); - // Don't set matching and ranking if this field is not attribute nor index - if (!field.doesIndexing() && !field.doesAttributing()) return; - Matching m = field.getMatching(); - if ( ! m.isTypeUserSet()) - m.setType(MatchType.WORD); - if (field.getRankType() == null || field.getRankType() == RankType.DEFAULT) - field.setRankType((RankType.TAGS)); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java deleted file mode 100644 index a253076f320..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Class that processes and validates tensor fields. - * - * @author geirst - */ -public class TensorFieldProcessor extends Processor { - - public TensorFieldProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (var field : schema.allConcreteFields()) { - if ( field.getDataType() instanceof TensorDataType ) { - if (validate) { - validateIndexingScripsForTensorField(field); - validateAttributeSettingForTensorField(field); - validateHnswIndexParametersRequiresIndexing(field); - } - processIndexSettingsForTensorField(field, validate); - } - else if (field.getDataType() instanceof CollectionDataType){ - if (validate) { - validateDataTypeForCollectionField(field); - } - } - } - } - - private void validateIndexingScripsForTensorField(SDField field) { - if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { - fail(schema, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + - "Currently, only tensors with 1 indexed dimension supports that."); - } - } - - private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { - var type = ((TensorDataType)field.getDataType()).getTensorType(); - // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). - if ((type.dimensions().size() == 1) && - type.dimensions().get(0).isIndexed()) { - return true; - } - return false; - } - - private boolean isTensorTypeThatSupportsDirectStore(ImmutableSDField field) { - var type = ((TensorDataType)field.getDataType()).getTensorType(); - // Tensors with at least one mapped/sparse dimensions can be "direct" - // (currenty triggered by fast-search flag) - for (var dim : type.dimensions()) { - if (dim.isMapped()) { - return true; - } - } - return false; - } - - private String tensorTypeToString(ImmutableSDField field) { - return ((TensorDataType)field.getDataType()).getTensorType().toString(); - } - - private void validateAttributeSettingForTensorField(SDField field) { - if (field.doesAttributing()) { - var attribute = field.getAttributes().get(field.getName()); - if (attribute != null && attribute.isFastSearch()) { - if (! isTensorTypeThatSupportsDirectStore(field)) { - fail(schema, field, "An attribute of type 'tensor' cannot be 'fast-search'."); - } - } - } - } - - private void validateHnswIndexParametersRequiresIndexing(SDField field) { - var index = field.getIndex(field.getName()); - if (index != null && index.getHnswIndexParams().isPresent() && !field.doesIndexing()) { - fail(schema, field, "A tensor that specifies hnsw index parameters must also specify 'index' in 'indexing'"); - } - } - - private void processIndexSettingsForTensorField(SDField field, boolean validate) { - if (!field.doesIndexing()) { - return; - } - if (isTensorTypeThatSupportsHnswIndex(field)) { - if (validate && !field.doesAttributing()) { - fail(schema, field, "A tensor that has an index must also be an attribute."); - } - var index = field.getIndex(field.getName()); - // TODO: Calculate default params based on tensor dimension size - var params = new HnswIndexParams(); - if (index != null) { - params = params.overrideFrom(index.getHnswIndexParams()); - } - field.getAttribute().setHnswIndexParams(params); - } - } - - private void validateDataTypeForCollectionField(SDField field) { - if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) - fail(schema, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java deleted file mode 100644 index c2b41edf454..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; -import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; -import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; -import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; -import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Set; -import java.util.TreeSet; - -/** - * @author Simon Thoresen Hult - */ -public class TextMatch extends Processor { - - public TextMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.getMatching().getType() != MatchType.TEXT) continue; - - ScriptExpression script = field.getIndexingScript(); - if (script == null) continue; - - DataType fieldType = field.getDataType(); - if (fieldType instanceof CollectionDataType) { - fieldType = ((CollectionDataType)fieldType).getNestedType(); - } - if (fieldType != DataType.STRING) continue; - - Set dynamicSummary = new TreeSet<>(); - Set staticSummary = new TreeSet<>(); - new IndexingOutputs(schema, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(schema, - field, - dynamicSummary, - staticSummary); - MyVisitor visitor = new MyVisitor(dynamicSummary); - visitor.visit(script); - if ( ! visitor.requiresTokenize) continue; - - ExpressionConverter converter = new MyStringTokenizer(schema, findAnnotatorConfig(schema, field)); - field.setIndexingScript((ScriptExpression)converter.convert(script)); - } - } - - private AnnotatorConfig findAnnotatorConfig(Schema schema, SDField field) { - AnnotatorConfig ret = new AnnotatorConfig(); - Stemming activeStemming = field.getStemming(); - if (activeStemming == null) { - activeStemming = schema.getStemming(); - } - ret.setStemMode(activeStemming.toStemMode()); - ret.setRemoveAccents(field.getNormalizing().doRemoveAccents()); - if ((field.getMatching() != null) && (field.getMatching().maxLength() != null)) { - ret.setMaxTokenLength(field.getMatching().maxLength()); - } - return ret; - } - - private static class MyVisitor extends ExpressionVisitor { - - final Set dynamicSummaryFields; - boolean requiresTokenize = false; - - MyVisitor(Set dynamicSummaryFields) { - this.dynamicSummaryFields = dynamicSummaryFields; - } - - @Override - protected void doVisit(Expression exp) { - if (exp instanceof IndexExpression) { - requiresTokenize = true; - } - if (exp instanceof SummaryExpression && - dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName())) - { - requiresTokenize = true; - } - } - - } - - private static class MyStringTokenizer extends TypedTransformProvider { - - final AnnotatorConfig annotatorCfg; - - MyStringTokenizer(Schema schema, AnnotatorConfig annotatorCfg) { - super(TokenizeExpression.class, schema); - this.annotatorCfg = annotatorCfg; - } - - @Override - protected boolean requiresTransform(Expression exp, DataType fieldType) { - return exp instanceof OutputExpression; - } - - @Override - protected Expression newTransform(DataType fieldType) { - Expression exp = new TokenizeExpression(null, annotatorCfg); - if (fieldType instanceof CollectionDataType) { - exp = new ForEachExpression(exp); - } - return exp; - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java deleted file mode 100644 index d6fdb901ab7..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; -import com.yahoo.vespa.indexinglanguage.expressions.*; - -/** - * @author Simon Thoresen Hult - */ -public abstract class TypedTransformProvider extends ValueTransformProvider { - - private final Schema schema; - private DataType fieldType; - - TypedTransformProvider(Class transformClass, Schema schema) { - super(transformClass); - this.schema = schema; - } - - @Override - protected final boolean requiresTransform(Expression exp) { - if (exp instanceof OutputExpression) { - String fieldName = ((OutputExpression)exp).getFieldName(); - if (exp instanceof AttributeExpression) { - Attribute attribute = schema.getAttribute(fieldName); - if (attribute == null) - throw new IllegalArgumentException("Attribute '" + fieldName + "' not found."); - fieldType = attribute.getDataType(); - } - else if (exp instanceof IndexExpression) { - Field field = schema.getConcreteField(fieldName); - if (field == null) - throw new IllegalArgumentException("Index field '" + fieldName + "' not found."); - fieldType = field.getDataType(); - } - else if (exp instanceof SummaryExpression) { - Field field = schema.getSummaryField(fieldName); - if (field == null) - throw new IllegalArgumentException("Summary field '" + fieldName + "' not found."); - fieldType = field.getDataType(); - } - else { - throw new UnsupportedOperationException(); - } - } - return requiresTransform(exp, fieldType); - } - - @Override - protected final Expression newTransform() { - return newTransform(fieldType); - } - - protected abstract boolean requiresTransform(Expression exp, DataType fieldType); - - protected abstract Expression newTransform(DataType fieldType); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java deleted file mode 100644 index 7397f9a289c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Arrays; -import java.util.List; - -/** - * @author baldersheim - */ -public class UriHack extends Processor { - - private static final List URL_SUFFIX = - Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); - - UriHack(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - if (field.doesIndexing()) { - DataType fieldType = field.getDataType(); - if (fieldType instanceof CollectionDataType) { - fieldType = ((CollectionDataType)fieldType).getNestedType(); - } - if (fieldType == DataType.URI) { - processField(schema, field); - } - } - } - } - - private void processField(Schema schema, SDField uriField) { - String uriName = uriField.getName(); - uriField.setStemming(Stemming.NONE); - DataType generatedType = DataType.STRING; - if (uriField.getDataType() instanceof ArrayDataType) { - generatedType = new ArrayDataType(DataType.STRING); - } - else if (uriField.getDataType() instanceof WeightedSetDataType) { - WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType(); - generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero()); - } - - for (String suffix : URL_SUFFIX) { - String partName = uriName + "." + suffix; - // I wonder if this is explicit in qrs or implicit in backend? - // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(schema.getDocument(), partName, generatedType); - partField.setIndexStructureField(uriField.doesIndexing()); - partField.setRankType(uriField.getRankType()); - partField.setStemming(Stemming.NONE); - partField.getNormalizing().inferLowercase(); - if (uriField.getIndex(suffix) != null) { - partField.addIndex(uriField.getIndex(suffix)); - } - schema.addExtraField(partField); - schema.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, partField.getName()); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java deleted file mode 100644 index 72f903f8365..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * @author bratseth - */ -public class UrlFieldValidator extends Processor { - - public UrlFieldValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (ImmutableSDField field : schema.allConcreteFields()) { - if ( ! field.getDataType().equals(DataType.URI)) continue; - - if (field.doesAttributing()) - throw new IllegalArgumentException("Error in " + field + " in " + schema + ": " + - "uri type fields cannot be attributes"); - } - - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java deleted file mode 100644 index 68bf2511a4b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashMap; -import java.util.Map; - -/** - * This Processor makes sure all fields with the same name have the same {@link DataType}. This check - * explicitly disregards whether a field is an index field, an attribute or a summary field. This is a requirement if we - * hope to move to a model where index fields, attributes and summary fields share a common field class. - * - * @author Simon Thoresen Hult - */ -public class ValidateFieldTypes extends Processor { - - public ValidateFieldTypes(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (!validate) return; - - String searchName = schema.getName(); - Map seenFields = new HashMap<>(); - verifySearchAndDocFields(searchName, seenFields); - verifySummaryFields(searchName, seenFields); - } - - final protected void verifySearchAndDocFields(String searchName, Map seenFields) { - schema.allFields().forEach(field -> { - checkFieldType(searchName, "index field", field.getName(), field.getDataType(), seenFields); - for (Map.Entry entry : field.getAttributes().entrySet()) { - checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), seenFields); - } - }); - - } - final protected void verifySummaryFields(String searchName, Map seenFields) { - for (DocumentSummary summary : schema.getSummaries().values()) { - for (SummaryField field : summary.getSummaryFields().values()) { - checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); - } - } - } - - private void checkFieldType(String searchName, String fieldDesc, String fieldName, DataType fieldType, - Map seenFields) { - DataType seenType = seenFields.get(fieldName); - if (seenType == null) { - seenFields.put(fieldName, fieldType); - } else if ( ! compatibleTypes(seenType, fieldType)) { - throw newProcessException(searchName, fieldName, "Incompatible types. Expected " + - seenType.getName() + " for " + fieldDesc + - " '" + fieldName + "', got " + fieldType.getName() + "."); - } - } - - private static boolean compatibleTypes(DataType seenType, DataType fieldType) { - // legacy tag field type compatibility; probably not needed any more (Oct 2016) - if ("tag".equals(seenType.getName())) { - return "tag".equals(fieldType.getName()) || "WeightedSet".equals(fieldType.getName()); - } - if ("tag".equals(fieldType.getName())) { - return "tag".equals(seenType.getName()) || "WeightedSet".equals(seenType.getName()); - } - if (seenType instanceof TensorDataType && fieldType instanceof TensorDataType) { - return fieldType.isAssignableFrom(seenType); // TODO: Just do this for all types - } - return seenType.equals(fieldType); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java deleted file mode 100644 index e8b218a80ad..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.HashMap; -import java.util.Map; - -public class ValidateFieldTypesDocumentsOnly extends ValidateFieldTypes { - public ValidateFieldTypesDocumentsOnly(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - String searchName = schema.getName(); - Map seenFields = new HashMap<>(); - verifySearchAndDocFields(searchName, seenFields); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java deleted file mode 100644 index 268a9f27b3c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.Ranking; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Check that fields with index settings actually creates an index or attribute - * - * @author bratseth - */ -public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { - - public ValidateFieldWithIndexSettingsCreatesIndex(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - Matching defaultMatching = new Matching(); - Ranking defaultRanking = new Ranking(); - for (SDField field : schema.allConcreteFields()) { - if (field.doesIndexing()) continue; - if (field.doesAttributing()) continue; - - if ( ! field.getRanking().equals(defaultRanking)) - fail(schema, field, - "Fields which are not creating an index or attribute can not contain rank settings."); - if ( ! field.getMatching().equals(defaultMatching)) - fail(schema, field, - "Fields which are not creating an index or attribute can not contain match settings."); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateStructTypeInheritance.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateStructTypeInheritance.java deleted file mode 100644 index d99832e3df6..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateStructTypeInheritance.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.document.StructDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.HashSet; -import java.util.Set; - -/** - * @author arnej - */ -public class ValidateStructTypeInheritance extends Processor { - - public ValidateStructTypeInheritance(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (!validate) return; - verifyNoRedeclarations(schema.getDocument()); - } - - void fail(Field field, String message) { - throw newProcessException(schema, field, message); - } - - void verifyNoRedeclarations(SDDocumentType docType) { - for (SDDocumentType type : docType.allTypes().values()) { - if (type.isStruct()) { - var inheritedTypes = new ArrayList(type.getInheritedTypes()); - for (int i = 0; i < inheritedTypes.size(); i++) { - SDDocumentType inherit = inheritedTypes.get(i); - for (var extra : inherit.getInheritedTypes()) { - if (! inheritedTypes.contains(extra)) { - inheritedTypes.add(extra); - } - } - } - if (inheritedTypes.isEmpty()) continue; - var seenFieldNames = new HashSet<>(); - for (var field : type.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { - if (seenFieldNames.contains(field.getName())) { - // cannot happen? - fail(field, "struct "+type.getName()+" has multiple fields with same name: "+field.getName()); - } - seenFieldNames.add(field.getName()); - } - for (SDDocumentType inherit : inheritedTypes) { - if (inherit.isStruct()) { - for (var field : inherit.getDocumentType().contentStruct().getFieldsThisTypeOnly()) { - if (seenFieldNames.contains(field.getName())) { - fail(field, "struct "+type.getName()+" cannot inherit from "+inherit.getName()+" and redeclare field "+field.getName()); - } - seenFieldNames.add(field.getName()); - } - } else { - fail(new Field("no field"), "struct cannot inherit from non-struct "+inherit.getName()+" class "+inherit.getClass()); - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java deleted file mode 100644 index 2a1afd616bc..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * The implementation of word matching - with word matching the field is assumed to contain a single "word" - some - * contiguous sequence of word and number characters - but without changing the data at the indexing side (as with text - * matching) to enforce this. Word matching is thus almost like exact matching on the indexing side (no action taken), - * and like text matching on the query side. This may be suitable for attributes, where people both expect the data to - * be left as in the input document, and trivially written queries to work by default. However, this may easily lead to - * data which cannot be matched at all as the indexing and query side does not agree. - * - * @author bratseth - */ -public class WordMatch extends Processor { - - public WordMatch(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - public void process(boolean validate, boolean documentsOnly) { - for (SDField field : schema.allConcreteFields()) { - processFieldRecursive(field); - } - } - - private void processFieldRecursive(SDField field) { - processField(field); - for (SDField structField : field.getStructFields()) { - processField(structField); - } - } - - private void processField(SDField field) { - if (!field.getMatching().getType().equals(MatchType.WORD)) { - return; - } - field.setStemming(Stemming.NONE); - field.getNormalizing().inferLowercase(); - field.addQueryCommand("word"); - } - - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java deleted file mode 100644 index 56926436545..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Level; - -/** - * Resolver-class for harmonizing index-commands in multifield indexes - */ -public class IndexCommandResolver extends MultiFieldResolver { - - /** Commands which don't have to be harmonized between fields */ - private static List ignoredCommands = new ArrayList<>(); - - /** Commands which must be harmonized between fields */ - private static List harmonizedCommands = new ArrayList<>(); - - static { - String[] ignore = { "complete-boost", "literal-boost", "highlight" }; - ignoredCommands.addAll(Arrays.asList(ignore)); - String[] harmonize = { "stemming", "normalizing" }; - harmonizedCommands.addAll(Arrays.asList(harmonize)); - } - - public IndexCommandResolver(String indexName, List fields, Schema schema, DeployLogger logger) { - super(indexName, fields, schema, logger); - } - - /** - * Check index-commands for each field, report and attempt to fix any - * inconsistencies - */ - public void resolve() { - for (SDField field : fields) { - for (String command : field.getQueryCommands()) { - if (!ignoredCommands.contains(command)) - checkCommand(command); - } - } - } - - private void checkCommand(String command) { - for (SDField field : fields) { - if (!field.hasQueryCommand(command)) { - if (harmonizedCommands.contains(command)) { - deployLogger.logApplicationPackage(Level.WARNING, command + " must be added to all fields going to the same index (" + indexName + ")" + - ", adding to field " + field.getName()); - field.addQueryCommand(command); - } else { - deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() + - "\' doesn't contain command \'" + command+"\'"); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java deleted file mode 100644 index 80bb4faeaa5..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; -import java.util.List; - -/** - * Abstract superclass of all multifield conflict resolvers - */ -public abstract class MultiFieldResolver { - - protected String indexName; - protected List fields; - protected Schema schema; - - protected DeployLogger deployLogger; - - public MultiFieldResolver(String indexName, List fields, Schema schema, DeployLogger logger) { - this.indexName = indexName; - this.fields = fields; - this.schema = schema; - this.deployLogger = logger; - } - - /** - * Checks the list of fields for specific conflicts, and reports and/or - * attempts to correct them - */ - public abstract void resolve(); - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java deleted file mode 100644 index fb7e67f2aab..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.FieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.search.query.profile.types.TensorFieldType; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.processing.Processor; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Map; -import java.util.Optional; - -/** - * This processes a schema and adds input type settings on all rank profiles. - * - * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. - * - * @author geirst - */ -public class RankProfileTypeSettingsProcessor extends Processor { - - public RankProfileTypeSettingsProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if (documentsOnly) return; - - processAttributeFields(); - processImportedFields(); - processQueryProfileTypes(); - } - - private void processAttributeFields() { - if (schema == null) return; // we're processing global profiles - for (ImmutableSDField field : schema.allConcreteFields()) { - Attribute attribute = field.getAttributes().get(field.getName()); - if (attribute != null && attribute.tensorType().isPresent()) { - addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString()); - } - } - } - - private void processImportedFields() { - if (schema == null) return; // we're processing global profiles - Optional importedFields = schema.importedFields(); - if (importedFields.isPresent()) { - importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field)); - } - } - - private void processImportedField(ImportedField field) { - ImmutableSDField targetField = field.targetField(); - Attribute attribute = targetField.getAttributes().get(targetField.getName()); - if (attribute != null && attribute.tensorType().isPresent()) { - addAttributeTypeToRankProfiles(field.fieldName(), attribute.tensorType().get().toString()); - } - } - - private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) { - for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - profile.addAttributeType(attributeName, attributeType); - } - } - - private void processQueryProfileTypes() { - for (QueryProfileType queryProfileType : queryProfiles.getRegistry().getTypeRegistry().allComponents()) { - for (Map.Entry fieldDescEntry : queryProfileType.fields().entrySet()) { - processFieldDescription(fieldDescEntry.getValue()); - } - } - } - - private void processFieldDescription(FieldDescription fieldDescription) { - FieldType fieldType = fieldDescription.getType(); - if (fieldType instanceof TensorFieldType) { - TensorFieldType tensorFieldType = (TensorFieldType)fieldType; - Optional reference = Reference.simple(fieldDescription.getName()); - if (reference.isPresent() && FeatureNames.isQueryFeature(reference.get())) - addQueryFeatureTypeToRankProfiles(reference.get(), tensorFieldType.asTensorType()); - } - } - - private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) { - for (RankProfile profile : rankProfileRegistry.all()) { - if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence - profile.addInput(queryFeature, - new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty())); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java deleted file mode 100644 index 231a97f30ec..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; - -import java.util.List; -import java.util.logging.Level; - -/** - * Checks if fields have defined different rank types for the same - * index (typically in an index-to statement), and if they have - * output a warning and use the first ranktype. - * - * @author hmusum - */ -public class RankTypeResolver extends MultiFieldResolver { - - public RankTypeResolver(String indexName, List fields, Schema schema, DeployLogger logger) { - super(indexName, fields, schema, logger); - } - - public void resolve() { - RankType rankType = null; - if (fields.size() > 0) { - boolean first = true; - for (SDField field : fields) { - if (first) { - rankType = fields.get(0).getRankType(); - first = false; - } else if (!field.getRankType().equals(rankType)) { - deployLogger.logApplicationPackage(Level.WARNING, "In field '" + field.getName() + "' " + - field.getRankType() + " for index '" + indexName + - "' conflicts with " + rankType + - " defined for the same index in field '" + - field.getName() + "'. Using " + - rankType + "."); - field.setRankType(rankType); - } - } - } - } -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java deleted file mode 100644 index 1bcf646c8ec..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; - -import java.util.List; -import java.util.logging.Level; - -/** - * Class resolving conflicts when fields with different stemming-settings are - * combined into the same index - */ -public class StemmingResolver extends MultiFieldResolver { - - public StemmingResolver(String indexName, List fields, Schema schema, DeployLogger logger) { - super(indexName, fields, schema, logger); - } - - @Override - public void resolve() { - checkStemmingForIndexFields(indexName, fields); - } - - private void checkStemmingForIndexFields(String indexName, List fields) { - Stemming stemming = null; - SDField stemmingField = null; - for (SDField field : fields) { - if (stemming == null && stemmingField==null) { - stemming = field.getStemming(schema); - stemmingField = field; - } else if (stemming != field.getStemming(schema)) { - deployLogger.logApplicationPackage(Level.WARNING, "Field '" + field.getName() + "' has " + field.getStemming(schema) + - ", whereas field '" + stemmingField.getName() + "' has " + stemming + - ". All fields indexing to the index '" + indexName + "' must have the same stemming." + - " This should be corrected as it will make indexing fail in a few cases."); - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java deleted file mode 100644 index 9d259154373..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Classes in this package (processors) implements some search - * definition features by reducing them to simpler features. - * The processors are run after parsing of the search definition, - * before creating the derived model. - * - * For simplicity, features should always be implemented here - * rather than in the derived model if possible. - * - * New processors must be added to the list in Processing. - */ -@com.yahoo.api.annotations.PackageMarker -package com.yahoo.searchdefinition.processing; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java index a63728c403d..df78118009d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java @@ -13,13 +13,12 @@ import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.document.internal.GeoPosType; import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.schema.document.annotation.SDAnnotationType; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 986746bd80c..1974774f7b5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -2,26 +2,21 @@ package com.yahoo.vespa.configmodel.producers; import com.yahoo.document.config.DocumentmanagerConfig; -import static com.yahoo.document.config.DocumentmanagerConfig.*; import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedTemporaryType; import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.schema.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 771d7c17162..bb57cd189c9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -5,13 +5,12 @@ import com.yahoo.document.*; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedTemporaryType; import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.schema.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 2edc845d6b7..337f5e11329 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -2,11 +2,9 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index da3070af55f..92a75cb051b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.TypedKey; +import com.yahoo.schema.document.TypedKey; import java.io.Serializable; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 149326130be..a919c19d754 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -11,7 +11,6 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.ApplicationConfigProducerRoot; @@ -30,15 +29,14 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.QrConfig; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.derived.FileDistributedOnnxModels; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RankProfileList; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RankProfileList; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index d145ce6bd24..01bf846d4cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SchemaInfo; -import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.document.ComplexAttributeFieldUtils; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java index fb70f2b769c..bc91c7cbad2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java @@ -6,9 +6,9 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.DistributableResource; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index 9de9c34737f..03f3cdba6cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.MatchAlgorithm; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.MatchAlgorithm; +import com.yahoo.schema.Index; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.DocumentDatabase; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 3492ccf0b21..5c531d28a8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -10,7 +10,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; -import com.yahoo.searchdefinition.DistributableResource; +import com.yahoo.schema.DistributableResource; import com.yahoo.system.ProcessExecuter; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.search.AttributesConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 162cbe7495f..f62247d788e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -11,10 +11,10 @@ import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SchemaInfo; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 438b28948c6..458f8dfbd68 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -6,9 +6,9 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.MatchType; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.MatchType; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 2820d0984db..0563aa02149 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -6,8 +6,8 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder; import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java index 2a3c5fdc6b0..432424c8027 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -5,12 +5,12 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.IndexSchema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.IndexSchema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.HnswIndexParams; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index b461b38c75f..bbfa939f8a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index f6f6b6abdee..a6fcfd652a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java index a10d2c36de1..0909afb71e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java @@ -8,9 +8,9 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ComplexAttributeFieldUtils; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 40df899c94d..cb8e6ba85ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.container; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.schema.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index 6532fed9abf..4afde25cafd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -18,7 +18,7 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.schema.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 708e7c75ac8..97c3f85dced 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,7 +8,7 @@ import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index 1ea84a33d2d..3d409263d8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -7,7 +7,7 @@ import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.cluster.QrMonitorConfig; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.search.config.ClusterConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 5d8d36caa5e..0bd93c6d0df 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -34,8 +34,8 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.derived.RankProfileList; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 2d01f9d903e..4be5153293f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -5,8 +5,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 255ce3a95f6..9ba87fd24bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -11,9 +11,9 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index e628907068d..64a1015d056 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.ImportedFieldsConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 44e574199ea..d63ae451f39 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -7,9 +7,9 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java index b4082308195..06a5929a430 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java @@ -1,11 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.io.Serializable; -import java.util.Collection; import java.util.List; /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 1141af6d79d..d727e6d6395 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -3,11 +3,8 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.derived.SchemaInfo; -import com.yahoo.searchdefinition.derived.SummaryMap; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.tensor.TensorType; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.derived.SummaryMap; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -15,7 +12,7 @@ import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.config.model.producer.AbstractConfigProducer; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 09ed8bad403..724ba70510f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 86a623071b5..47b7909eb69 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -11,7 +11,7 @@ options { PARSER_BEGIN(SchemaParser) -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; @@ -19,21 +19,21 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile.DiversitySettings; -import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.MatchAlgorithm; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.Sorting; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.fieldoperation.IndexingOperation; +import com.yahoo.schema.DistributableResource; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile.DiversitySettings; +import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.MatchAlgorithm; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.Sorting; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.fieldoperation.IndexingOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 3acc44113d0..57a61e40d89 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -13,8 +13,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.document.DataType; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java index 96186150108..273102a00c5 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import java.io.File; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index d4ebbe62507..4a3d85c6a76 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -4,11 +4,11 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java index 36a4c7bdafc..4b0d4ee7a9a 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java @@ -2,8 +2,8 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; import org.junit.Test; import static org.junit.Assert.fail; diff --git a/config-model/src/test/java/com/yahoo/schema/AbstractSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/AbstractSchemaTestCase.java new file mode 100644 index 00000000000..e816456249f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/AbstractSchemaTestCase.java @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.io.IOUtils; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; + +import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals; +import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; + +public abstract class AbstractSchemaTestCase { + + protected static void assertConfigFile(String filename, String cfg) throws IOException { + IOUtils.writeFile(filename + ".actual", cfg, false); + if (! cfg.endsWith("\n")) { + IOUtils.writeFile(filename + ".actual", "\n", true); + } + assertSerializedConfigFileEquals(filename, cfg); + } + + protected static void assertConfigFiles(String expectedFile, + String cfgFile, + boolean orderMatters, + boolean updateOnAssert) throws IOException { + try { + assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile), orderMatters); + } catch (AssertionError e) { + if (updateOnAssert) { + BufferedWriter writer = IOUtils.createWriter(expectedFile, false); + writer.write(readAndCensorIndexes(cfgFile)); + writer.newLine(); + writer.flush(); + writer.close(); + System.err.println(e.getMessage() + " [not equal files: >>>"+expectedFile+"<<< and >>>"+cfgFile+"<<< in assertConfigFiles]"); + return; + } + throw new AssertionError(e.getMessage() + " [not equal files: >>>"+expectedFile+"<<< and >>>"+cfgFile+"<<< in assertConfigFiles]", e); + } + } + /** + * This is to avoid having to keep those pesky array index numbers in the config format up to date + * as new entries are added and removed. + */ + private static String readAndCensorIndexes(String file) throws IOException { + StringBuilder b = new StringBuilder(); + try (BufferedReader r = IOUtils.createReader(file)) { + int character; + boolean lastWasNewline = false; + boolean inBrackets = false; + while (-1 != (character = r.read())) { + // skip empty lines + if (character == '\n') { + if (lastWasNewline) continue; + lastWasNewline = true; + } + else { + lastWasNewline = false; + } + + // skip quoted strings + if (character == '"') { + b.appendCodePoint(character); + while (-1 != (character = r.read()) && character != '"') { + b.appendCodePoint(character); + } + } + + // skip bracket content + if (character == ']') + inBrackets = false; + if (! inBrackets) + b.appendCodePoint(character); + if (character == '[') + inBrackets = true; + } + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/AnnotationReferenceTestCase.java b/config-model/src/test/java/com/yahoo/schema/AnnotationReferenceTestCase.java new file mode 100644 index 00000000000..dbe827ed67f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/AnnotationReferenceTestCase.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.DataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.Field; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.config.model.deploy.TestProperties; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author arnej + */ +public class AnnotationReferenceTestCase { + + static final String sd = + joinLines("search test {", + " document test { ", + " struct mystruct {", + " field x type int {}", + " }", + " field a type string {}", + " field b type mystruct {}", + " annotation marker {}", + " annotation person {", + " field name type string {}", + " field age type int {}", + " }", + " annotation complex {", + " field title type string {}", + " field tag type annotationreference {}", + " field owner type annotationreference {}", + " }", + " }", + "}"); + + @Test + public void noAnnotationReferenceInDocument() throws Exception { + var builder = new ApplicationBuilder(new TestProperties()); + builder.addSchema(sd); + builder.build(true); + var doc = builder.getSchema().getDocument(); + checkForAnnRef(doc); + var complex = doc.findAnnotation("complex"); + var dt = complex.getDataType(); + assertTrue(dt instanceof StructDataType); + var struct = (StructDataType)dt; + var field = struct.getField("owner"); + assertTrue(field.getDataType() instanceof AnnotationReferenceDataType); + } + + void checkForAnnRef(SDDocumentType doc) { + for (var child : doc.getTypes()) { + checkForAnnRef(child); + } + for (Field field : doc.fieldSet()) { + DataType fieldType = field.getDataType(); + assertFalse(fieldType instanceof AnnotationReferenceDataType); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/schema/ArraysTestCase.java new file mode 100644 index 00000000000..aab79617556 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/ArraysTestCase.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * tests importing of document containing array type fields + * + * @author bratseth + */ +public class ArraysTestCase extends AbstractSchemaTestCase { + + @Test + public void testArrayImporting() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arrays.sd"); + + SDField tags = (SDField) schema.getDocument().getField("tags"); + assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); + + SDField ratings = (SDField) schema.getDocument().getField("ratings"); + assertTrue(ratings.getDataType() instanceof ArrayDataType); + assertEquals(DataType.INT, ((ArrayDataType)ratings.getDataType()).getNestedType()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/schema/ArraysWeightedSetsTestCase.java new file mode 100644 index 00000000000..57331bea6bd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/ArraysWeightedSetsTestCase.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.schema.document.SDField; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * tests importing of document containing array type fields and weighted set type fields, new syntax. + * + * @author Einar M R Rosenvinge + */ +public class ArraysWeightedSetsTestCase extends AbstractSchemaTestCase { + @Test + public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.schema.parser.ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); + + SDField tags = (SDField) schema.getDocument().getField("tags"); + assertTrue(tags.getDataType() instanceof ArrayDataType); + assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); + + SDField ratings = (SDField) schema.getDocument().getField("ratings"); + assertTrue(ratings.getDataType() instanceof ArrayDataType); + assertEquals(DataType.INT, ((CollectionDataType)ratings.getDataType()).getNestedType()); + + SDField flags = (SDField) schema.getDocument().getField("flags"); + assertTrue(flags.getDataType() instanceof WeightedSetDataType); + assertEquals(DataType.STRING, ((CollectionDataType)flags.getDataType()).getNestedType()); + + SDField banners = (SDField) schema.getDocument().getField("banners"); + assertTrue(banners.getDataType() instanceof WeightedSetDataType); + assertEquals(DataType.INT, ((CollectionDataType)banners.getDataType()).getNestedType()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java new file mode 100644 index 00000000000..bbe63f95787 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java @@ -0,0 +1,344 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.StructDataType; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.IndexingScript; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import org.junit.Test; + +import java.io.IOException; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Attribute settings + * + * @author bratseth + */ +public class AttributeSettingsTestCase extends AbstractSchemaTestCase { + + @Test + public void testAttributeSettings() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + + SDField f1=(SDField) schema.getDocument().getField("f1"); + assertEquals(1, f1.getAttributes().size()); + Attribute a1 = f1.getAttributes().get(f1.getName()); + assertEquals(Attribute.Type.LONG, a1.getType()); + assertEquals(Attribute.CollectionType.SINGLE, a1.getCollectionType()); + assertTrue(a1.isHuge()); + assertFalse(a1.isFastSearch()); + assertFalse(a1.isFastAccess()); + assertFalse(a1.isRemoveIfZero()); + assertFalse(a1.isCreateIfNonExistent()); + + SDField f2=(SDField) schema.getDocument().getField("f2"); + assertEquals(1, f2.getAttributes().size()); + Attribute a2 = f2.getAttributes().get(f2.getName()); + assertEquals(Attribute.Type.LONG, a2.getType()); + assertEquals(Attribute.CollectionType.SINGLE, a2.getCollectionType()); + assertFalse(a2.isHuge()); + assertTrue(a2.isFastSearch()); + assertFalse(a2.isFastAccess()); + assertFalse(a2.isRemoveIfZero()); + assertFalse(a2.isCreateIfNonExistent()); + assertEquals("f2", f2.getAliasToName().get("f2alias")); + SDField f3=(SDField) schema.getDocument().getField("f3"); + assertEquals(1, f3.getAttributes().size()); + assertEquals("f3", f3.getAliasToName().get("f3alias")); + + Attribute a3 = f3.getAttributes().get(f3.getName()); + assertEquals(Attribute.Type.LONG, a3.getType()); + assertEquals(Attribute.CollectionType.SINGLE, a3.getCollectionType()); + assertFalse(a3.isHuge()); + assertFalse(a3.isFastSearch()); + assertFalse(a3.isFastAccess()); + assertFalse(a3.isRemoveIfZero()); + assertFalse(a3.isCreateIfNonExistent()); + + assertWeightedSet(schema, "f4", true, true); + assertWeightedSet(schema, "f5", true, true); + assertWeightedSet(schema, "f6", true, true); + assertWeightedSet(schema, "f7", true, false); + assertWeightedSet(schema, "f8", true, false); + assertWeightedSet(schema, "f9", false, true); + assertWeightedSet(schema, "f10", false, true); + + assertAttrSettings(schema, "f4", false, false, false); + assertAttrSettings(schema, "f5", true, true, true); + assertAttrSettings(schema, "f6", false, false, false); + assertAttrSettings(schema, "f7", false, false, false); + assertAttrSettings(schema, "f8", false, false, false); + assertAttrSettings(schema, "f9", false, false, false); + assertAttrSettings(schema, "f10", false, false, false); + } + + private void assertWeightedSet(Schema schema, String name, boolean createIfNonExistent, boolean removeIfZero) { + SDField f4 = (SDField) schema.getDocument().getField(name); + assertEquals(1, f4.getAttributes().size()); + Attribute a4 = f4.getAttributes().get(f4.getName()); + assertEquals(Attribute.Type.STRING, a4.getType()); + assertEquals(Attribute.CollectionType.WEIGHTEDSET, a4.getCollectionType()); + assertEquals(a4.isRemoveIfZero(), removeIfZero); + assertEquals(a4.isCreateIfNonExistent(), createIfNonExistent); + } + + private void assertAttrSettings(Schema schema, String name, boolean fastAccess, boolean fastSearch, boolean paged) { + SDField f4 = (SDField) schema.getDocument().getField(name); + assertEquals(1, f4.getAttributes().size()); + Attribute a4 = f4.getAttributes().get(f4.getName()); + assertEquals(Attribute.Type.STRING, a4.getType()); + assertEquals(Attribute.CollectionType.WEIGHTEDSET, a4.getCollectionType()); + assertEquals(a4.isFastSearch(), fastSearch); + assertEquals(a4.isFastAccess(), fastAccess); + assertEquals(a4.isPaged(), paged); + } + + @Test + public void requireThatFastAccessCanBeSet() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); + SDField field = (SDField) schema.getDocument().getField("fast_access"); + assertEquals(1, field.getAttributes().size()); + Attribute attr = field.getAttributes().get(field.getName()); + assertTrue(attr.isFastAccess()); + } + + private Schema getSchema(String sd) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(sd); + builder.build(true); + return builder.getSchema(); + } + + private Attribute getAttributeF(String sd) throws ParseException { + Schema schema = getSchema(sd); + SDField field = (SDField) schema.getDocument().getField("f"); + return field.getAttributes().get(field.getName()); + } + + @Test + public void requireThatPagedIsDefaultOff() throws ParseException { + Attribute attr = getAttributeF( + "search test {\n" + + " document test { \n" + + " field f type tensor(x[2]) { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + "}\n"); + assertFalse(attr.isPaged()); + } + @Test + public void requireThatPagedCanBeSet() throws ParseException { + Attribute attr = getAttributeF( + "search test {\n" + + " document test { \n" + + " field f type tensor(x[2]) { \n" + + " indexing: attribute \n" + + " attribute: paged \n" + + " }\n" + + " }\n" + + "}\n"); + assertTrue(attr.isPaged()); + } + + @Test + public void requireThatMutableIsDefaultOff() throws ParseException { + Attribute attr = getAttributeF( + "search test {\n" + + " document test { \n" + + " field f type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + "}\n"); + assertFalse(attr.isMutable()); + } + + @Test + public void requireThatMutableCanNotbeSetInDocument() throws ParseException { + try { + getSchema("search test {\n" + + " document test {\n" + + " field f type int {\n" + + " indexing: attribute\n" + + " attribute: mutable\n" + + " }\n" + + " }\n" + + "}\n"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Field 'f' in 'test' can not be marked mutable as it is inside the document clause.", e.getMessage()); + } + } + + @Test + public void requireThatMutableExtraFieldCanBeSet() throws ParseException { + Attribute attr = getAttributeF( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " field f type long {\n" + + " indexing: 0 | to_long | attribute\n" + + " attribute: mutable\n" + + " }\n" + + "}\n"); + assertTrue(attr.isMutable()); + } + + private Schema getSearchWithMutables() throws ParseException { + return getSchema( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " field m type long {\n" + + " indexing: attribute\n" + + " attribute: mutable\n" + + " }\n" + + " field f type long {\n" + + " indexing: 0 | to_long | attribute\n" + + " }\n" + + "}\n"); + } + + @Test + public void requireThatMutableConfigIsProperlyPropagated() throws ParseException { + AttributeFields attributes = new AttributeFields(getSearchWithMutables()); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, true); + AttributesConfig cfg = builder.build(); + assertEquals("a", cfg.attribute().get(0).name()); + assertFalse(cfg.attribute().get(0).ismutable()); + + assertEquals("f", cfg.attribute().get(1).name()); + assertFalse(cfg.attribute().get(1).ismutable()); + + assertEquals("m", cfg.attribute().get(2).name()); + assertTrue(cfg.attribute().get(2).ismutable()); + } + + @Test + public void requireMaxUnCommittedMemoryIsProperlyPropagated() throws ParseException { + AttributeFields attributes = new AttributeFields(getSearchWithMutables()); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, true); + AttributesConfig cfg = builder.build(); + assertEquals("a", cfg.attribute().get(0).name()); + assertEquals(13333, cfg.attribute().get(0).maxuncommittedmemory()); + + assertEquals("f", cfg.attribute().get(1).name()); + assertEquals(13333, cfg.attribute().get(1).maxuncommittedmemory()); + + assertEquals("m", cfg.attribute().get(2).name()); + assertEquals(13333, cfg.attribute().get(2).maxuncommittedmemory()); + } + + private void verifyEnableBitVectorDefault(Schema schema, boolean enableBitVectors) { + AttributeFields attributes = new AttributeFields(schema); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, enableBitVectors); + AttributesConfig cfg = builder.build(); + assertEquals("a", cfg.attribute().get(0).name()); + assertEquals(enableBitVectors, cfg.attribute().get(0).enablebitvectors()); + + assertEquals("b", cfg.attribute().get(1).name()); + assertFalse(cfg.attribute().get(1).enablebitvectors()); + } + + @Test + public void requireEnableBitVectorsIsProperlyPropagated() throws ParseException { + Schema schema = getSchema( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " attribute: fast-search\n" + + " }\n" + + " field b type int { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + "}\n"); + verifyEnableBitVectorDefault(schema, false); + verifyEnableBitVectorDefault(schema, true); + } + + @Test + public void requireThatMutableIsAllowedThroughIndexing() throws ParseException { + IndexingScript script = new IndexingScript(getSearchWithMutables()); + IlscriptsConfig.Builder builder = new IlscriptsConfig.Builder(); + script.getConfig(builder); + IlscriptsConfig cfg = builder.build(); + assertEquals(1, cfg.ilscript().size()); + IlscriptsConfig.Ilscript ils = cfg.ilscript(0); + assertEquals("test", ils.doctype()); + assertEquals(2, ils.docfield().size()); + assertEquals("a", ils.docfield(0)); + assertEquals("m", ils.docfield(1)); + + } + + @Test + public void attribute_convert_to_array_copies_internal_state() { + StructDataType refType = new StructDataType("my_struct"); + Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE, + Optional.of(TensorType.fromSpec("tensor(x{})")), Optional.of(refType)); + single.setRemoveIfZero(true); + single.setCreateIfNonExistent(true); + single.setPrefetch(Boolean.TRUE); + single.setEnableBitVectors(true); + single.setEnableOnlyBitVector(true); + single.setFastSearch(true); + single.setHuge(true); + single.setPaged(true); + single.setFastAccess(true); + single.setPosition(true); + single.setArity(5); + single.setLowerBound(7); + single.setUpperBound(11); + single.setDensePostingListThreshold(13.3); + single.getSorting().setAscending(); + single.getAliases().add("foo"); + + Attribute array = single.convertToArray(); + assertEquals("foo", array.getName()); + assertEquals(Attribute.Type.STRING, array.getType()); + assertEquals(Attribute.CollectionType.ARRAY, array.getCollectionType()); + assertEquals(Optional.of(TensorType.fromSpec("tensor(x{})")), array.tensorType()); + assertSame(single.referenceDocumentType(), array.referenceDocumentType()); + assertTrue(array.isRemoveIfZero()); + assertTrue(array.isCreateIfNonExistent()); + assertTrue(array.isPrefetch()); + assertTrue(array.isEnabledBitVectors()); + assertTrue(array.isEnabledOnlyBitVector()); + assertTrue(array.isFastSearch()); + assertTrue(array.isHuge()); + assertTrue(array.isPaged()); + assertTrue(array.isFastAccess()); + assertTrue(array.isPosition()); + assertEquals(5, array.arity()); + assertEquals(7, array.lowerBound()); + assertEquals(11, array.upperBound()); + assertEquals(13.3, array.densePostingListThreshold(), 0.00001); + assertSame(single.getSorting(), array.getSorting()); + assertSame(single.getAliases(), array.getAliases()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java new file mode 100644 index 00000000000..fb7d68faf2d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDField; + +/** + * Convenience class for tests that need to set attribute properties on fields. + */ +public class AttributeUtils { + + public static void addAttributeAspect(SDField field) { + field.parseIndexingScript("{ attribute }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/CommentTestCase.java b/config-model/src/test/java/com/yahoo/schema/CommentTestCase.java new file mode 100644 index 00000000000..4df4ead171b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/CommentTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Tests comment handling + * + * @author bratseth + */ +public class CommentTestCase extends AbstractSchemaTestCase { + + @Test + public void testComments() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/comment.sd"); + SDField field = schema.getConcreteField("a"); + assertEquals("{ input a | tokenize normalize stem:\"BEST\" | summary a | index a; }", + field.getIndexingScript().toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java new file mode 100644 index 00000000000..482bc877081 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java @@ -0,0 +1,110 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.search.query.ranking.Diversity; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; +import static org.junit.Assert.fail; + +import static org.junit.Assert.assertEquals; + +/** + * @author baldersheim + */ +public class DiversityTestCase { + @Test + public void testDiversity() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " attribute: fast-search\n" + + " }\n" + + " field b type int {\n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " match-phase {\n" + + " diversity {\n" + + " attribute: b\n" + + " min-groups: 74\n" + + " cutoff-factor: 17.3\n" + + " cutoff-strategy: strict" + + " }\n" + + " attribute: a\n" + + " max-hits: 120\n" + + " max-filter-coverage: 0.065" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); + RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); + assertEquals("b", diversity.getAttribute()); + assertEquals(74, diversity.getMinGroups()); + assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); + assertEquals(Diversity.CutoffStrategy.strict, diversity.getCutoffStrategy()); + assertEquals(120, matchPhase.getMaxHits()); + assertEquals("a", matchPhase.getAttribute()); + assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); + } + + private static String getMessagePrefix() { + return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; + } + @Test + public void requireSingleNumericOrString() throws ParseException { + ApplicationBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); + + try { + builder.build(true); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'predicate'", e.getMessage()); + } + } + + @Test + public void requireSingle() throws ParseException { + ApplicationBuilder builder = getSearchBuilder("field b type array { indexing: attribute }"); + + try { + builder.build(true); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array'", e.getMessage()); + } + } + private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type int { \n" + + " indexing: attribute \n" + + " attribute: fast-search\n" + + " }\n" + + diversity + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " match-phase {\n" + + " diversity {\n" + + " attribute: b\n" + + " min-groups: 74\n" + + " }\n" + + " attribute: a\n" + + " max-hits: 120\n" + + " }\n" + + " }\n" + + "}\n"); + return builder; + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java new file mode 100644 index 00000000000..ef4d8e05540 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java @@ -0,0 +1,166 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bjorncs + */ +public class DocumentGraphValidatorTest { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void simple_ref_dag_is_allowed() { + Schema advertiserSchema = createSearchWithName("advertiser"); + Schema campaignSchema = createSearchWithName("campaign"); + Schema adSchema = createSearchWithName("ad"); + createDocumentReference(adSchema, advertiserSchema, "advertiser_ref"); + createDocumentReference(adSchema, campaignSchema, "campaign_ref"); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + validator.validateDocumentGraph(documentListOf(advertiserSchema, campaignSchema, adSchema)); + } + + @Test + public void simple_inheritance_dag_is_allowed() { + Schema grandfather = createSearchWithName("grandfather"); + Schema father = createSearchWithName("father", grandfather); + Schema son = createSearchWithName("son", father); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + validator.validateDocumentGraph(documentListOf(son, father, grandfather)); + } + + @Test + public void complex_dag_is_allowed() { + Schema grandfather = createSearchWithName("grandfather"); + Schema father = createSearchWithName("father", grandfather); + Schema mother = createSearchWithName("mother", grandfather); + createDocumentReference(father, mother, "wife_ref"); + Schema son = createSearchWithName("son", father, mother); + Schema daughter = createSearchWithName("daughter", father, mother); + createDocumentReference(daughter, son, "brother_ref"); + + Schema randomGuy1 = createSearchWithName("randomguy1"); + Schema randomGuy2 = createSearchWithName("randomguy2"); + createDocumentReference(randomGuy1, mother, "secret_ref"); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + validator.validateDocumentGraph(documentListOf(son, father, grandfather, son, daughter, randomGuy1, randomGuy2)); + } + + @Test + public void ref_cycle_is_forbidden() { + Schema schema1 = createSearchWithName("doc1"); + Schema schema2 = createSearchWithName("doc2"); + Schema schema3 = createSearchWithName("doc3"); + createDocumentReference(schema1, schema2, "ref_2"); + createDocumentReference(schema2, schema3, "ref_3"); + createDocumentReference(schema3, schema1, "ref_1"); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); + exceptionRule.expectMessage("Document dependency cycle detected: doc1->doc2->doc3->doc1."); + validator.validateDocumentGraph(documentListOf(schema1, schema2, schema3)); + } + + @Test + public void inherit_cycle_is_forbidden() { + Schema schema1 = createSearchWithName("doc1"); + Schema schema2 = createSearchWithName("doc2", schema1); + Schema schema3 = createSearchWithName("doc3", schema2); + schema1.getDocument().inherit(schema3.getDocument()); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); + exceptionRule.expectMessage("Document dependency cycle detected: doc1->doc3->doc2->doc1."); + validator.validateDocumentGraph(documentListOf(schema1, schema2, schema3)); + } + + @Test + public void combined_inherit_and_ref_cycle_is_forbidden() { + Schema schema1 = createSearchWithName("doc1"); + Schema schema2 = createSearchWithName("doc2", schema1); + Schema schema3 = createSearchWithName("doc3", schema2); + createDocumentReference(schema1, schema3, "ref_1"); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); + exceptionRule.expectMessage("Document dependency cycle detected: doc1->doc3->doc2->doc1."); + validator.validateDocumentGraph(documentListOf(schema1, schema2, schema3)); + } + + @Test + public void self_reference_is_forbidden() { + Schema adSchema = createSearchWithName("ad"); + createDocumentReference(adSchema, adSchema, "ad_ref"); + + DocumentGraphValidator validator = new DocumentGraphValidator(); + exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); + exceptionRule.expectMessage("Document dependency cycle detected: ad->ad."); + validator.validateDocumentGraph(documentListOf(adSchema)); + } + + /** + * Self inheritance is checked early because it is possible, and because it otherwise + * produces a stack overflow before getting to graph validation. + */ + @Test + public void self_inheritance_forbidden() { + try { + Schema adSchema = createSearchWithName("ad"); + SDDocumentType document = adSchema.getDocument(); + document.inherit(document); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Document type 'ad' cannot inherit itself", e.getMessage()); + } + } + + private static List documentListOf(Schema... schemas) { + return Arrays.stream(schemas).map(Schema::getDocument).collect(toList()); + } + + private static Schema createSearchWithName(String name, Schema... parents) { + Schema campaignSchema = new Schema(name, MockApplicationPackage.createEmpty()); + SDDocumentType document = new SDDocumentType(name); + campaignSchema.addDocument(document); + document.setDocumentReferences(new DocumentReferences(Collections.emptyMap())); + Arrays.stream(parents) + .map(Schema::getDocument) + .forEach(document::inherit); + return campaignSchema; + } + + @SuppressWarnings("deprecation") + private static void createDocumentReference(Schema from, Schema to, String refFieldName) { + SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, NewDocumentReferenceDataType.forDocumentName(to.getName())); + fromDocument.addField(refField); + Map originalMap = fromDocument.getDocumentReferences().get().referenceMap(); + HashMap modifiedMap = new HashMap<>(originalMap); + modifiedMap.put(refFieldName, new DocumentReference(refField, to)); + fromDocument.setDocumentReferences(new DocumentReferences(modifiedMap)); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java new file mode 100644 index 00000000000..1592060f466 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java @@ -0,0 +1,103 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Map; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class DocumentReferenceResolverTest { + + private static final String BAR = "bar"; + private static final String FOO = "foo"; + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void reference_from_one_document_to_another_is_resolved() { + // Create bar document with no fields + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); + SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); + barSchema.addDocument(barDocument); + + // Create foo document with document reference to bar and add another field + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", new NewDocumentReferenceDataType(barDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToBarField); + SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); + fooDocument.addField(fooRefToBarField); + fooDocument.addField(irrelevantField); + fooSchema.addDocument(fooDocument); + + DocumentReferenceResolver resolver = new DocumentReferenceResolver(asList(fooSchema, barSchema)); + resolver.resolveReferences(fooDocument); + assertTrue(fooDocument.getDocumentReferences().isPresent()); + + Map fooReferenceMap = fooDocument.getDocumentReferences().get().referenceMap(); + assertEquals(1, fooReferenceMap.size()); + assertSame(barSchema, fooReferenceMap.get("bar_ref").targetSearch()); + assertSame(fooRefToBarField, fooReferenceMap.get("bar_ref").referenceField()); + } + + @SuppressWarnings("deprecation") + @Test + public void throws_user_friendly_exception_if_referenced_document_does_not_exist() { + // Create foo document with document reference to non-existing document bar + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField( + fooDocument, + "bar_ref", NewDocumentReferenceDataType.forDocumentName("bar")); + AttributeUtils.addAttributeAspect(fooRefToBarField); + fooDocument.addField(fooRefToBarField); + fooSchema.addDocument(fooDocument); + + DocumentReferenceResolver resolver = new DocumentReferenceResolver(singletonList(fooSchema)); + + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "Invalid document reference 'bar_ref': Could not find document type 'bar'"); + resolver.resolveReferences(fooDocument); + } + + @Test + public void throws_exception_if_reference_is_not_an_attribute() { + // Create bar document with no fields + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); + SDDocumentType barDocument = new SDDocumentType("bar", barSchema); + barSchema.addDocument(barDocument); + + // Create foo document with document reference to bar + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); + SDField fooRefToBarField = new SDField + (fooDocument, "bar_ref", new NewDocumentReferenceDataType(barDocument.getDocumentType())); + fooDocument.addField(fooRefToBarField); + fooSchema.addDocument(fooDocument); + + DocumentReferenceResolver resolver = new DocumentReferenceResolver(asList(fooSchema, barSchema)); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "The field 'bar_ref' is an invalid document reference. The field must be an attribute."); + resolver.resolveReferences(fooDocument); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/FeatureNamesTestCase.java new file mode 100644 index 00000000000..bff4b434408 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/FeatureNamesTestCase.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import org.junit.Ignore; +import org.junit.Test; + +import java.util.function.Function; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests rank feature names. + * + * @author bratseth + */ +public class FeatureNamesTestCase { + + @Test + public void testArgument() { + assertFalse(FeatureNames.argumentOf("foo(bar)").isPresent()); + assertFalse(FeatureNames.argumentOf("foo(bar.baz)").isPresent()); + assertEquals("bar", FeatureNames.argumentOf("query(bar)").get()); + assertEquals("bar.baz", FeatureNames.argumentOf("query(bar.baz)").get()); + assertEquals("bar", FeatureNames.argumentOf("attribute(bar)").get()); + assertEquals("bar.baz", FeatureNames.argumentOf("attribute(bar.baz)").get()); + assertEquals("bar", FeatureNames.argumentOf("constant(bar)").get()); + assertEquals("bar.baz", FeatureNames.argumentOf("constant(bar.baz)").get()); + } + + @Test + public void testConstantFeature() { + assertEquals("constant(foo)", + FeatureNames.asConstantFeature("foo").toString()); + } + + @Test + public void testAttributeFeature() { + assertEquals("attribute(foo)", + FeatureNames.asAttributeFeature("foo").toString()); + } + + @Test + public void testQueryFeature() { + assertEquals("query(\"foo.bar\")", + FeatureNames.asQueryFeature("foo.bar").toString()); + } + + @Test + public void testLegalFeatureNames() { + assertTrue(FeatureNames.notNeedQuotes("_")); + assertFalse(FeatureNames.notNeedQuotes("-")); + assertTrue(FeatureNames.notNeedQuotes("_-")); + assertTrue(FeatureNames.notNeedQuotes("0_-azAZxy98-_")); + assertFalse(FeatureNames.notNeedQuotes("0_-azAZxy98-_+")); + } + + @Test + @Ignore + /* + * Unignore to verify performance + * 2021/09/05 performance was a factor of 5.25 + * 'Identifier handcoded validity check took 4301ms + * Identifier regexp validity check took 22609ms' + */ + public void benchMarkPatternMatching() { + Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); + String[] strings = new String[1000]; + for (int i = 0; i < strings.length; i++) { + strings[i] = i + "-legal_string" + i; + } + + countValid(strings, 1000, "handcoded warmup", FeatureNames::notNeedQuotes); + countValid(strings, 1000, "regexp warmup", (s) -> identifierRegexp.matcher(s).matches()); + + countValid(strings, 100000, "handcoded", FeatureNames::notNeedQuotes); + countValid(strings, 100000, "regexp", (s) -> identifierRegexp.matcher(s).matches()); + } + + private void countValid(String [] strings, int numReps, String text, Function func) { + long start = System.nanoTime(); + int validCount = 0; + for (int i = 0; i < numReps; i++) { + for (String s : strings) { + if (func.apply(s)) validCount++; + } + } + long end = System.nanoTime(); + assertEquals(strings.length * numReps, validCount); + System.out.println("Identifier " + text + " validity check took " + (end - start)/1000000 + "ms"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/schema/FieldOfTypeDocumentTestCase.java new file mode 100644 index 00000000000..4a590288d53 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/FieldOfTypeDocumentTestCase.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.Field; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.schema.derived.Deriver; +import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author Einar M R Rosenvinge + */ +public class FieldOfTypeDocumentTestCase extends AbstractSchemaTestCase { + + @Test + public void testDocument() throws IOException { + + List sds = new ArrayList<>(); + sds.add("src/test/examples/music.sd"); + sds.add("src/test/examples/fieldoftypedocument.sd"); + DocumentmanagerConfig.Builder value = Deriver.getDocumentManagerConfig(sds); + assertConfigFile("src/test/examples/fieldoftypedocument.cfg", + new DocumentmanagerConfig(value).toString() + "\n"); + + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "raw:" + new DocumentmanagerConfig(value).toString()); + + + DocumentType musicType = manager.getDocumentType("music"); + assertEquals(3, musicType.getFieldCount()); + + Field intField = musicType.getField("intfield"); + assertEquals(DataType.INT, intField.getDataType()); + Field stringField = musicType.getField("stringfield"); + assertEquals(DataType.STRING, stringField.getDataType()); + Field longField = musicType.getField("longfield"); + assertEquals(DataType.LONG, longField.getDataType()); + + + DocumentType bookType = manager.getDocumentType("book"); + assertEquals(1, bookType.getFieldCount()); + + Field musicField = bookType.getField("soundtrack"); + assertSame(musicType, musicField.getDataType()); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java new file mode 100644 index 00000000000..92d11b3a18a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImportedFieldsEnumeratorTest { + + @Test + public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { + String PARENT = "parent"; + Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); + SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); + var parentField = new SDField(parentDocument, "their_field", DataType.INT); + AttributeUtils.addAttributeAspect(parentField); + parentDocument.addField(parentField); + parentSchema.addDocument(parentDocument); + + String FOO = "foo"; + Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); + /* + SDField fooRefToParent = new SDField( + "foo_ref", NewDocumentReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToParent); + */ + var fooImports = fooSchema.temporaryImportedFields().get(); + fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); + fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); + SDDocumentType fooDocument = new SDDocumentType(FOO, fooSchema); + fooSchema.addDocument(fooDocument); + + String BAR = "bar"; + Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); + /* + SDField barRefToParent = new SDField( + "bar_ref", NewDocumentReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(barRefToParent); + */ + var barImports = barSchema.temporaryImportedFields().get(); + barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); + SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); + barSchema.addDocument(barDocument); + + var enumerator = new ImportedFieldsEnumerator(List.of(parentSchema, fooSchema, barSchema)); + + enumerator.enumerateImportedFields(parentDocument); + assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent + + enumerator.enumerateImportedFields(fooDocument); + assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); + + enumerator.enumerateImportedFields(barDocument); + assertImportedFieldsAre(barDocument, List.of("my_cool_import")); + } + + private void assertImportedFieldsAre(SDDocumentType documentType, List expectedNames) { + assertNotNull(documentType.getTemporaryImportedFields()); + var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); + var expectedNameSet = new HashSet<>(expectedNames); + assertEquals(expectedNameSet, actualNames); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/schema/IncorrectRankingExpressionFileRefTestCase.java new file mode 100644 index 00000000000..87e168adb66 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/IncorrectRankingExpressionFileRefTestCase.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class IncorrectRankingExpressionFileRefTestCase extends AbstractSchemaTestCase { + + @Test + public void testIncorrectRef() throws IOException, ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", + registry, + new QueryProfileRegistry()); + new DerivedConfiguration(schema, registry); // cause rank profile parsing + fail("parsing should have failed"); + } catch (IllegalArgumentException e) { + String message = Exceptions.toMessageString(e); + assertTrue(message.contains("Could not read ranking expression file")); + assertTrue(message.contains("wrongending.expr.expression")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/IncorrectSummaryTypesTestCase.java new file mode 100644 index 00000000000..e58cce6472a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/IncorrectSummaryTypesTestCase.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +/** + * Tests importing a search definition with conflicting summary types + * + * @author bratseth + */ +public class IncorrectSummaryTypesTestCase extends AbstractSchemaTestCase { + @Test + public void testImportingIncorrect() throws ParseException { + try { + ApplicationBuilder.createFromString( + "search incorrectsummarytypes {\n" + + " document incorrectsummarytypes {\n" + + " field somestring type string {\n" + + " indexing: summary\n" + + " }\n" + + " }\n" + + " document-summary incorrect {\n" + + " summary somestring type int {\n" + + " }\n" + + " }\n" + + "}\n"); + fail("processing should have failed"); + } catch (RuntimeException e) { + assertEquals("'summary somestring type string' in 'destinations(default )' is inconsistent with 'summary somestring type int' in 'destinations(incorrect )': All declarations of the same summary field must have the same type", e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/IndexSettingsTestCase.java new file mode 100644 index 00000000000..6082372b428 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/IndexSettingsTestCase.java @@ -0,0 +1,62 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Rank settings + * + * @author bratseth + */ +public class IndexSettingsTestCase extends AbstractSchemaTestCase { + + @Test + public void testStemmingSettings() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/indexsettings.sd"); + + SDField usingDefault=(SDField) schema.getDocument().getField("usingdefault"); + assertEquals(Stemming.SHORTEST,usingDefault.getStemming(schema)); + + SDField notStemmed=(SDField) schema.getDocument().getField("notstemmed"); + assertEquals(Stemming.NONE,notStemmed.getStemming(schema)); + + SDField allStemmed=(SDField) schema.getDocument().getField("allstemmed"); + assertEquals(Stemming.SHORTEST,allStemmed.getStemming(schema)); + + SDField multiStemmed=(SDField) schema.getDocument().getField("multiplestems"); + assertEquals(Stemming.MULTIPLE, multiStemmed.getStemming(schema)); + } + + @Test + public void requireThatInterlavedFeaturesAreSetOnExtraField() throws ParseException { + ApplicationBuilder builder = ApplicationBuilder.createFromString(joinLines( + "search test {", + " document test {", + " field content type string {", + " indexing: index | summary", + " index: enable-bm25", + " }", + " }", + " field extra type string {", + " indexing: input content | index | summary", + " index: enable-bm25", + " }", + "}" + )); + Schema schema = builder.getSchema(); + Index contentIndex = schema.getIndex("content"); + assertTrue(contentIndex.useInterleavedFeatures()); + Index extraIndex = schema.getIndex("extra"); + assertTrue(extraIndex.useInterleavedFeatures()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/schema/IndexingParsingTestCase.java new file mode 100644 index 00000000000..6a51000fffe --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/IndexingParsingTestCase.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; + +/** + * Tests that indexing statements are parsed correctly. + * + * @author frodelu + */ +public class IndexingParsingTestCase extends AbstractSchemaTestCase { + + @Test + public void requireThatIndexingExpressionsCanBeParsed() throws Exception { + assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/indexing.sd")); + } + + @Test + public void requireThatParseExceptionPositionIsCorrect() throws Exception { + try { + ApplicationBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); + } catch (ParseException e) { + if (!e.getMessage().contains("at line 5, column 57.")) { + throw e; + } + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/MultipleSummariesTestCase.java new file mode 100644 index 00000000000..6d6249dc372 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/MultipleSummariesTestCase.java @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * tests importing of document containing array type fields + * + * @author bratseth + */ +public class MultipleSummariesTestCase extends AbstractSchemaTestCase { + + @Test + public void testArrayImporting() throws IOException, ParseException { + var builder = new ApplicationBuilder(new TestProperties()); + builder.addSchemaFile("src/test/examples/multiplesummaries.sd"); + builder.build(true); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/schema/NameFieldCheckTestCase.java new file mode 100644 index 00000000000..9b4b6864309 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/NameFieldCheckTestCase.java @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests that "name" is not allowed as name for a field. + * + * And that duplicate names are not allowed. + * + * @author Lars Christian Jensen + */ +public class NameFieldCheckTestCase extends AbstractSchemaTestCase { + + @Test + public void testNameField() { + try { + ApplicationBuilder.createFromString( + "search simple {\n" + + " document name-check {\n" + + " field title type string {\n" + + " indexing: summary | index\n" + + " }\n" + + " # reserved name, should trigger error\n" + + " field sddocname type string {\n" + + " indexing: index\n" + + " }\n" + + " }\n" + + "}"); + fail("Should throw exception."); + } catch (Exception expected) { + // Success + } + } + + @Test + public void testDuplicateNamesInSearchDifferentType() { + try { + ApplicationBuilder.createFromString( + "search duplicatenamesinsearch {\n" + + " document {\n" + + " field grpphotoids64 type string { }\n" + + " }\n" + + " field grpphotoids64 type array {\n" + + " indexing: input grpphotoids64 | split \" \" | for_each {\n" + + " base64decode } | attribute\n" + + " }\n" + + "}"); + fail("Should throw exception."); + } catch (Exception e) { + assertEquals("For schema 'duplicatenamesinsearch', field 'grpphotoids64': " + + "Incompatible types. Expected Array for index field 'grpphotoids64', got string.", e.getMessage()); + } + } + + @Test + public void testDuplicateNamesInDoc() { + try { + ApplicationBuilder.createFromString( + "search duplicatenamesindoc {\n" + + " document {\n" + + " field foo type int {\n" + + " indexing: attribute\n" + + " }\n" + + " field fOo type string {\n" + + " indexing: index\n" + + " }\n" + + " }\n" + + "}"); + fail("Should throw exception."); + } catch (Exception e) { + assertTrue(e.getMessage().matches(".*Duplicate.*")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/schema/OutsideTestCase.java new file mode 100644 index 00000000000..0c0684e23e3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/OutsideTestCase.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +/** + * Tests settings outside the document + * + * @author bratseth + */ +public class OutsideTestCase extends AbstractSchemaTestCase { + + @Test + public void testOutsideIndex() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); + + Index defaultIndex= schema.getIndex("default"); + assertTrue(defaultIndex.isPrefix()); + assertEquals("default.default",defaultIndex.aliasIterator().next()); + } + + @Test + public void testOutsideSummary() throws IOException, ParseException { + ApplicationBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/schema/PredicateDataTypeTestCase.java new file mode 100644 index 00000000000..dbea8fb8aeb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/PredicateDataTypeTestCase.java @@ -0,0 +1,199 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.ImmutableSDField; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import com.yahoo.document.DataType; +import com.yahoo.schema.parser.ParseException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Lester Solbakken + */ + +public class PredicateDataTypeTestCase { + + private String searchSd(String field) { + return "search p {\n document p {\n" + field + "}\n}\n"; + } + + private String predicateFieldSd(String index) { + return "field pf type predicate {\n" + index + "}\n"; + } + + private String arrayPredicateFieldSd(String index) { + return "field apf type array {\n" + index + "}\n"; + } + + private String stringFieldSd(String index) { + return "field sf type string {\n" + index + "}\n"; + } + + private String attributeFieldSd(String terms) { + return "indexing: attribute\n index {\n" + terms + "}\n"; + } + + private String arityParameter(int arity) { + return "arity: " + arity + "\n"; + } + + private String lowerBoundParameter(long bound) { + return "lower-bound: " + bound + "\n"; + } + + private String upperBoundParameter(long bound) { + return "upper-bound: " + bound + "\n"; + } + + @SuppressWarnings("deprecation") + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void requireThatBuilderSetsIndexParametersCorrectly() throws ParseException { + int arity = 2; + long lowerBound = -100; + long upperBound = 100; + String sd = searchSd( + predicateFieldSd( + attributeFieldSd( + arityParameter(arity) + + lowerBoundParameter(lowerBound) + + upperBoundParameter(upperBound)))); + + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); + for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { + if (field.getDataType() == DataType.PREDICATE) { + for (Index index : field.getIndices().values()) { + assertTrue(index.getBooleanIndexDefiniton().hasArity()); + assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); + assertTrue(index.getBooleanIndexDefiniton().hasLowerBound()); + assertEquals(lowerBound, index.getBooleanIndexDefiniton().getLowerBound()); + assertTrue(index.getBooleanIndexDefiniton().hasUpperBound()); + assertEquals(upperBound, index.getBooleanIndexDefiniton().getUpperBound()); + } + } + } + } + + @Test + public void requireThatBuilderHandlesLongValues() throws ParseException { + int arity = 2; + long lowerBound = -100000000000000000L; + long upperBound = 1000000000000000000L; + String sd = searchSd( + predicateFieldSd( + attributeFieldSd( + arityParameter(arity) + + "lower-bound: -100000000000000000L\n" + // +'L' + upperBoundParameter(upperBound)))); + + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); + for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { + if (field.getDataType() == DataType.PREDICATE) { + for (Index index : field.getIndices().values()) { + assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); + assertEquals(lowerBound, index.getBooleanIndexDefiniton().getLowerBound()); + assertEquals(upperBound, index.getBooleanIndexDefiniton().getUpperBound()); + } + } + } + } + + @Test + public void requireThatBuilderHandlesMissingParameters() throws ParseException { + String sd = searchSd( + predicateFieldSd( + attributeFieldSd( + arityParameter(2)))); + ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); + for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { + if (field.getDataType() == DataType.PREDICATE) { + for (Index index : field.getIndices().values()) { + assertTrue(index.getBooleanIndexDefiniton().hasArity()); + assertFalse(index.getBooleanIndexDefiniton().hasLowerBound()); + assertFalse(index.getBooleanIndexDefiniton().hasUpperBound()); + } + } + } + } + + @Test + public void requireThatBuilderFailsIfNoArityValue() throws ParseException { + String sd = searchSd(predicateFieldSd(attributeFieldSd(""))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Missing arity value in predicate field."); + ApplicationBuilder.createFromString(sd); + fail(); + } + + @Test + public void requireThatBuilderFailsIfBothIndexAndAttribute() throws ParseException { + String sd = searchSd(predicateFieldSd("indexing: summary | index | attribute\nindex { arity: 2 }")); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); + ApplicationBuilder.createFromString(sd); + } + + @Test + public void requireThatBuilderFailsIfIndex() throws ParseException { + String sd = searchSd(predicateFieldSd("indexing: summary | index \nindex { arity: 2 }")); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); + ApplicationBuilder.createFromString(sd); + } + + + @Test + public void requireThatBuilderFailsIfIllegalArityValue() throws ParseException { + String sd = searchSd(predicateFieldSd(attributeFieldSd(arityParameter(0)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid arity value in predicate field, must be greater than 1."); + ApplicationBuilder.createFromString(sd); + } + + @Test + public void requireThatBuilderFailsIfArityParameterExistButNotPredicateField() throws ParseException { + String sd = searchSd(stringFieldSd(attributeFieldSd(arityParameter(2)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Arity parameter is used only for predicate type fields."); + ApplicationBuilder.createFromString(sd); + } + + @Test + public void requireThatBuilderFailsIfBoundParametersExistButNotPredicateField() throws ParseException { + String sd = searchSd( + stringFieldSd( + attributeFieldSd( + lowerBoundParameter(100) + upperBoundParameter(1000)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Parameters lower-bound and upper-bound are used only for predicate type fields."); + ApplicationBuilder.createFromString(sd); + } + + @Test + public void requireThatArrayOfPredicateFails() throws ParseException { + String sd = searchSd( + arrayPredicateFieldSd( + attributeFieldSd( + arityParameter(1)))); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Collections of predicates are not allowed."); + ApplicationBuilder.createFromString(sd); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/schema/RankProfileRegistryTest.java new file mode 100644 index 00000000000..de061defb87 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileRegistryTest.java @@ -0,0 +1,58 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.model.test.TestDriver; +import com.yahoo.config.model.test.TestRoot; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import org.junit.Test; + +import java.io.File; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * @author Ulf Lilleengen + */ +public class RankProfileRegistryTest { + + private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; + + @Test + public void testRankProfileInheritance() { + TestRoot root = new TestDriver().buildModel(FilesApplicationPackage.fromFile(new File(TESTDIR))); + RankProfilesConfig left = root.getConfig(RankProfilesConfig.class, "inherit/search/cluster.inherit/left"); + RankProfilesConfig right = root.getConfig(RankProfilesConfig.class, "inherit/search/cluster.inherit/right"); + assertEquals(3, left.rankprofile().size()); + assertEquals(2, right.rankprofile().size()); + } + + @Test(expected = IllegalArgumentException.class) + public void testRankProfileDuplicateNameIsIllegal() { + Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + RankProfile barRankProfile = new RankProfile("bar", schema, rankProfileRegistry); + rankProfileRegistry.add(barRankProfile); + rankProfileRegistry.add(barRankProfile); + } + + @Test + public void testRankProfileDuplicateNameLegalForOverridableRankProfiles() { + Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + + for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { + assertNull(rankProfileRegistry.get(schema, rankProfileName).getFunctions().get("foo")); + RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, schema, rankProfileRegistry); + rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); + rankProfileRegistry.add(rankProfileWithAddedFunction); + assertNotNull(rankProfileRegistry.get(schema, rankProfileName).getFunctions().get("foo")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java new file mode 100644 index 00000000000..c66d44556ca --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -0,0 +1,436 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +import org.junit.Test; + +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests rank profiles + * + * @author bratseth + */ +public class RankProfileTestCase extends AbstractSchemaTestCase { + + @Test + public void testRankProfileInheritance() { + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + SDDocumentType document = new SDDocumentType("test"); + SDField a = document.addField("a", DataType.STRING); + a.setRankType(RankType.IDENTITY); + document.addField("b", DataType.STRING); + schema.addDocument(document); + RankProfile child = new RankProfile("child", schema, rankProfileRegistry); + child.inherit("default"); + rankProfileRegistry.add(child); + + Iterator i = child.rankSettingIterator(); + + RankProfile.RankSetting setting = i.next(); + assertEquals(RankType.IDENTITY, setting.getValue()); + assertEquals("a", setting.getFieldName()); + assertEquals(RankProfile.RankSetting.Type.RANKTYPE, setting.getType()); + + setting = i.next(); + assertEquals(RankType.DEFAULT, setting.getValue()); + } + + @Test + public void requireThatIllegalInheritanceIsChecked() throws ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "search test {", + " document test { } ", + " rank-profile p1 inherits notexist {}", + "}")); + builder.build(true); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("rank-profile 'p1' inherits 'notexist', but this is not found in schema 'test'", e.getMessage()); + } + } + + @Test + public void requireThatSelfInheritanceIsIllegal() throws ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "schema test {", + " document test { } ", + " rank-profile self inherits self {}", + "}")); + builder.build(true); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("There is a cycle in the inheritance for rank-profile 'test.self' = [test.self, test.self]", e.getMessage()); + } + } + + @Test + public void requireThatSelfInheritanceIsLegalWhenOverloading() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "schema base {", + " document base { } ", + " rank-profile self inherits default {}", + "}")); + builder.addSchema(joinLines( + "schema test {", + " document test inherits base { } ", + " rank-profile self inherits self {}", + "}")); + builder.build(true); + } + + @Test + public void requireThatSidewaysInheritanceIsImpossible() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "schema child1 {", + " document child1 {", + " field field1 type int {", + " indexing: attribute", + " }", + " }", + " rank-profile child inherits parent {", + " function function2() {", + " expression: attribute(field1) + 5", + " }", + " first-phase {", + " expression: function2() * function1()", + " }", + " summary-features {", + " function1", + " function2", + " attribute(field1)", + " }", + " }", + "}\n")); + builder.addSchema(joinLines( + "schema child2 {", + " document child2 {", + " field field1 type int {", + " indexing: attribute", + " }", + " }", + " rank-profile parent {", + " first-phase {", + " expression: function1()", + " }", + " function function1() {", + " expression: attribute(field1) + 7", + " }", + " summary-features {", + " function1", + " attribute(field1)", + " }", + " }", + "}")); + try { + builder.build(true); + fail("Sideways inheritance should have been enforced"); + } catch (IllegalArgumentException e) { + assertEquals("rank-profile 'child' inherits 'parent', but this is not found in schema 'child1'", e.getMessage()); + } + } + + @Test + public void requireThatDefaultInheritingDefaultIsIgnored() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "schema test {", + " document test { } ", + " rank-profile default inherits default {}", + "}")); + builder.build(true); + } + + @Test + public void requireThatCyclicInheritanceIsIllegal() throws ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "search test {", + " document test { } ", + " rank-profile a inherits b {}", + " rank-profile b inherits c {}", + " rank-profile c inherits a {}", + "}")); + builder.build(true); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("There is a cycle in the inheritance for rank-profile 'test.c' = [test.c, test.a, test.b, test.c]", e.getMessage()); + } + } + + @Test + public void requireThatRankProfilesCanInheritNotYetSeenProfiles() throws ParseException + { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "search test {", + " document test { } ", + " rank-profile p1 inherits not_yet_defined {}", + " rank-profile not_yet_defined {}", + "}")); + builder.build(true); + assertNotNull(registry.get("test","p1")); + assertTrue(registry.get("test","p1").inherits("not_yet_defined")); + assertNotNull(registry.get("test","not_yet_defined")); + } + + private String createSD(Double termwiseLimit) { + return joinLines( + "search test {", + " document test { ", + " field a type string { ", + " indexing: index ", + " }", + " }", + " ", + " rank-profile parent {", + (termwiseLimit != null ? (" termwise-limit:" + termwiseLimit + "\n") : ""), + " num-threads-per-search:8", + " min-hits-per-thread:70", + " num-search-partitions:1200", + " }", + " rank-profile child inherits parent { }", + "}"); + } + + @Test + public void testTermwiseLimitWithDeployOverride() throws ParseException { + verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties(), createSD(null), null); + verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties(), createSD(0.78), 0.78); + verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties().setDefaultTermwiseLimit(0.09), createSD(null), 0.09); + verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties().setDefaultTermwiseLimit(0.09), createSD(0.37), 0.37); + } + + private void verifyTermwiseLimitAndSomeMoreIncludingInheritance(ModelContext.Properties deployProperties, String sd, Double termwiseLimit) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(sd); + builder.build(true); + Schema schema = builder.getSchema(); + AttributeFields attributeFields = new AttributeFields(schema); + verifyRankProfile(rankProfileRegistry.get(schema, "parent"), attributeFields, deployProperties, termwiseLimit); + verifyRankProfile(rankProfileRegistry.get(schema, "child"), attributeFields, deployProperties, termwiseLimit); + } + + private void verifyRankProfile(RankProfile rankProfile, AttributeFields attributeFields, ModelContext.Properties deployProperties, + Double expectedTermwiseLimit) { + assertEquals(8, rankProfile.getNumThreadsPerSearch()); + assertEquals(70, rankProfile.getMinHitsPerThread()); + assertEquals(1200, rankProfile.getNumSearchPartitions()); + RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), + new ImportedMlModels(), attributeFields, deployProperties); + if (expectedTermwiseLimit != null) { + assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent()); + assertEquals(String.valueOf(expectedTermwiseLimit), findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").get()); + } else { + assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent()); + } + assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").isPresent()); + assertEquals("8", findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").get()); + assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.minhitsperthread").isPresent()); + assertEquals("70", findProperty(rawRankProfile.configProperties(), "vespa.matching.minhitsperthread").get()); + assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numsearchpartitions").isPresent()); + assertEquals("1200", findProperty(rawRankProfile.configProperties(), "vespa.matching.numsearchpartitions").get()); + } + + @Test + public void requireThatConfigIsDerivedForAttributeTypeSettings() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10]) { indexing: attribute }", + " field b type tensor(y{}) { indexing: attribute }", + " field c type tensor(x[5]) { indexing: attribute }", + " }", + " rank-profile p1 {}", + " rank-profile p2 {}", + "}")); + builder.build(true); + Schema schema = builder.getSchema(); + + assertEquals(4, registry.all().size()); + assertAttributeTypeSettings(registry.get(schema, "default"), schema); + assertAttributeTypeSettings(registry.get(schema, "unranked"), schema); + assertAttributeTypeSettings(registry.get(schema, "p1"), schema); + assertAttributeTypeSettings(registry.get(schema, "p2"), schema); + } + + @Test + public void requireThatDenseDimensionsMustBeBound() throws ParseException { + try { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[]) { indexing: attribute }", + " }", + "}")); + builder.build(true); + } + catch (IllegalArgumentException e) { + assertEquals("Illegal type in field a type tensor(x[]): Dense tensor dimensions must have a size", + e.getMessage()); + } + } + + private static RawRankProfile createRawRankProfile(RankProfile profile, Schema schema) { + return new RawRankProfile(profile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(schema), new TestProperties()); + } + + private static void assertAttributeTypeSettings(RankProfile profile, Schema schema) { + RawRankProfile rawProfile = createRawRankProfile(profile, schema); + assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.a").get()); + assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.attribute.b").get()); + assertEquals("tensor(x[5])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); + } + + @Test + public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); + builder.addSchema(joinLines( + "search test {", + " document test { } ", + " rank-profile p1 {}", + " rank-profile p2 {}", + "}")); + builder.build(true); + Schema schema = builder.getSchema(); + + assertEquals(4, registry.all().size()); + assertQueryFeatureTypeSettings(registry.get(schema, "default"), schema); + assertQueryFeatureTypeSettings(registry.get(schema, "unranked"), schema); + assertQueryFeatureTypeSettings(registry.get(schema, "p1"), schema); + assertQueryFeatureTypeSettings(registry.get(schema, "p2"), schema); + } + + private static QueryProfileRegistry setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(tensor1)", + FieldType.fromString("tensor(x[10])", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(tensor2)", + FieldType.fromString("tensor(y{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.invalid(tensor3)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(numeric)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + var profile = new QueryProfile(new ComponentId("testprofile")); + profile.setType(type); + registry.register(profile); + return registry; + } + + private static void assertQueryFeatureTypeSettings(RankProfile profile, Schema schema) { + RawRankProfile rawProfile =createRawRankProfile(profile, schema); + assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor1").get()); + assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor2").get()); + assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.query.tensor3").isPresent()); + assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.query.numeric").isPresent()); + } + + private static Optional findProperty(List> properties, String key) { + for (Pair property : properties) + if (property.getFirst().equals(key)) + return Optional.of(property.getSecond()); + return Optional.empty(); + } + + @Test + public void approximate_nearest_neighbor_threshold_settings_are_configurable() throws ParseException { + verifyApproximateNearestNeighborThresholdSettings(0.7, null); + verifyApproximateNearestNeighborThresholdSettings(null, 0.3); + verifyApproximateNearestNeighborThresholdSettings(0.7, 0.3); + } + + private void verifyApproximateNearestNeighborThresholdSettings(Double postFilterThreshold, Double approximateThreshold) throws ParseException { + var rankProfileRegistry = new RankProfileRegistry(); + var props = new TestProperties(); + var queryProfileRegistry = new QueryProfileRegistry(); + var builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry, props); + builder.addSchema(createSDWithRankProfileThresholds(postFilterThreshold, approximateThreshold)); + builder.build(true); + + var schema = builder.getSchema(); + var rankProfile = rankProfileRegistry.get(schema, "my_profile"); + var rawRankProfile = new RawRankProfile(rankProfile, new LargeRankExpressions(new MockFileRegistry()), queryProfileRegistry, + new ImportedMlModels(), new AttributeFields(schema), props); + + if (postFilterThreshold != null) { + assertEquals((double)postFilterThreshold, rankProfile.getPostFilterThreshold().getAsDouble(), 0.000001); + assertEquals(String.valueOf(postFilterThreshold), findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.upper_limit").get()); + } else { + assertTrue(rankProfile.getPostFilterThreshold().isEmpty()); + assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.upper_limit").isPresent()); + } + + if (approximateThreshold != null) { + assertEquals((double)approximateThreshold, rankProfile.getApproximateThreshold().getAsDouble(), 0.000001); + assertEquals(String.valueOf(approximateThreshold), findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.lower_limit").get()); + } else { + assertTrue(rankProfile.getApproximateThreshold().isEmpty()); + assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.lower_limit").isPresent()); + } + } + + private String createSDWithRankProfileThresholds(Double postFilterThreshold, Double approximateThreshold) { + return joinLines( + "search test {", + " document test {}", + " rank-profile my_profile {", + (postFilterThreshold != null ? (" post-filter-threshold: " + postFilterThreshold) : ""), + (approximateThreshold != null ? (" approximate-threshold: " + approximateThreshold) : ""), + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java new file mode 100644 index 00000000000..c3595717220 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java @@ -0,0 +1,163 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.util.List; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class RankPropertiesTestCase extends AbstractSchemaTestCase { + + @Test + public void testRankPropertyInheritance() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( + "search test {", + " document test {", + " field a type string { ", + " indexing: index ", + " }", + " }", + " rank-profile parent {", + " first-phase {", + " expression: a", + " }", + " rank-properties {", + " query(a): 1500 ", + " }", + " }", + " rank-profile child inherits parent {", + " first-phase {", + " expression: a", + " }", + " rank-properties {", + " query(a): 2000 ", + " }", + " }", + "}")); + builder.build(true); + Schema schema = builder.getSchema(); + AttributeFields attributeFields = new AttributeFields(schema); + + { + // Check declared model + RankProfile parent = rankProfileRegistry.get(schema, "parent"); + assertEquals("query(a) = 1500", parent.getRankProperties().get(0).toString()); + + // Check derived model + RawRankProfile rawParent = new RawRankProfile(parent, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), attributeFields, new TestProperties()); + assertEquals("(query(a), 1500)", rawParent.configProperties().get(0).toString()); + } + + { + // Check declared model + RankProfile parent = rankProfileRegistry.get(schema, "child"); + assertEquals("query(a) = 2000", parent.getRankProperties().get(0).toString()); + + // Check derived model + RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.get(schema, "child"), + new LargeRankExpressions(new MockFileRegistry()), + new QueryProfileRegistry(), + new ImportedMlModels(), + attributeFields, + new TestProperties()); + assertEquals("(query(a), 2000)", rawChild.configProperties().get(0).toString()); + } + } + @Test + public void testRankProfileMutate() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( + "search test {", + " document test {", + " field a type int { ", + " indexing: attribute ", + " }", + " }", + " field synthetic_attribute_a type int {", + " indexing: attribute", + " attribute: mutable", + " }", + " field synthetic_attribute_b type double {", + " indexing: attribute", + " attribute: mutable", + " }", + " field synthetic_attribute_c type long {", + " indexing: attribute", + " attribute: mutable", + " }", + " rank-profile a {", + " mutate {", + " on-match {", + " synthetic_attribute_a += 7", + " }", + " on-first-phase {", + " synthetic_attribute_b +=1", + " }", + " on-second-phase {", + " synthetic_attribute_b = 1.01", + " }", + " on-summary {", + " synthetic_attribute_c -= 1", + " }", + " }", + " first-phase {", + " expression: a", + " }", + " }", + " rank-profile b {", + " first-phase {", + " expression: a", + " }", + " second-phase {", + " expression: a", + " }", + " }", + "}")); + builder.build(true); + Schema schema = builder.getSchema(); + RankProfile a = rankProfileRegistry.get(schema, "a"); + List operations = a.getMutateOperations(); + assertEquals(4, operations.size()); + assertEquals(RankProfile.MutateOperation.Phase.on_match, operations.get(0).phase); + assertEquals("synthetic_attribute_a", operations.get(0).attribute); + assertEquals("+=7", operations.get(0).operation); + assertEquals(RankProfile.MutateOperation.Phase.on_first_phase, operations.get(1).phase); + assertEquals("synthetic_attribute_b", operations.get(1).attribute); + assertEquals("+=1", operations.get(1).operation); + assertEquals(RankProfile.MutateOperation.Phase.on_second_phase, operations.get(2).phase); + assertEquals("synthetic_attribute_b", operations.get(2).attribute); + assertEquals("=1.01", operations.get(2).operation); + assertEquals(RankProfile.MutateOperation.Phase.on_summary, operations.get(3).phase); + assertEquals("synthetic_attribute_c", operations.get(3).attribute); + assertEquals("-=1", operations.get(3).operation); + + AttributeFields attributeFields = new AttributeFields(schema); + RawRankProfile raw = new RawRankProfile(a, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), attributeFields, new TestProperties()); + assertEquals(9, raw.configProperties().size()); + assertEquals("(vespa.mutate.on_match.attribute, synthetic_attribute_a)", raw.configProperties().get(0).toString()); + assertEquals("(vespa.mutate.on_match.operation, +=7)", raw.configProperties().get(1).toString()); + assertEquals("(vespa.mutate.on_first_phase.attribute, synthetic_attribute_b)", raw.configProperties().get(2).toString()); + assertEquals("(vespa.mutate.on_first_phase.operation, +=1)", raw.configProperties().get(3).toString()); + assertEquals("(vespa.mutate.on_second_phase.attribute, synthetic_attribute_b)", raw.configProperties().get(4).toString()); + assertEquals("(vespa.mutate.on_second_phase.operation, =1.01)", raw.configProperties().get(5).toString()); + assertEquals("(vespa.mutate.on_summary.attribute, synthetic_attribute_c)", raw.configProperties().get(6).toString()); + assertEquals("(vespa.mutate.on_summary.operation, -=1)", raw.configProperties().get(7).toString()); + assertEquals("(vespa.rank.firstphase, a)", raw.configProperties().get(8).toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/schema/RankingConstantTest.java new file mode 100644 index 00000000000..883e6b50abb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankingConstantTest.java @@ -0,0 +1,213 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public class RankingConstantTest { + + @SuppressWarnings("deprecation") + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void tensor_constant_properties_are_set() throws Exception { + final String TENSOR_NAME = "my_global_tensor"; + final String TENSOR_FILE = "path/my-tensor-file.json"; + final String TENSOR_TYPE = "tensor(x{})"; + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(constant(my_global_tensor))", + " }", + " }", + " constant " + TENSOR_NAME + " {", + " file: " + TENSOR_FILE, + " type: " + TENSOR_TYPE, + " }", + "}" + )); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + + Iterator constantIterator = schema.constants().values().iterator(); + RankProfile.Constant constant = constantIterator.next(); + assertEquals(TENSOR_NAME, constant.name().simpleArgument().get()); + assertEquals(TENSOR_FILE, constant.valuePath().get()); + assertEquals(TENSOR_TYPE, constant.type().toString()); + assertEquals(DistributableResource.PathType.FILE, constant.pathType().get()); + + assertFalse(constantIterator.hasNext()); + } + + @Test + public void tensor_constant_must_have_a_type() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("must have a type"); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " file: bar.baz", + " }", + "}" + )); + } + + @Test + public void tensor_constant_must_have_a_file() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("must have a file"); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x[])", + " }", + "}" + )); + } + + @Test + public void constant_file_does_not_need_path_or_ending() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " file: simplename", + " }", + "}" + )); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals("simplename", constant.valuePath().get()); + } + + @Test + public void constant_uri_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: http://somewhere.far.away/in/another-galaxy", + " }", + "}" + )); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("http://somewhere.far.away/in/another-galaxy", constant.valuePath().get()); + } + + @Test + public void constant_https_uri_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: https://somewhere.far.away:4443/in/another-galaxy", + " }", + "}" + )); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("https://somewhere.far.away:4443/in/another-galaxy", constant.valuePath().get()); + } + + @Test + public void constant_uri_with_port_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: http://somewhere.far.away:4080/in/another-galaxy", + " }", + "}" + )); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("http://somewhere.far.away:4080/in/another-galaxy", constant.valuePath().get()); + } + + @Test + public void constant_uri_no_dual_slashes_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: http:somewhere.far.away/in/another-galaxy", + " }", + "}" + )); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("http:somewhere.far.away/in/another-galaxy", constant.valuePath().get()); + } + + @Test + public void constant_uri_only_supports_http_and_https() { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); + String expectedMessage = "Encountered \" \"ftp\"\" at line 5, column 10.\n\n" + + "Was expecting:\n\n" + + " ..."; + try { + schemaBuilder.addSchema(joinLines( + "schema test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: ftp:somewhere.far.away/in/another-galaxy", + " }", + "}" + )); + } catch (ParseException e) { + if (! e.getMessage().startsWith(expectedMessage)) + fail("Expected exception with message starting with:\n'" + expectedMessage + "\nBut got:\n'" + e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionConstantsTestCase.java new file mode 100644 index 00000000000..bd0bd65295c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionConstantsTestCase.java @@ -0,0 +1,229 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.collections.Pair; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.yolean.Exceptions; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { + + @Test + public void testConstants() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "schema test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " constants {\n" + + " p1 double: 7 \n" + + " constant(p2) double: 0 \n" + + " }\n" + + " first-phase {\n" + + " expression: p2 * (1.3 + p1 )\n" + + " }\n" + + " }\n" + + " rank-profile child1 inherits parent {\n" + + " first-phase {\n" + + " expression: a + b + c \n" + + " }\n" + + " second-phase {\n" + + " expression: a + p1 + c \n" + + " }\n" + + " constants {\n" + + " a: 1.0 \n" + + " constant(b): 2 \n" + + " c: 3.5 \n" + + " }\n" + + " }\n" + + " rank-profile child2 inherits parent {\n" + + " constants {\n" + + " p2: 2.0 \n" + + " }\n" + + " function foo() {\n" + + " expression: p2*p1\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile parent = rankProfileRegistry.get(s, "parent").compile(queryProfileRegistry, new ImportedMlModels()); + assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); + + RankProfile child1 = rankProfileRegistry.get(s, "child1").compile(queryProfileRegistry, new ImportedMlModels()); + assertEquals("6.5", child1.getFirstPhaseRanking().getRoot().toString()); + assertEquals("11.5", child1.getSecondPhaseRanking().getRoot().toString()); + + RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedMlModels()); + assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); + assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); + List> rankProperties = new RawRankProfile(child2, + new LargeRankExpressions(new MockFileRegistry()), + queryProfileRegistry, + new ImportedMlModels(), + new AttributeFields(s), + new TestProperties()).configProperties(); + assertEquals("(rankingExpression(foo).rankingScript, 14.0)", rankProperties.get(0).toString()); + assertEquals("(rankingExpression(firstphase).rankingScript, 16.6)", rankProperties.get(2).toString()); + } + + @Test + public void testNameCollision() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "schema test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " constants {\n" + + " c: 7 \n" + + " }\n" + + " function c() {\n" + + " expression: p2*p1\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + try { + rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); + fail("Should have caused an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testNegativeLiteralArgument() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function POP_SLOW_SCORE() {\n" + + " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile profile = rankProfileRegistry.get(s, "test"); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); + } + + @Test + public void testNegativeConstantArgument() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "schema test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " constants {\n" + + " myValue: -9.21034037\n" + + " }\n" + + " function POP_SLOW_SCORE() {\n" + + " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile profile = rankProfileRegistry.get(s, "test"); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", + profile.compile(new QueryProfileRegistry(), new ImportedMlModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); + } + + @Test + public void testConstantDivisorInFunction() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function rank_default(){\n" + + " expression: k1 + (k2 + k3) / 100000000.0\n\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile profile = rankProfileRegistry.get(s, "test"); + assertEquals("k1 + (k2 + k3) / 1.0E8", + profile.compile(new QueryProfileRegistry(), new ImportedMlModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); + } + + @Test + public void test3() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field rating_yelp type int {" + + " indexing: attribute" + + " }" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function rank_default(){\n" + + " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile profile = rankProfileRegistry.get(s, "test"); + assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", + profile.compile(new QueryProfileRegistry(), new ImportedMlModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java new file mode 100644 index 00000000000..0695e20d780 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java @@ -0,0 +1,274 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Optional; +import java.util.logging.Level; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { + + @Test + public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type double { \n" + + " indexing: attribute \n" + + " }\n" + + " field b type double { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " constants {\n" + + " p1: 7 \n" + + " p2: 0 \n" + + " }\n" + + " first-phase {\n" + + " expression: p1 * add\n" + + " }\n" + + " function inline add() {\n" + + " expression: 3 + attribute(a) + attribute(b) * mul3\n" + + " }\n" + + " function inline mul3() {\n" + + " expression: attribute(a) * 3 + singleif\n" + + " }\n" + + " function inline singleif() {\n" + + " expression: if (p1 < attribute(a), 1, 2) == 0\n" + + " }\n" + + " }\n" + + " rank-profile child inherits parent {\n" + + " function inline add() {\n" + + " expression: 9 + attribute(a)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + + RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); + assertEquals("7.0 * (3 + attribute(a) + attribute(b) * (attribute(a) * 3 + if (7.0 < attribute(a), 1, 2) == 0))", + parent.getFirstPhaseRanking().getRoot().toString()); + RankProfile child = rankProfileRegistry.get(s, "child").compile(new QueryProfileRegistry(), new ImportedMlModels()); + assertEquals("7.0 * (9 + attribute(a))", + child.getFirstPhaseRanking().getRoot().toString()); + } + + @Test + public void testConstants() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile parent {\n" + + " constants {\n" + + " p1: 7 \n" + + " p2: 0 \n" + + " }\n" + + " first-phase {\n" + + " expression: p1 + foo\n" + + " }\n" + + " second-phase {\n" + + " expression: p2 * foo\n" + + " }\n" + + " function inline foo() {\n" + + " expression: 3 + p1 + p2\n" + + " }\n" + + " }\n" + + " rank-profile child inherits parent {\n" + + " first-phase {\n" + + " expression: p1 + foo + baz + bar + arg(4.0)\n" + + " }\n" + + " constants {\n" + + " p2: 2.0 \n" + + " }\n" + + " function bar() {\n" + + " expression: p2*p1\n" + + " }\n" + + " function inline baz() {\n" + + " expression: p2+p1+boz\n" + + " }\n" + + " function inline boz() {\n" + + " expression: 3.0\n" + + " }\n" + + " function inline arg(a1) {\n" + + " expression: a1*2\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + + RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); + assertEquals("17.0", parent.getFirstPhaseRanking().getRoot().toString()); + assertEquals("0.0", parent.getSecondPhaseRanking().getRoot().toString()); + assertEquals("10.0", getRankingExpression("foo", parent, s)); + assertEquals("17.0", getRankingExpression("firstphase", parent, s)); + assertEquals("0.0", getRankingExpression("secondphase", parent, s)); + + RankProfile child = rankProfileRegistry.get(s, "child").compile(new QueryProfileRegistry(), new ImportedMlModels()); + assertEquals("31.0 + bar + arg(4.0)", child.getFirstPhaseRanking().getRoot().toString()); + assertEquals("24.0", child.getSecondPhaseRanking().getRoot().toString()); + assertEquals("12.0", getRankingExpression("foo", child, s)); + assertEquals("12.0", getRankingExpression("baz", child, s)); + assertEquals("3.0", getRankingExpression("boz", child, s)); + assertEquals("14.0", getRankingExpression("bar", child, s)); + assertEquals("a1 * 2", getRankingExpression("arg", child, s)); + assertEquals("31.0 + rankingExpression(bar) + rankingExpression(arg@)", getRankingExpression("firstphase", child, s)); + assertEquals("24.0", getRankingExpression("secondphase", child, s)); + } + + @Test + public void testNonTopLevelInlining() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type double { \n" + + " indexing: attribute \n" + + " }\n" + + " field b type double { \n" + + " indexing: attribute \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: A + C + D\n" + + " }\n" + + " function inline D() {\n" + + " expression: B + 1\n" + + " }\n" + + " function C() {\n" + + " expression: A + B\n" + + " }\n" + + " function inline B() {\n" + + " expression: attribute(b)\n" + + " }\n" + + " function inline A() {\n" + + " expression: attribute(a)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + + RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); + assertEquals("attribute(a) + C + (attribute(b) + 1)", test.getFirstPhaseRanking().getRoot().toString()); + assertEquals("attribute(a) + attribute(b)", getRankingExpression("C", test, s)); + assertEquals("attribute(b) + 1", getRankingExpression("D", test, s)); + } + + @Test + public void testFunctionInliningWithReplacement() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + MockDeployLogger deployLogger = new MockDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + deployLogger, + new TestProperties(), + rankProfileRegistry, + new QueryProfileRegistry()); + builder.addSchema( + "search test {\n" + + " document test { }\n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo\n" + + " }\n" + + " function foo(x) {\n" + + " expression: x + x\n" + + " }\n" + + " function inline foo() {\n" + // replaces previous "foo" during parsing + " expression: foo(2)\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); + assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); + assertTrue("Does not contain expected warning", + deployLogger.contains("Function 'foo' is defined twice in rank profile 'test'")); + } + + /** + * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of + * each function for each binding, using hashes to name the bound variants of the function. + * This method censors those hashes for string comparison. + */ + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + + if ( ! areInHash) + b.append(current); + + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + + private String getRankingExpression(String name, RankProfile rankProfile, Schema schema) { + Optional rankExpression = + new RawRankProfile(rankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(schema), new TestProperties()) + .configProperties() + .stream() + .filter(r -> r.getFirst().equals("rankingExpression(" + name + ").rankingScript")) + .map(Pair::getSecond) + .findFirst(); + assertTrue(rankExpression.isPresent()); + return censorBindingHash(rankExpression.get()); + } + + private static class MockDeployLogger implements DeployLogger { + private final ArrayList msgs = new ArrayList<>(); + + @Override + public void log(Level level, String message) { + msgs.add(message); + } + + public boolean contains(String expected) { + return msgs.stream().anyMatch(msg -> msg.equals(expected)); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionLoopDetectionTestCase.java new file mode 100644 index 00000000000..dd69fb6c591 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionLoopDetectionTestCase.java @@ -0,0 +1,246 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class RankingExpressionLoopDetectionTestCase { + + @Test + public void testSelfLoop() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo\n" + + " }\n" + + " function foo() {\n" + + " expression: foo\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + try { + builder.build(true); + fail("Excepted exception"); + } + catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank profile 'test': The function 'foo' is invalid: foo is invalid: Invocation loop: foo -> foo", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testNestedLoop() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo\n" + + " }\n" + + " function foo() {\n" + + " expression: arg(5)\n" + + " }\n" + + " function arg(a1) {\n" + + " expression: foo + a1*2\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + try { + builder.build(true); + fail("Excepted exception"); + } + catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank profile 'test': The function 'foo' is invalid: arg(5) is invalid: foo is invalid: arg(5) is invalid: Invocation loop: arg(5) -> foo -> arg(5)", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testSelfArgumentLoop() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo\n" + + " }\n" + + " function foo() {\n" + + " expression: arg(foo)\n" + + " }\n" + + " function arg(a1) {\n" + + " expression: a1*2\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + try { + builder.build(true); + fail("Excepted exception"); + } + catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank profile 'test': The function 'foo' is invalid: arg(foo) is invalid: a1 is invalid: foo is invalid: arg(foo) is invalid: Invocation loop: arg(foo) -> foo -> arg(foo)", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testNoLoopWithSameLocalArgument() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo(3)\n" + + " }\n" + + " function foo(a1) {\n" + + " expression: bar(3)\n" + + " }\n" + + " function bar(a1) {\n" + + " expression: a1*2\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + } + + @Test + public void testNoLoopWithMultipleInvocations() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo(3)\n" + + " }\n" + + " function foo(a1) {\n" + + " expression: bar(3) + bar(a1)\n" + + " }\n" + + " function bar(a1) {\n" + + " expression: a1*2\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + } + + @Test + public void testNoLoopWithBoundIdentifiers() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " }\n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo(bar(2))\n" + + " }\n" + + " function foo(x) {\n" + + " expression: x * x\n" + + " }\n" + + " function bar(x) {\n" + + " expression: x + x\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + } + + @Test + public void testNoLoopWithTheSameNestedIdentifierWhichIsUnbound() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " }\n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo()\n" + + " }\n" + + " function foo() {\n" + + " expression: bar(x)\n" + + " }\n" + + " function bar(x) {\n" + + " expression: x + x\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + } + + @Test + public void testNoLoopWithTheSameAlternatingNestedIdentifierWhichIsUnbound() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " }\n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo()\n" + + " }\n" + + " function foo() {\n" + + " expression: bar(x)\n" + + " }\n" + + " function bar(y) {\n" + + " expression: baz(y)\n" + + " }\n" + + " function baz(x) {\n" + + " expression: x + x\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionShadowingTestCase.java new file mode 100644 index 00000000000..250879b1570 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionShadowingTestCase.java @@ -0,0 +1,251 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.collections.Pair; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author lesters + */ +public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { + + @Test + public void testBasicFunctionShadowing() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function sin(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sin(2)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); + List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); + assertEquals("(rankingExpression(sin@).rankingScript, 2 * 2)", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(sin).rankingScript, x * x)", + testRankProperties.get(1).toString()); + assertEquals("(vespa.rank.firstphase, rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(2).toString())); + } + + + @Test + public void testMultiLevelFunctionShadowing() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function tan(x) {\n" + + " expression: x * x\n" + + " }\n" + + " function cos(x) {\n" + + " expression: tan(x)\n" + + " }\n" + + " function sin(x) {\n" + + " expression: cos(x)\n" + + " }\n" + + " first-phase {\n" + + " expression: sin(2)\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); + List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); + assertEquals("(rankingExpression(tan@).rankingScript, 2 * 2)", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(cos@).rankingScript, rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin@).rankingScript, rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(rankingExpression(tan).rankingScript, x * x)", + testRankProperties.get(3).toString()); + assertEquals("(rankingExpression(tan@).rankingScript, x * x)", + censorBindingHash(testRankProperties.get(4).toString())); + assertEquals("(rankingExpression(cos).rankingScript, rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(5).toString())); + assertEquals("(rankingExpression(cos@).rankingScript, rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(6).toString())); + assertEquals("(rankingExpression(sin).rankingScript, rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(7).toString())); + assertEquals("(vespa.rank.firstphase, rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(8).toString())); + } + + @Test + public void testFunctionShadowingArguments() throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function sin(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: cos(sin(2*2)) + sin(cos(1+4))\n" + + " }\n" + + " }\n" + + "\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); + List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); + assertEquals("(rankingExpression(sin@).rankingScript, 4.0 * 4.0)", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(sin@).rankingScript, cos(5.0) * cos(5.0))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin).rankingScript, x * x)", + testRankProperties.get(2).toString()); + assertEquals("(vespa.rank.firstphase, rankingExpression(firstphase))", + censorBindingHash(testRankProperties.get(3).toString())); + assertEquals("(rankingExpression(firstphase).rankingScript, cos(rankingExpression(sin@)) + rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(4).toString())); + } + + @Test + public void testNeuralNetworkSetup() throws ParseException { + // Note: the type assigned to query profile and constant tensors here is not the correct type + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(input[1])"); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test {\n" + + " function relu(x) {\n" + // relu is a built in function, redefined here + " expression: max(1.0, x)\n" + + " }\n" + + " function hidden_layer() {\n" + + " expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" + + " }\n" + + " function final_layer() {\n" + + " expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" + + " }\n" + + " second-phase {\n" + + " expression: sum(final_layer)\n" + + " }\n" + + " }\n" + + " constant W_hidden {\n" + + " type: tensor(hidden[1])\n" + + " file: ignored.json\n" + + " }\n" + + " constant b_input {\n" + + " type: tensor(hidden[1])\n" + + " file: ignored.json\n" + + " }\n" + + " constant W_final {\n" + + " type: tensor(final[1])\n" + + " file: ignored.json\n" + + " }\n" + + " constant b_final {\n" + + " type: tensor(final[1])\n" + + " file: ignored.json\n" + + " }\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); + List> testRankProperties = createRawRankProfile(test, queryProfiles, s).configProperties(); + assertEquals("(rankingExpression(autogenerated_ranking_feature@).rankingScript, reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input))", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(relu@).rankingScript, max(1.0,rankingExpression(autogenerated_ranking_feature@)))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(hidden_layer).rankingScript, rankingExpression(relu@))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(rankingExpression(final_layer).rankingScript, sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", + testRankProperties.get(4).toString()); + assertEquals("(rankingExpression(relu).rankingScript, max(1.0,x))", + testRankProperties.get(6).toString()); + assertEquals("(vespa.rank.secondphase, rankingExpression(secondphase))", + testRankProperties.get(7).toString()); + assertEquals("(rankingExpression(secondphase).rankingScript, reduce(rankingExpression(final_layer), sum))", + testRankProperties.get(8).toString()); + } + + private static RawRankProfile createRawRankProfile(RankProfile profile, QueryProfileRegistry queryProfiles, Schema schema) { + return new RawRankProfile(profile, + new LargeRankExpressions(new MockFileRegistry()), + queryProfiles, + new ImportedMlModels(), + new AttributeFields(schema), + new TestProperties()); + } + + private QueryProfileRegistry queryProfileWith(String field, String type) { + QueryProfileType queryProfileType = new QueryProfileType("root"); + queryProfileType.addField(new FieldDescription(field, type)); + QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); + queryProfileRegistry.getTypeRegistry().register(queryProfileType); + QueryProfile profile = new QueryProfile("default"); + profile.setType(queryProfileType); + queryProfileRegistry.register(profile); + return queryProfileRegistry; + } + + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + if ( ! areInHash) + b.append(current); + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionValidationTestCase.java new file mode 100644 index 00000000000..e42acee9bed --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionValidationTestCase.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +/** + * @author bratseth + */ +public class RankingExpressionValidationTestCase extends AbstractSchemaTestCase { + + @Test + public void testInvalidExpressionProducesException() throws ParseException { + assertFailsExpression("&/%(/%&"); + assertFailsExpression("if(a==b,b)"); + } + + private void assertFailsExpression(String expression) throws ParseException { + try { + RankProfileRegistry registry = new RankProfileRegistry(); + Schema schema = importWithExpression(expression, registry); + new DerivedConfiguration(schema, registry); // cause rank profile parsing + fail("No exception on incorrect ranking expression " + expression); + } catch (IllegalArgumentException e) { + // Success + assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); + } + } + + private Schema importWithExpression(String expression, RankProfileRegistry registry) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(registry); + builder.addSchema("search test {" + + " document test { " + + " field a type string { " + + " indexing: index " + + " }" + + " }" + + " rank-profile default {" + + " first-phase {" + + " expression: " + expression + + " }" + + " }" + + "}"); + builder.build(true); + return builder.getSchema(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/ReservedWordsAsFieldNamesTestCase.java new file mode 100644 index 00000000000..df9d4a63650 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/ReservedWordsAsFieldNamesTestCase.java @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +/** + * @author bratseth + */ +public class ReservedWordsAsFieldNamesTestCase extends AbstractSchemaTestCase { + + @Test + public void testIt() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/reserved_words_as_field_names.sd"); + assertNotNull(schema.getDocument().getField("inline")); + assertNotNull(schema.getDocument().getField("constants")); + assertNotNull(schema.getDocument().getField("reference")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java new file mode 100755 index 00000000000..45780d39021 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDDocumentType; +import com.yahoo.schema.document.TemporarySDField; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author Einar M R Rosenvinge + */ +public class SDDocumentTypeOrdererTestCase { + + @Test + public void testOrder() { + List types = new ArrayList<>(); + + SDDocumentType a = new SDDocumentType("a"); + SDDocumentType b = new SDDocumentType("b"); + SDDocumentType c = new SDDocumentType("c"); + SDDocumentType d = new SDDocumentType("d"); + SDDocumentType e = new SDDocumentType("e"); + SDDocumentType f = new SDDocumentType("f"); + SDDocumentType g = new SDDocumentType("g"); + b.inherit(new TemporarySDDocumentType(new DataTypeName("a"))); + c.inherit(new TemporarySDDocumentType(new DataTypeName("b"))); + d.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); + g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); + g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); + + SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); + a.addField(aFieldTypeB); + + SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); + b.addField(bFieldTypeC); + + SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); + c.addField(cFieldTypeG); + + SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); + g.addField(gFieldTypeF); + + SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); + f.addField(fFieldTypeC); + + SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); + d.addField(dFieldTypeE); + + types.add(a); + types.add(b); + types.add(c); + types.add(d); + types.add(e); + types.add(f); + types.add(g); + + SDDocumentTypeOrderer app = new SDDocumentTypeOrderer(types, new BaseDeployLogger()); + app.process(); + assertEquals(7, app.processingOrder.size()); + assertEquals(a, app.processingOrder.get(0)); + assertEquals(b, app.processingOrder.get(1)); + assertEquals(c, app.processingOrder.get(2)); + assertEquals(e, app.processingOrder.get(3)); + assertEquals(d, app.processingOrder.get(4)); + assertEquals(f, app.processingOrder.get(5)); + assertEquals(g, app.processingOrder.get(6)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java new file mode 100644 index 00000000000..e93dd0e0a8f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -0,0 +1,189 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.MakeAliases; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static com.google.common.collect.testing.Helpers.assertEmpty; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests importing of search definitions + * + * @author bratseth + */ +public class SchemaImporterTestCase extends AbstractSchemaTestCase { + + @Test + @SuppressWarnings("deprecation") + public void testSimpleImporting() throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); + sb.addSchemaFile("src/test/examples/simple.sd"); + sb.build(true); + Schema schema = sb.getSchema(); + assertEquals("simple", schema.getName()); + assertTrue(schema.hasDocument()); + + SDDocumentType document = schema.getDocument(); + assertEquals("simple", document.getName()); + assertEquals(23, document.getFieldCount()); + + SDField field; + Attribute attribute; + + new MakeAliases(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); + + // First field + field = (SDField) document.getField("title"); + assertEquals(DataType.STRING, field.getDataType()); + assertEquals("{ input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", field.getIndexingScript().toString()); + assertFalse(schema.getIndex("default").isPrefix()); + assertTrue(schema.getIndex("title").isPrefix()); + Iterator titleAliases = schema.getIndex("title").aliasIterator(); + assertEquals("aliaz", titleAliases.next()); + assertEquals("analias.totitle", titleAliases.next()); + assertEquals("analias.todefault", + schema.getIndex("default").aliasIterator().next()); + assertEquals(RankType.IDENTITY, field.getRankType()); + assertEquals(0, field.getAttributes().size()); + assertNull(field.getStemming()); + assertTrue(field.getNormalizing().doRemoveAccents()); + assertTrue(field.isHeader()); + + // Second field + field = (SDField) document.getField("description"); + assertEquals(RankType.ABOUT, field.getRankType()); + assertEquals(SummaryTransform.NONE, + field.getSummaryField("description").getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, + field.getSummaryField("dyndesc").getTransform()); + assertNull(field.getStemming()); + assertTrue(field.getNormalizing().doRemoveAccents()); + assertEquals("hallo", schema.getIndex("description").aliasIterator().next()); + + // Third field + field = (SDField) document.getField("chatter"); + assertEquals(RankType.ABOUT, field.getRankType()); + assertNull(field.getStemming()); + assertTrue(field.getNormalizing().doRemoveAccents()); + + // Fourth field + field = (SDField) document.getField("category"); + assertEquals(0, field.getAttributes().size()); + assertEquals(Stemming.NONE, field.getStemming()); + assertFalse(field.getNormalizing().doRemoveAccents()); + + // Fifth field + field = (SDField) document.getField("popularity"); + assertEquals("{ input popularity | attribute popularity; }", + field.getIndexingScript().toString()); + + // Sixth field + field = (SDField) document.getField("measurement"); + assertEquals(DataType.INT, field.getDataType()); + assertEquals(RankType.EMPTY, field.getRankType()); + assertEquals(1, field.getAttributes().size()); + + // Seventh field + field = schema.getConcreteField("categories"); + assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", + field.getIndexingScript().toString()); + assertTrue(field.isHeader()); + + // Eight field + field= schema.getConcreteField("categoriesagain"); + assertEquals("{ input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", + field.getIndexingScript().toString()); + assertTrue(field.isHeader()); + + // Ninth field + field= schema.getConcreteField("exactemento"); + assertEquals("{ input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", + field.getIndexingScript().toString()); + + // Tenth field + field = schema.getConcreteField("category_arr"); + assertEquals(1, field.getAttributes().size()); + attribute = field.getAttributes().get("category_arr"); + assertNotNull(attribute); + assertEquals("category_arr", attribute.getName()); + assertEquals(Attribute.Type.STRING, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + assertTrue(field.isHeader()); + + // Eleventh field + field = schema.getConcreteField("measurement_arr"); + assertEquals(1, field.getAttributes().size()); + attribute = field.getAttributes().get("measurement_arr"); + assertEquals("measurement_arr", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + // Rank Profiles + RankProfile profile = rankProfileRegistry.get(schema, "default"); + assertNotNull(profile); + assertEmpty(profile.inheritedNames()); + assertNull(profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE)); + assertEquals(RankType.EMPTY, + profile.getRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); + profile = rankProfileRegistry.get(schema, "experimental"); + assertNotNull(profile); + assertEquals("default", profile.inheritedNames().get(0)); + assertEquals(RankType.IDENTITY, + profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); + + profile = rankProfileRegistry.get(schema, "other"); + assertNotNull(profile); + assertEquals("experimental", profile.inheritedNames().get(0)); + + // The extra-document field + SDField exact = schema.getConcreteField("exact"); + assertNotNull("Extra field was parsed", exact); + assertEquals("exact", exact.getName()); + assertEquals(Stemming.NONE, exact.getStemming()); + assertFalse(exact.getNormalizing().doRemoveAccents()); + assertEquals("{ input title . \" \" . input category | tokenize | summary exact | index exact; }", + exact.getIndexingScript().toString()); + assertEquals(RankType.IDENTITY, exact.getRankType()); + } + + @Test + public void testDocumentImporting() throws IOException, ParseException { + try { + // Having two documents in one sd-file is illegal. + ApplicationBuilder.buildFromFile("src/test/examples/documents.sd"); + fail(); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testIdImporting() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/strange.sd"); + SDField idecidemyide = (SDField)schema.getDocument().getField("idecidemyide"); + assertEquals(5, idecidemyide.getId()); + SDField sodoi = (SDField) schema.getDocument().getField("sodoi"); + assertEquals(7, sodoi.getId()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaParsingTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaParsingTestCase.java new file mode 100644 index 00000000000..8fe691db802 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/SchemaParsingTestCase.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import java.io.IOException; + +import com.yahoo.schema.parser.ParseException; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Tests that search definitions are parsed correctly and that correct line number is reported in + * error message. + * + * @author hmusum + */ +public class SchemaParsingTestCase extends AbstractSchemaTestCase { + + @Test + public void requireThatIndexingExpressionsCanBeParsed() throws Exception { + assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); + } + + @Test + public void requireThatParseExceptionPositionIsCorrect() throws Exception { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); + } catch (ParseException e) { + if ( ! e.getMessage().contains("at line 5, column 36.")) { + throw e; + } + } + } + + @Test + public void requireThatParserHandlesLexicalError() throws Exception { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); + } catch (ParseException e) { + if (!e.getMessage().contains("at line 7, column 27.")) { + throw e; + } + } + } + + @Test + public void requireErrorWhenJunkAfterSearchBlock() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); + fail("Illegal junk at end of SD passed"); + } catch (ParseException e) { + if (!e.getMessage().contains("at line 10, column 1")) { + throw e; + } + } + } + + @Test + public void requireErrorWhenMissingClosingSearchBracket() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); + fail("SD without closing bracket passed"); + } catch (ParseException e) { + if (!e.getMessage().contains("Encountered \"\" at line 8, column 1")) { + throw e; + } + } + } + + @Test + public void illegalSearchDefinitionName() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid-name.sd"); + fail("Name with dash passed"); + } catch (ParseException e) { + if ( ! e.getMessage().contains("invalid-name")) { + throw e; + } + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java new file mode 100644 index 00000000000..67d8ce4ff78 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java @@ -0,0 +1,443 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.ImportedFieldsResolver; +import com.yahoo.schema.processing.OnnxModelTypeResolver; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Schema tests that don't depend on files. + * + * @author bratseth + */ +public class SchemaTestCase { + + @Test + public void testValidationOfInheritedSchema() throws ParseException { + try { + String schema = joinLines( + "schema test inherits nonesuch {" + + " document test inherits nonesuch {" + + " }" + + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromStrings(logger, schema); + assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", + logger.entries.get(0).message); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", e.getMessage()); + } + } + + @Test + public void testValidationOfSchemaAndDocumentInheritanceConsistency() throws ParseException { + try { + String parent = joinLines( + "schema parent {" + + " document parent {" + + " field pf1 type string {" + + " indexing: summary" + + " }" + + " }" + + "}"); + String child = joinLines( + "schema child inherits parent {" + + " document child {" + + " field cf1 type string {" + + " indexing: summary" + + " }" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), parent, child); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("schema 'child' inherits 'parent', " + + "but its document type does not inherit the parent's document type" + , e.getMessage()); + } + } + + @Test + public void testSchemaInheritance() throws ParseException { + String parentLines = joinLines( + "schema parent {" + + " document parent {" + + " field pf1 type string {" + + " indexing: summary" + + " }" + + " }" + + " fieldset parent_set {" + + " fields: pf1" + + " }" + + " stemming: none" + + " index parent_index {" + + " stemming: best" + + " }" + + " field parent_field type string {" + + " indexing: input pf1 | lowercase | index | attribute | summary" + + " }" + + " rank-profile parent_profile {" + + " }" + + " constant parent_constant {" + + " file: constants/my_constant_tensor_file.json" + + " type: tensor(x{},y{})" + + " }" + + " onnx-model parent_model {" + + " file: models/my_model.onnx" + + " }" + + " document-summary parent_summary {" + + " summary pf1 type string {}" + + " }" + + " import field parentschema_ref.name as parent_imported {}" + + " raw-as-base64-in-summary" + + "}"); + String child1Lines = joinLines( + "schema child1 inherits parent {" + + " document child1 inherits parent {" + + " field c1f1 type string {" + + " indexing: summary" + + " }" + + " }" + + " fieldset child1_set {" + + " fields: c1f1, pf1" + + " }" + + " stemming: shortest" + + " index child1_index {" + + " stemming: shortest" + + " }" + + " field child1_field type string {" + + " indexing: input pf1 | lowercase | index | attribute | summary" + + " }" + + " rank-profile child1_profile inherits parent_profile {" + + " constants {" + + " child1_constant tensor(x{},y{}): file:constants/my_constant_tensor_file.json" + + " }" + + " }" + + " onnx-model child1_model {" + + " file: models/my_model.onnx" + + " }" + + " document-summary child1_summary inherits parent_summary {" + + " summary c1f1 type string {}" + + " }" + + " import field parentschema_ref.name as child1_imported {}" + + "}"); + String child2Lines = joinLines( + "schema child2 inherits parent {" + + " document child2 inherits parent {" + + " field c2f1 type string {" + + " indexing: summary" + + " }" + + " }" + + " fieldset child2_set {" + + " fields: c2f1, pf1" + + " }" + + " stemming: shortest" + + " index child2_index {" + + " stemming: shortest" + + " }" + + " field child2_field type string {" + + " indexing: input pf1 | lowercase | index | attribute | summary" + + " }" + + " rank-profile child2_profile inherits parent_profile {" + + " }" + + " constant child2_constant {" + + " file: constants/my_constant_tensor_file.json" + + " type: tensor(x{},y{})" + + " }" + + " onnx-model child2_model {" + + " file: models/my_model.onnx" + + " }" + + " document-summary child2_summary inherits parent_summary {" + + " summary c2f1 type string {}" + + " }" + + " import field parentschema_ref.name as child2_imported {}" + + "}"); + + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); + builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist + builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere + builder.addSchema(parentLines); + builder.addSchema(child1Lines); + builder.addSchema(child2Lines); + builder.build(true); + var application = builder.application(); + + var child1 = application.schemas().get("child1"); + assertEquals("pf1", child1.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); + assertEquals("[c1f1, pf1]", child1.fieldSets().userFieldSets().get("child1_set").getFieldNames().toString()); + assertEquals(Stemming.SHORTEST, child1.getStemming()); + assertEquals(Stemming.BEST, child1.getIndex("parent_index").getStemming()); + assertEquals(Stemming.SHORTEST, child1.getIndex("child1_index").getStemming()); + assertNotNull(child1.getField("parent_field")); + assertNotNull(child1.getField("child1_field")); + assertNotNull(child1.getExtraField("parent_field")); + assertNotNull(child1.getExtraField("child1_field")); + assertNotNull(builder.getRankProfileRegistry().get(child1, "parent_profile")); + assertNotNull(builder.getRankProfileRegistry().get(child1, "child1_profile")); + var child1profile = builder.getRankProfileRegistry().get(child1, "child1_profile"); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").inheritedNames().get(0)); + assertNotNull(child1.constants().get(FeatureNames.asConstantFeature("parent_constant"))); + assertNotNull(child1profile.constants().get(FeatureNames.asConstantFeature("child1_constant"))); + assertTrue(child1.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); + assertTrue(child1profile.constants().containsKey(FeatureNames.asConstantFeature("child1_constant"))); + assertTrue(child1profile.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); + assertNotNull(child1.onnxModels().get("parent_model")); + assertNotNull(child1.onnxModels().get("child1_model")); + assertTrue(child1.onnxModels().containsKey("parent_model")); + assertTrue(child1.onnxModels().containsKey("child1_model")); + assertNotNull(child1.getSummary("parent_summary")); + assertNotNull(child1.getSummary("child1_summary")); + assertEquals("parent_summary", child1.getSummary("child1_summary").inherited().get().getName()); + assertTrue(child1.getSummaries().containsKey("parent_summary")); + assertTrue(child1.getSummaries().containsKey("child1_summary")); + assertNotNull(child1.getSummaryField("pf1")); + assertNotNull(child1.getSummaryField("c1f1")); + assertNotNull(child1.getExplicitSummaryField("pf1")); + assertNotNull(child1.getExplicitSummaryField("c1f1")); + assertNotNull(child1.getUniqueNamedSummaryFields().get("pf1")); + assertNotNull(child1.getUniqueNamedSummaryFields().get("c1f1")); + assertNotNull(child1.temporaryImportedFields().get().fields().get("parent_imported")); + assertNotNull(child1.temporaryImportedFields().get().fields().get("child1_imported")); + + var child2 = application.schemas().get("child2"); + assertEquals("pf1", child2.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); + assertEquals("[c2f1, pf1]", child2.fieldSets().userFieldSets().get("child2_set").getFieldNames().toString()); + assertEquals(Stemming.SHORTEST, child2.getStemming()); + assertEquals(Stemming.BEST, child2.getIndex("parent_index").getStemming()); + assertEquals(Stemming.SHORTEST, child2.getIndex("child2_index").getStemming()); + assertNotNull(child2.getField("parent_field")); + assertNotNull(child2.getField("child2_field")); + assertNotNull(child2.getExtraField("parent_field")); + assertNotNull(child2.getExtraField("child2_field")); + assertNotNull(builder.getRankProfileRegistry().get(child2, "parent_profile")); + assertNotNull(builder.getRankProfileRegistry().get(child2, "child2_profile")); + assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").inheritedNames().get(0)); + assertNotNull(child2.constants().get(FeatureNames.asConstantFeature("parent_constant"))); + assertNotNull(child2.constants().get(FeatureNames.asConstantFeature("child2_constant"))); + assertTrue(child2.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); + assertTrue(child2.constants().containsKey(FeatureNames.asConstantFeature("child2_constant"))); + assertNotNull(child2.onnxModels().get("parent_model")); + assertNotNull(child2.onnxModels().get("child2_model")); + assertTrue(child2.onnxModels().containsKey("parent_model")); + assertTrue(child2.onnxModels().containsKey("child2_model")); + assertNotNull(child2.getSummary("parent_summary")); + assertNotNull(child2.getSummary("child2_summary")); + assertEquals("parent_summary", child2.getSummary("child2_summary").inherited().get().getName()); + assertTrue(child2.getSummaries().containsKey("parent_summary")); + assertTrue(child2.getSummaries().containsKey("child2_summary")); + assertNotNull(child2.getSummaryField("pf1")); + assertNotNull(child2.getSummaryField("c2f1")); + assertNotNull(child2.getExplicitSummaryField("pf1")); + assertNotNull(child2.getExplicitSummaryField("c2f1")); + assertNotNull(child2.getUniqueNamedSummaryFields().get("pf1")); + assertNotNull(child2.getUniqueNamedSummaryFields().get("c2f1")); + assertNotNull(child2.temporaryImportedFields().get().fields().get("parent_imported")); + assertNotNull(child2.temporaryImportedFields().get().fields().get("child2_imported")); + DocumentSummary child2DefaultSummary = child2.getSummary("default"); + assertEquals(6, child2DefaultSummary.getSummaryFields().size()); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("child2_field")); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("parent_field")); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("pf1")); + assertTrue(child2DefaultSummary.getSummaryFields().containsKey("c2f1")); + DocumentSummary child2AttributeprefetchSummary = child2.getSummary("attributeprefetch"); + assertEquals(4, child2AttributeprefetchSummary.getSummaryFields().size()); + assertTrue(child2AttributeprefetchSummary.getSummaryFields().containsKey("child2_field")); + assertTrue(child2AttributeprefetchSummary.getSummaryFields().containsKey("parent_field")); + } + + @Test + public void testSchemaInheritanceEmptyChildren() throws ParseException { + String parentLines = joinLines( + "schema parent {" + + " document parent {" + + " field pf1 type string {" + + " indexing: summary" + + " }" + + " }" + + " fieldset parent_set {" + + " fields: pf1" + + " }" + + " stemming: none" + + " index parent_index {" + + " stemming: best" + + " }" + + " field parent_field type string {" + + " indexing: input pf1 | lowercase | index | attribute | summary" + + " }" + + " rank-profile parent_profile {" + + " }" + + " constant parent_constant {" + + " file: constants/my_constant_tensor_file.json" + + " type: tensor(x{},y{})" + + " }" + + " onnx-model parent_model {" + + " file: models/my_model.onnx" + + " }" + + " document-summary parent_summary {" + + " summary pf1 type string {}" + + " }" + + " import field parentschema_ref.name as parent_imported {}" + + " raw-as-base64-in-summary" + + "}"); + String childLines = joinLines( + "schema child inherits parent {" + + " document child inherits parent {" + + " field cf1 type string {" + + " indexing: summary" + + " }" + + " }" + + "}"); + String grandchildLines = joinLines( + "schema grandchild inherits child {" + + " document grandchild inherits child {" + + " field gf1 type string {" + + " indexing: summary" + + " }" + + " }" + + "}"); + + ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); + builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist + builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere + builder.addSchema(parentLines); + builder.addSchema(childLines); + builder.addSchema(grandchildLines); + builder.build(true); + var application = builder.application(); + + assertInheritedFromParent(application.schemas().get("child"), builder.getRankProfileRegistry()); + assertInheritedFromParent(application.schemas().get("grandchild"), builder.getRankProfileRegistry()); + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed1() throws ParseException { + try { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Only one of the profiles inherited by rank profile 'r3' can contain first-phase expression, but it is present in multiple", + e.getMessage()); + } + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsAllowedWhenDefinedInChild() throws ParseException { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " field myFilter type string {" + + " indexing: attribute\n" + + " rank: filter" + + " }" + + " }" + + " rank-profile r1 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " first-phase {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " first-phase {" + // Redefined here so this does not cause failure + " expression: nativeRank" + + " }" + + " }" + + "}"); + var builder = ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + var r3 = builder.getRankProfileRegistry().resolve(builder.application().schemas().get("test").getDocument(), "r3"); + assertEquals(1, r3.allFilterFields().size()); + } + + @Test + public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed2() throws ParseException { + try { + String profile = joinLines( + "schema test {" + + " document test {" + + " field title type string {" + + " indexing: summary" + + " }" + + " }" + + " rank-profile r1 {" + + " function f1() {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r2 {" + + " function f1() {" + + " expression: fieldMatch(title)" + + " }" + + " }" + + " rank-profile r3 inherits r1, r2 {" + + " }" + + "}"); + ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("rank profile 'r3' inherits rank profile 'r2' which contains function 'f1', but this function is already defined in another profile this inherits", + e.getMessage()); + } + } + + private void assertInheritedFromParent(Schema schema, RankProfileRegistry rankProfileRegistry) { + assertEquals("pf1", schema.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); + assertEquals(Stemming.NONE, schema.getStemming()); + assertEquals(Stemming.BEST, schema.getIndex("parent_index").getStemming()); + assertNotNull(schema.getField("parent_field")); + assertNotNull(schema.getExtraField("parent_field")); + assertNotNull(rankProfileRegistry.get(schema, "parent_profile")); + assertNotNull(schema.constants().get(FeatureNames.asConstantFeature("parent_constant"))); + assertTrue(schema.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); + assertNotNull(schema.onnxModels().get("parent_model")); + assertTrue(schema.onnxModels().containsKey("parent_model")); + assertNotNull(schema.getSummary("parent_summary")); + assertTrue(schema.getSummaries().containsKey("parent_summary")); + assertNotNull(schema.getSummaryField("pf1")); + assertNotNull(schema.getExplicitSummaryField("pf1")); + assertNotNull(schema.getUniqueNamedSummaryFields().get("pf1")); + assertNotNull(schema.temporaryImportedFields().get().fields().get("parent_imported")); + assertTrue(schema.isRawAsBase64()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/schema/StemmingSettingTestCase.java new file mode 100644 index 00000000000..5dd75166783 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/StemmingSettingTestCase.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Stemming settings test + * + * @author bratseth + */ +public class StemmingSettingTestCase extends AbstractSchemaTestCase { + + @Test + public void testStemmingSettings() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingsetting.sd"); + + SDField artist = (SDField) schema.getDocument().getField("artist"); + assertEquals(Stemming.SHORTEST, artist.getStemming(schema)); + + SDField title = (SDField) schema.getDocument().getField("title"); + assertEquals(Stemming.NONE, title.getStemming(schema)); + + SDField song = (SDField) schema.getDocument().getField("song"); + assertEquals(Stemming.MULTIPLE, song.getStemming(schema)); + + SDField track = (SDField) schema.getDocument().getField("track"); + assertEquals(Stemming.SHORTEST, track.getStemming(schema)); + + SDField backward = (SDField) schema.getDocument().getField("backward"); + assertEquals(Stemming.SHORTEST, backward.getStemming(schema)); + + Index defaultIndex = schema.getIndex("default"); + assertEquals(Stemming.SHORTEST, defaultIndex.getStemming()); + } + + @Test + public void requireThatStemmingIsDefaultBest() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); + assertNull(schema.getConcreteField("my_str").getStemming()); + assertEquals(Stemming.BEST, schema.getConcreteField("my_str").getStemming(schema)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/StructTestCase.java b/config-model/src/test/java/com/yahoo/schema/StructTestCase.java new file mode 100755 index 00000000000..b140892ed5e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/StructTestCase.java @@ -0,0 +1,59 @@ + +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.document.config.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.schema.derived.Deriver; +import com.yahoo.schema.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; +import java.io.IOException; +import static org.junit.Assert.fail; + +/** + * Tests importing of document containing array type fields + * + * @author bratseth + */ +public class StructTestCase extends AbstractSchemaTestCase { + + @Test + public void testStruct() throws IOException { + assertConfigFile("src/test/examples/structresult.cfg", + new DocumentmanagerConfig(Deriver.getDocumentManagerConfig("src/test/examples/struct.sd")) + "\n"); + } + + @Test + public void testBadStruct() throws IOException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/badstruct.sd"); + fail("Should throw exception."); + } catch (IllegalArgumentException|ParseException expected) { + System.err.println("As expected, with message: "+expected.getMessage()); + // success + } + } + + @Test + @Ignore + public void testStructAndDocumentWithSameNames() { + try { + DocumenttypesConfig.Builder dt = Deriver.getDocumentTypesConfig("src/test/examples/structanddocumentwithsamenames.sd"); + // while the above line may work, the config generated will fail. + // See also NameCollisionTestCase. + } catch (Exception e) { + fail("Should not have thrown exception " + e); + } + } + + /** + * Declaring a struct before a document should work + */ + @Test + public void testStructOutsideDocumentLegal() throws IOException, ParseException { + new ApplicationBuilder().addSchemaFile("src/test/examples/structoutsideofdocument.sd"); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java new file mode 100644 index 00000000000..0fab9b381fc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -0,0 +1,286 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; +import com.yahoo.vespa.objects.FieldBase; +import org.junit.Test; +import static com.yahoo.config.model.test.TestUtil.joinLines; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests summary validation + * + * @author bratseth + */ +public class SummaryTestCase { + + @Test + public void testMemorySummary() throws ParseException { + String sd = joinLines( + "schema memorysummary {", + " document memorysummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index # no summary, so ignored", + " }", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromString(sd, logger); + assertTrue(logger.entries.isEmpty()); + } + + @Test + public void testDiskSummary() throws ParseException { + String sd = joinLines( + "schema disksummary {", + " document-summary foobar {", + " summary foo1 type string { source: inmemory }", + " summary foo2 type string { source: ondisk }", + " }", + " document disksummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index | summary", + " }", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromString(sd, logger); + assertEquals(1, logger.entries.size()); + assertEquals(Level.WARNING, logger.entries.get(0).level); + assertEquals("summary field 'foo2' in document summary 'foobar' references source field 'ondisk', " + + "which is not an attribute: Using this summary will cause disk accesses. " + + "Set 'from-disk' on this summary class to silence this warning.", + logger.entries.get(0).message); + } + + @Test + public void testDiskSummaryExplicit() throws ParseException { + String sd = joinLines( + "schema disksummary {", + " document disksummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index | summary", + " }", + " }", + " document-summary foobar {", + " summary foo1 type string { source: inmemory }", + " summary foo2 type string { source: ondisk }", + " from-disk", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromString(sd, logger); + assertTrue(logger.entries.isEmpty()); + } + + @Test + public void testStructMemorySummary() throws ParseException { + String sd = joinLines( + "schema structmemorysummary {", + " document structmemorysummary {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + " field elem_array type array {", + " indexing: summary", + " struct-field name {", + " indexing: attribute", + " }", + " struct-field weight {", + " indexing: attribute", + " }", + " }", + " }", + " document-summary filtered {", + " summary elem_array_filtered type array {", + " source: elem_array", + " matched-elements-only", + " }", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromString(sd, logger); + assertTrue(logger.entries.isEmpty()); + } + + @Test + public void testInheritance() throws Exception { + String sd = joinLines( + "schema music {", + " document music {", + " field title type string {", + " indexing: summary | attribute | index", + " }", + " field artist type string {", + " indexing: summary | attribute | index", + " }", + " field album type string {", + " indexing: summary | attribute | index", + " }", + " }", + " document-summary title {", + " summary title type string {", + " source: title", + " }", + " }", + " document-summary title_artist inherits title {", + " summary artist type string {", + " source: artist", + " }", + " }", + " document-summary everything inherits title_artist {", + " summary album type string {", + " source: album", + " }", + " }", + "}"); + var logger = new DeployLoggerStub(); + var search = ApplicationBuilder.createFromString(sd, logger).getSchema(); + assertEquals(List.of(), logger.entries); + + var titleField = "title"; + var artistField = "artist"; + var albumField = "album"; + var titleSummary = search.getSummary(titleField); + var titleArtistSummary = search.getSummary(titleField + "_" + artistField); + var everythingSummary = search.getSummary("everything"); + + var implicitFields = List.of("rankfeatures", "summaryfeatures"); + var tests = List.of( + new TestValue(titleSummary, null, List.of(List.of(titleField), implicitFields)), + new TestValue(titleArtistSummary, titleSummary, List.of(List.of(titleField), implicitFields, List.of(artistField))), + new TestValue(everythingSummary, titleArtistSummary, List.of(List.of(titleField), implicitFields, List.of(artistField, albumField))) + ); + tests.forEach(testValue -> { + var actualFields = testValue.summary.getSummaryFields().values().stream() + .map(FieldBase::getName) + .collect(Collectors.toList()); + assertEquals(testValue.summary.getName() + (testValue.parent == null ? " does not inherit anything" : " inherits " + testValue.parent.getName()), + Optional.ofNullable(testValue.parent), + testValue.summary.inherited()); + assertEquals("Summary " + testValue.summary.getName() + " has expected fields", testValue.fields, actualFields); + }); + } + + @Test + public void testRedeclaringInheritedFieldFails() throws Exception { + String sd = joinLines( + "schema music {", + " document music {", + " field title type string {", + " indexing: summary | attribute | index", + " }", + " field title_short type string {", + " indexing: summary | attribute | index", + " }", + " }", + " document-summary title {", + " summary title type string {", + " source: title", + " }", + " }", + " document-summary title2 inherits title {", + " summary title type string {", + " source: title_short", + " }", + " }", + "}"); + var logger = new DeployLoggerStub(); + try { + ApplicationBuilder.createFromString(sd, logger); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'music', summary class 'title2', summary field 'title': Can not use " + + "source 'title_short' for this summary field, an equally named field in summary class 'title' " + + "uses a different source: 'title'.", e.getMessage()); + } + } + + @Test + public void testValidationOfInheritedSummary() throws ParseException { + try { + String schema = joinLines( + "schema test {" + + " document test {" + + " }" + + " document-summary test_summary inherits nonesuch {" + + " }" + + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromStrings(logger, schema); + assertEquals("document summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", + logger.entries.get(0).message); + // fail("Expected failure"); + } + catch (IllegalArgumentException e) { + // assertEquals("document summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", + // e.getMessage()); + } + } + + @Test + public void testInheritingParentSummary() throws ParseException { + String parent = joinLines( + "schema parent {" + + " document parent {" + + " field pf1 type string {" + + " indexing: summary" + + " }" + + " }" + + " document-summary parent_summary {" + + " summary pf1 type string {}" + + " }" + + "}"); + String child = joinLines( + "schema child inherits parent {" + + " document child inherits parent {" + + " field cf1 type string {" + + " indexing: summary" + + " }" + + " }" + + " document-summary child_summary inherits parent_summary {" + + " summary cf1 type string {}" + + " }" + + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromStrings(logger, parent, child); + logger.entries.forEach(e -> System.out.println(e)); + //assertTrue(logger.entries.isEmpty()); + } + + private static class TestValue { + + private final DocumentSummary summary; + private final DocumentSummary parent; + private final List fields; + + public TestValue(DocumentSummary summary, DocumentSummary parent, List> fields) { + this.summary = summary; + this.parent = parent; + this.fields = fields.stream().flatMap(Collection::stream).collect(Collectors.toList());; + } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/UrlFieldValidationTestCase.java new file mode 100644 index 00000000000..83bb0d4548c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/UrlFieldValidationTestCase.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class UrlFieldValidationTestCase { + + @Test + public void requireThatInheritedRiseFieldsStillCanBeInConflictButDontThrowException() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema("search test {" + + " document test { " + + " field a type uri { indexing: attribute | summary }" + + " }" + + "}"); + try { + builder.build(true); + fail("Should have caused an exception"); + // success + } catch (IllegalArgumentException e) { + assertEquals("Error in field 'a' in schema 'test': uri type fields cannot be attributes", + Exceptions.toMessageString(e)); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java new file mode 100644 index 00000000000..ad2c2d6078d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java @@ -0,0 +1,165 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.config.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.File; +import java.io.IOException; + +/** + * Superclass of tests needing file comparisons + * + * @author bratseth + */ +public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { + + private static final String tempDir = "temp/"; + private static final String schemaRoot = "src/test/derived/"; + + private DerivedConfiguration derive(String dirName, + String schemaName, + TestProperties properties, + DeployLogger logger) throws IOException, ParseException { + File toDir = new File(tempDir + dirName); + toDir.mkdirs(); + deleteContent(toDir); + + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory(schemaRoot + dirName + "/", + new MockFileRegistry(), + logger, + properties); + return derive(dirName, schemaName, properties, builder, logger); + } + + private DerivedConfiguration derive(String dirName, + String schemaName, + TestProperties properties, + ApplicationBuilder builder, + DeployLogger logger) throws IOException { + DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(schemaName), + new DeployState.Builder().properties(properties) + .deployLogger(logger) + .rankProfileRegistry(builder.getRankProfileRegistry()) + .queryProfiles(builder.getQueryProfileRegistry()) + .build()); + return export(dirName, builder, config); + } + + DerivedConfiguration derive(String dirName, ApplicationBuilder builder, Schema schema) throws IOException { + DerivedConfiguration config = new DerivedConfiguration(schema, + builder.getRankProfileRegistry(), + builder.getQueryProfileRegistry()); + return export(dirName, builder, config); + } + + private DerivedConfiguration export(String name, ApplicationBuilder builder, DerivedConfiguration config) throws IOException { + String path = exportConfig(name, config); + DerivedConfiguration.exportDocuments(new DocumentManager() + .produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); + DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); + DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); + config.exportConstants(path); + return config; + } + + private String exportConfig(String name, DerivedConfiguration config) throws IOException { + String path = tempDir + name; + config.export(path); + return path; + } + + /** + * Derives a config from name/name.sd below the test dir and verifies that every .cfg file in name/ has a + * corresponding file with the same content in temp/name. Versions can and should be omitted from the .cfg file + * names. This will fail if the search definition dir has multiple search definitions. + * + * @param dirName the name of the directory containing the searchdef file to verify + * @throws ParseException if the .sd file could not be parsed + * @throws IOException if file access failed + */ + protected DerivedConfiguration assertCorrectDeriving(String dirName) throws IOException, ParseException { + return assertCorrectDeriving(dirName, new TestableDeployLogger()); + } + protected DerivedConfiguration assertCorrectDeriving(String dirName, DeployLogger logger) throws IOException, ParseException { + return assertCorrectDeriving(dirName, null, logger); + } + + protected DerivedConfiguration assertCorrectDeriving(String dirName, + String searchDefinitionName, + DeployLogger logger) throws IOException, ParseException { + return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger); + } + + protected DerivedConfiguration assertCorrectDeriving(String dirName, + TestProperties properties) throws IOException, ParseException { + return assertCorrectDeriving(dirName, null, properties, new TestableDeployLogger()); + } + + protected DerivedConfiguration assertCorrectDeriving(String dirName, + String schemaName, + TestProperties properties, + DeployLogger logger) throws IOException, ParseException { + DerivedConfiguration derived = derive(dirName, schemaName, properties, logger); + assertCorrectConfigFiles(dirName); + return derived; + } + + /** + * Asserts config is correctly derived given a builder. + * This will fail if the builder contains multiple search definitions. + */ + protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, String dirName, DeployLogger logger) throws IOException { + builder.build(true); + DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger); + assertCorrectConfigFiles(dirName); + return derived; + } + + protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, Schema schema, String name) throws IOException { + DerivedConfiguration derived = derive(name, builder, schema); + assertCorrectConfigFiles(name); + return derived; + } + + /** + * Assert that search is derived into the files in the directory given by name. + * + * @param name the local name of the directory containing the files to check + * @throws IOException if file access failed + */ + void assertCorrectConfigFiles(String name) throws IOException { + File[] files = new File(schemaRoot, name).listFiles(); + if (files == null) return; + for (File file : files) { + if ( ! file.getName().endsWith(".cfg")) continue; + boolean orderMatters = file.getName().equals("ilscripts.cfg"); + assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName(), orderMatters); + } + } + + static void assertEqualFiles(String correctFileName, String checkFileName, boolean orderMatters) throws IOException { + // Set updateOnAssert to true if you want update the files with correct answer. + assertConfigFiles(correctFileName, checkFileName, orderMatters, false); + } + + void deleteContent(File dir) { + File[] files = dir.listFiles(); + if (files == null) return; + + for (File file : files) { + file.delete(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AnnotationsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AnnotationsTestCase.java new file mode 100755 index 00000000000..60867261f93 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/AnnotationsTestCase.java @@ -0,0 +1,66 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author Einar M R Rosenvinge + */ +public class AnnotationsTestCase extends AbstractExportingTestCase { + + @Test + public void requireThatStructRegistersIfOnlyUsedByAnnotation() throws IOException, ParseException { + assertCorrectDeriving("annotationsstruct"); + } + + @Test + public void requireThatStructRegistersIfOnlyUsedAsArrayByAnnotation() throws IOException, ParseException { + assertCorrectDeriving("annotationsstructarray"); + } + + @Test + public void testSimpleAnnotationDeriving() throws IOException, ParseException { + assertCorrectDeriving("annotationssimple"); + } + + @Test + public void testAnnotationDerivingWithImplicitStruct() throws IOException, ParseException { + assertCorrectDeriving("annotationsimplicitstruct"); + } + + @Test + public void testAnnotationDerivingInheritance() throws IOException, ParseException { + assertCorrectDeriving("annotationsinheritance"); + } + + @Test + public void testAnnotationDerivingInheritance2() throws IOException, ParseException { + assertCorrectDeriving("annotationsinheritance2"); + } + + @Test + public void testSimpleReference() throws IOException, ParseException { + assertCorrectDeriving("annotationsreference"); + } + + @Test + public void testAdvancedReference() throws IOException, ParseException { + assertCorrectDeriving("annotationsreference2"); + } + + @Test + public void testAnnotationsPolymorphy() throws IOException, ParseException { + assertCorrectDeriving("annotationspolymorphy"); + } + + /** + * An annotation declared before document {} should work. + */ + @Test + public void testAnnotationOutsideOfDocumentNew() throws IOException, ParseException { + assertCorrectDeriving("annotationsoutsideofdocument"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ArraysTestCase.java new file mode 100644 index 00000000000..5b138413a7a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ArraysTestCase.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests array type deriving. Indexing statements over array + * types is not yet supported, so this tests document type + * configuration deriving only. Expand later. + * + * @author bratseth + */ +public class ArraysTestCase extends AbstractExportingTestCase { + + @Test + public void testDocumentDeriving() throws IOException, ParseException { + assertCorrectDeriving("arrays"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java new file mode 100644 index 00000000000..1c51d3ec365 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * Tests attribute deriving + * + * @author bratseth + */ +public class AttributeListTestCase extends AbstractSchemaTestCase { + + @Test + public void testDeriving() throws IOException, ParseException { + // Test attribute importing + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); + + // Test attribute deriving + AttributeFields attributeFields = new AttributeFields(schema); + Iterator attributes = attributeFields.attributeIterator(); + Attribute attribute; + attribute = (Attribute)attributes.next(); + assertEquals("popularity", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("measurement", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("smallattribute", attribute.getName()); + assertEquals(Attribute.Type.BYTE, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("access", attribute.getName()); + assertEquals(Attribute.Type.BYTE, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("category_arr", attribute.getName()); + assertEquals(Attribute.Type.STRING, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("measurement_arr", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); + + attribute = (Attribute)attributes.next(); + assertEquals("popsiness", attribute.getName()); + assertEquals(Attribute.Type.INTEGER, attribute.getType()); + assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); + + assertFalse(attributes.hasNext()); + } + + @Test + public void fields_in_array_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + Iterator attributes = new AttributeFields(schema).attributeIterator(); + + assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertFalse(attributes.hasNext()); + } + + @Test + public void fields_in_map_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + Iterator attributes = new AttributeFields(schema).attributeIterator(); + + assertAttribute("str_elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertAttribute("str_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("str_elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("int_elem_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("int_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertFalse(attributes.hasNext()); + } + + private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, boolean isFastSearch, Attribute attr) { + assertEquals(name, attr.getName()); + assertEquals(type, attr.getType()); + assertEquals(collection, attr.getCollectionType()); + assertEquals(isFastSearch, attr.isFastSearch()); + } + + @Test + public void only_zcurve_attribute_is_derived_from_array_of_position_field() throws ParseException { + Schema schema = ApplicationBuilder.createFromString( + joinLines("search test {", + " document test {", + " field pos_array type array {", + " indexing: attribute", + " }", + " }", + "}")).getSchema(); + Iterator attributes = new AttributeFields(schema).attributeIterator(); + + assertAttribute("pos_array_zcurve", Attribute.Type.LONG, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertFalse(attributes.hasNext()); + } + + @Test + public void fields_in_map_of_primitive_are_derived_into_array_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); + Iterator attributes = new AttributeFields(schema).attributeIterator(); + + assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); + assertAttribute("str_map.value", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertAttribute("int_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); + assertFalse(attributes.hasNext()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributesTestCase.java new file mode 100644 index 00000000000..3d08805acdf --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/AttributesTestCase.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests attribute settings + * + * @author bratseth + */ +public class AttributesTestCase extends AbstractExportingTestCase { + + @Test + public void testDocumentDeriving() throws IOException, ParseException { + assertCorrectDeriving("attributes"); + } + + @Test + public void testArrayOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("array_of_struct_attribute"); + } + + @Test + public void testMapOfStructAttribute() throws IOException, ParseException { + assertCorrectDeriving("map_of_struct_attribute"); + } + + @Test + public void testMapOfPrimitiveAttribute() throws IOException, ParseException { + assertCorrectDeriving("map_attribute"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/CasingTestCase.java new file mode 100644 index 00000000000..80ebcb825f4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/CasingTestCase.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Correct casing for derived attributes + * + * @author vegardh + */ +public class CasingTestCase extends AbstractSchemaTestCase { + + @Test + public void testCasing() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/casing.sd"); + assertEquals(schema.getIndex("color").getName(), "color"); + assertEquals(schema.getIndex("Foo").getName(), "Foo"); + assertEquals(schema.getIndex("Price").getName(), "Price"); + assertEquals(schema.getAttribute("artist").getName(), "artist"); + assertEquals(schema.getAttribute("Drummer").getName(), "Drummer"); + assertEquals(schema.getAttribute("guitarist").getName(), "guitarist"); + assertEquals(schema.getAttribute("title").getName(), "title"); + assertEquals(schema.getAttribute("Trumpetist").getName(), "Trumpetist"); + assertEquals(schema.getAttribute("Saxophonist").getName(), "Saxophonist"); + assertEquals(schema.getAttribute("TenorSaxophonist").getName(), "TenorSaxophonist"); + assertEquals(schema.getAttribute("Flutist").getName(), "Flutist"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/CombinedAttributeAndIndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/CombinedAttributeAndIndexSchemaTestCase.java new file mode 100644 index 00000000000..542320d9670 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/CombinedAttributeAndIndexSchemaTestCase.java @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving a configuration with multiple summaries + * + * @author bratseth + */ +public class CombinedAttributeAndIndexSchemaTestCase extends AbstractExportingTestCase { + + @Test + public void testMultipleSummaries() throws IOException, ParseException { + assertCorrectDeriving("combinedattributeandindexsearch"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/DeriverTestCase.java new file mode 100644 index 00000000000..422f4522b26 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/DeriverTestCase.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.schema.AbstractSchemaTestCase; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Tests deriving using the Deriver facade + * + * @author bratseth + */ +public class DeriverTestCase extends AbstractSchemaTestCase { + + @Test + public void testDeriveDocManager() { + DocumentTypeManager dtm = new DocumentTypeManager(new DocumentmanagerConfig( + Deriver.getDocumentManagerConfig(List.of( + "src/test/derived/deriver/child.sd", + "src/test/derived/deriver/parent.sd", + "src/test/derived/deriver/grandparent.sd")))); + assertEquals(dtm.getDocumentType("child").getField("a").getDataType(), DataType.STRING); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/DuplicateStructTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/DuplicateStructTestCase.java new file mode 100644 index 00000000000..7915a1d7763 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/DuplicateStructTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import org.junit.Test; + +/** + * @author arnej + */ +public class DuplicateStructTestCase extends AbstractExportingTestCase { + + @Test + public void exact_duplicate_struct_works() throws Exception { + assertCorrectDeriving("duplicate_struct", "foobar", + new TestProperties(), + new TestableDeployLogger()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/EmptyRankProfileTestCase.java new file mode 100644 index 00000000000..c3195d1a626 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/EmptyRankProfileTestCase.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import org.junit.Test; + +/** + * Tests deriving rank for files from search definitions + * + * @author bratseth + */ +public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { + + @Test + public void testDeriving() { + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + SDDocumentType doc = new SDDocumentType("test"); + schema.addDocument(doc); + doc.addField(new SDField(doc, "a", DataType.STRING)); + SDField field = new SDField(doc, "b", DataType.STRING); + field.setLiteralBoost(500); + doc.addField(field); + doc.addField(new SDField(doc, "c", DataType.STRING)); + + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + new DerivedConfiguration(schema, rankProfileRegistry); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExactMatchTestCase.java new file mode 100644 index 00000000000..13a0a8201fb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExactMatchTestCase.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseth + */ +public class ExactMatchTestCase extends AbstractExportingTestCase { + @Test + public void testExactString() throws IOException, ParseException { + assertCorrectDeriving("exactmatch"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java new file mode 100644 index 00000000000..16a9a459dcb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java @@ -0,0 +1,195 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Tests exporting + * + * @author bratseth + */ +public class ExportingTestCase extends AbstractExportingTestCase { + + @Test + public void testIndexInfoLowerCase() throws IOException, ParseException { + assertCorrectDeriving("indexinfo_lowercase"); + } + + @Test + public void testPositionArray() throws IOException, ParseException { + assertCorrectDeriving("position_array", + new TestProperties().setUseV8GeoPositions(true)); + } + + @Test + public void testPositionAttribute() throws IOException, ParseException { + assertCorrectDeriving("position_attribute", + new TestProperties().setUseV8GeoPositions(true)); + } + + @Test + public void testPositionExtra() throws IOException, ParseException { + assertCorrectDeriving("position_extra", + new TestProperties().setUseV8GeoPositions(true)); + } + + @Test + public void testPositionNoSummary() throws IOException, ParseException { + assertCorrectDeriving("position_nosummary", + new TestProperties().setUseV8GeoPositions(true)); + } + + @Test + public void testPositionSummary() throws IOException, ParseException { + assertCorrectDeriving("position_summary", + new TestProperties().setUseV8GeoPositions(true)); + } + + @Test + public void testUriArray() throws IOException, ParseException { + assertCorrectDeriving("uri_array"); + } + + @Test + public void testUriWSet() throws IOException, ParseException { + assertCorrectDeriving("uri_wset"); + } + + @Test + public void testMusic() throws IOException, ParseException { + assertCorrectDeriving("music"); + } + + @Test + public void testComplexPhysicalExporting() throws IOException, ParseException { + assertCorrectDeriving("complex"); + } + + @Test + public void testAttributePrefetch() throws IOException, ParseException { + assertCorrectDeriving("attributeprefetch"); + } + + @Test + public void testAdvancedIL() throws IOException, ParseException { + assertCorrectDeriving("advanced"); + } + + @Test + public void testEmptyDefaultIndex() throws IOException, ParseException { + assertCorrectDeriving("emptydefault"); + } + + @Test + public void testIndexSwitches() throws IOException, ParseException { + assertCorrectDeriving("indexswitches"); + } + + @Test + public void testRankTypes() throws IOException, ParseException { + assertCorrectDeriving("ranktypes"); + } + + @Test + public void testAttributeRank() throws IOException, ParseException { + assertCorrectDeriving("attributerank"); + } + + @Test + public void testNewRank() throws IOException, ParseException { + assertCorrectDeriving("newrank"); + } + + @Test + public void testRankingExpression() throws IOException, ParseException { + assertCorrectDeriving("rankingexpression"); + } + + @Test + public void testAvoidRenamingRankingExpression() throws IOException, ParseException { + assertCorrectDeriving("renamedfeatures", "foo", + new TestProperties().setAvoidRenamingSummaryFeatures(true), + new TestableDeployLogger()); + } + + @Test + public void testMlr() throws IOException, ParseException { + assertCorrectDeriving("mlr"); + } + + @Test + public void testMusic3() throws IOException, ParseException { + assertCorrectDeriving("music3"); + } + + @Test + public void testIndexSchema() throws IOException, ParseException { + assertCorrectDeriving("indexschema"); + } + + @Test + public void testIndexinfoFieldsets() throws IOException, ParseException { + assertCorrectDeriving("indexinfo_fieldsets"); + } + + @Test + public void testStreamingJuniper() throws IOException, ParseException { + assertCorrectDeriving("streamingjuniper"); + } + + @Test + public void testPredicateAttribute() throws IOException, ParseException { + assertCorrectDeriving("predicate_attribute"); + } + + @Test + public void testTensor() throws IOException, ParseException { + assertCorrectDeriving("tensor"); + } + + @Test + public void testTensor2() throws IOException, ParseException { + String dir = "src/test/derived/tensor2/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "first.sd"); + builder.addSchemaFile(dir + "second.sd"); + builder.build(true); + derive("tensor2", builder, builder.getSchema("second")); + assertCorrectConfigFiles("tensor2"); + } + + @Test + public void testHnswIndex() throws IOException, ParseException { + assertCorrectDeriving("hnsw_index"); + } + + @Test + public void testRankProfileInheritance() throws IOException, ParseException { + assertCorrectDeriving("rankprofileinheritance", "child", new TestableDeployLogger()); + } + + @Test + public void testLanguage() throws IOException, ParseException { + TestableDeployLogger logger = new TestableDeployLogger(); + assertCorrectDeriving("language", logger); + assertEquals(0, logger.warnings.size()); + } + + @Test + public void testRankProfileModularity() throws IOException, ParseException { + assertCorrectDeriving("rankprofilemodularity"); + } + + @Test + public void testStructAndFieldSet() throws IOException, ParseException { + assertCorrectDeriving("structandfieldset"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ExpressionsAsArgsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExpressionsAsArgsTestCase.java new file mode 100644 index 00000000000..d2020305bc1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExpressionsAsArgsTestCase.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests correct deriving of expressions as arguments to functions. + * + * @author lesters + */ +public class ExpressionsAsArgsTestCase extends AbstractExportingTestCase { + + @Test + public void testDocumentDeriving() throws IOException, ParseException { + assertCorrectDeriving("function_arguments"); + assertCorrectDeriving("function_arguments_with_expressions"); + } + +} + + + diff --git a/config-model/src/test/java/com/yahoo/schema/derived/FieldsetTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/FieldsetTestCase.java new file mode 100644 index 00000000000..fdab49c9fff --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/FieldsetTestCase.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +public class FieldsetTestCase extends AbstractExportingTestCase { + + @Test + public void testRankProfiles() throws IOException, ParseException { + assertCorrectDeriving("fieldset"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GeminiTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GeminiTestCase.java new file mode 100644 index 00000000000..5531fb65942 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/GeminiTestCase.java @@ -0,0 +1,70 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.collections.Pair; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class GeminiTestCase extends AbstractExportingTestCase { + + @Test + public void testRanking2() throws IOException, ParseException { + DerivedConfiguration c = assertCorrectDeriving("gemini2"); + RawRankProfile p = c.getRankProfileList().getRankProfiles().get("test"); + Map ranking = removePartKeySuffixes(asMap(p.configProperties())); + assertEquals("attribute(right)", resolve(lookup("toplevel", ranking), ranking)); + } + + private Map asMap(List> properties) { + Map map = new HashMap<>(); + for (Pair property : properties) + map.put(property.getFirst(), property.getSecond()); + return map; + } + + private Map removePartKeySuffixes(Map p) { + Map pWithoutSuffixes = new HashMap<>(); + for (Map.Entry entry : p.entrySet()) + pWithoutSuffixes.put(removePartSuffix(entry.getKey()), entry.getValue()); + return pWithoutSuffixes; + } + + private String removePartSuffix(String s) { + int partIndex = s.indexOf(".part"); + if (partIndex <= 0) return s; + return s.substring(0, partIndex); + } + + /** + * Recursively resolves references to other ranking expressions - rankingExpression(name) - + * and replaces the reference by the expression + */ + private String resolve(String expression, Map ranking) { + int referenceStartIndex; + while ((referenceStartIndex = expression.indexOf("rankingExpression(")) >= 0) { + int referenceEndIndex = expression.indexOf(")", referenceStartIndex); + expression = expression.substring(0, referenceStartIndex) + + resolve(lookup(expression.substring(referenceStartIndex + "rankingExpression(".length(), referenceEndIndex), ranking), ranking) + + expression.substring(referenceEndIndex + 1); + } + return expression; + } + + private String lookup(String expressionName, Map ranking) { + String value = ranking.get("rankingExpression(" + expressionName + ").rankingScript"); + if (value == null) { + return expressionName; + } + return value; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java new file mode 100644 index 00000000000..1e57d52e3b0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * Tests that documents ids are treated as they should + * + * @author bratseth + */ +public class IdTestCase extends AbstractExportingTestCase { + + @Test + public void testExplicitUpperCaseIdField() { + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); + SDDocumentType document = new SDDocumentType("test"); + schema.addDocument(document); + SDField uri = new SDField(document, "URI", DataType.URI); + uri.parseIndexingScript("{ summary | index }"); + document.addField(uri); + + new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), + true, false, Set.of()); + + assertNull(document.getField("uri")); + assertNull(document.getField("Uri")); + assertNotNull(document.getField("URI")); + } + + @Test + public void testCompleteDeriving() throws Exception { + assertCorrectDeriving("id"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ImportedFieldsTestCase.java new file mode 100644 index 00000000000..5578a1a602b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ImportedFieldsTestCase.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author geirst + */ +public class ImportedFieldsTestCase extends AbstractExportingTestCase { + + @Test + public void configs_for_imported_fields_are_derived() throws IOException, ParseException { + assertCorrectDeriving("importedfields", "child", new TestableDeployLogger()); + } + + @Test + public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException { + assertCorrectDeriving("imported_struct_fields", "child", + new TestProperties(), + new TestableDeployLogger()); + } + + @Test + public void configs_for_imported_position_field_are_derived() throws IOException, ParseException { + assertCorrectDeriving("imported_position_field", "child", new TestableDeployLogger()); + } + + @Test + public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException { + assertCorrectDeriving("imported_position_field_summary", "child", new TestableDeployLogger()); + } + + @Test + public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException { + assertCorrectDeriving("imported_fields_inherited_reference", "child_c", new TestableDeployLogger()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java new file mode 100644 index 00000000000..1f40c6bcb50 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java @@ -0,0 +1,208 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen Hult + */ +public class IndexSchemaTestCase { + + @Test + public void requireThatPrimitiveIsNotFlattened() { + assertFlat(new Field("foo", DataType.BYTE), new Field("foo", DataType.BYTE)); + assertFlat(new Field("foo", DataType.DOUBLE), new Field("foo", DataType.DOUBLE)); + assertFlat(new Field("foo", DataType.FLOAT), new Field("foo", DataType.FLOAT)); + assertFlat(new Field("foo", DataType.INT), new Field("foo", DataType.INT)); + assertFlat(new Field("foo", DataType.LONG), new Field("foo", DataType.LONG)); + assertFlat(new Field("foo", DataType.RAW), new Field("foo", DataType.RAW)); + assertFlat(new Field("foo", DataType.STRING), new Field("foo", DataType.STRING)); + assertFlat(new Field("foo", DataType.URI), new Field("foo", DataType.URI)); + assertFlat(new Field("foo", DataType.PREDICATE), new Field("foo", DataType.PREDICATE)); + } + + @Test + public void requireThatArrayOfPrimitiveIsNotFlattened() { + assertFlat(new Field("foo", DataType.getArray(DataType.BYTE)), + new Field("foo", DataType.getArray(DataType.BYTE))); + assertFlat(new Field("foo", DataType.getArray(DataType.DOUBLE)), + new Field("foo", DataType.getArray(DataType.DOUBLE))); + assertFlat(new Field("foo", DataType.getArray(DataType.FLOAT)), + new Field("foo", DataType.getArray(DataType.FLOAT))); + assertFlat(new Field("foo", DataType.getArray(DataType.INT)), + new Field("foo", DataType.getArray(DataType.INT))); + assertFlat(new Field("foo", DataType.getArray(DataType.LONG)), + new Field("foo", DataType.getArray(DataType.LONG))); + assertFlat(new Field("foo", DataType.getArray(DataType.RAW)), + new Field("foo", DataType.getArray(DataType.RAW))); + assertFlat(new Field("foo", DataType.getArray(DataType.STRING)), + new Field("foo", DataType.getArray(DataType.STRING))); + assertFlat(new Field("foo", DataType.getArray(DataType.URI)), + new Field("foo", DataType.getArray(DataType.URI))); + assertFlat(new Field("foo", DataType.getArray(DataType.PREDICATE)), + new Field("foo", DataType.getArray(DataType.PREDICATE))); + } + + @Test + public void requireThatStructIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.BYTE)); + type.addField(new Field("my_double", DataType.DOUBLE)); + type.addField(new Field("my_float", DataType.FLOAT)); + type.addField(new Field("my_int", DataType.INT)); + type.addField(new Field("my_long", DataType.LONG)); + type.addField(new Field("my_raw", DataType.RAW)); + type.addField(new Field("my_string", DataType.STRING)); + type.addField(new Field("my_uri", DataType.URI)); + + assertFlat(new Field("foo", type), + new Field("foo.my_byte", DataType.BYTE), + new Field("foo.my_double", DataType.DOUBLE), + new Field("foo.my_float", DataType.FLOAT), + new Field("foo.my_int", DataType.INT), + new Field("foo.my_long", DataType.LONG), + new Field("foo.my_raw", DataType.RAW), + new Field("foo.my_string", DataType.STRING), + new Field("foo.my_uri", DataType.URI)); + } + + @Test + public void requireThatArrayOfStructIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.BYTE)); + type.addField(new Field("my_double", DataType.DOUBLE)); + type.addField(new Field("my_float", DataType.FLOAT)); + type.addField(new Field("my_int", DataType.INT)); + type.addField(new Field("my_long", DataType.LONG)); + type.addField(new Field("my_raw", DataType.RAW)); + type.addField(new Field("my_string", DataType.STRING)); + type.addField(new Field("my_uri", DataType.URI)); + + assertFlat(new Field("foo", DataType.getArray(type)), + new Field("foo.my_byte", DataType.getArray(DataType.BYTE)), + new Field("foo.my_double", DataType.getArray(DataType.DOUBLE)), + new Field("foo.my_float", DataType.getArray(DataType.FLOAT)), + new Field("foo.my_int", DataType.getArray(DataType.INT)), + new Field("foo.my_long", DataType.getArray(DataType.LONG)), + new Field("foo.my_raw", DataType.getArray(DataType.RAW)), + new Field("foo.my_string", DataType.getArray(DataType.STRING)), + new Field("foo.my_uri", DataType.getArray(DataType.URI))); + } + + @Test + public void requireThatArrayOfArrayOfStructIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.BYTE)); + type.addField(new Field("my_double", DataType.DOUBLE)); + type.addField(new Field("my_float", DataType.FLOAT)); + type.addField(new Field("my_int", DataType.INT)); + type.addField(new Field("my_long", DataType.LONG)); + type.addField(new Field("my_raw", DataType.RAW)); + type.addField(new Field("my_string", DataType.STRING)); + type.addField(new Field("my_uri", DataType.URI)); + + assertFlat(new Field("foo", DataType.getArray(DataType.getArray(type))), + new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), + new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), + new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), + new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), + new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), + new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), + new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), + new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + } + + @Test + public void requireThatStructWithArrayFieldIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.getArray(DataType.BYTE))); + type.addField(new Field("my_double", DataType.getArray(DataType.DOUBLE))); + type.addField(new Field("my_float", DataType.getArray(DataType.FLOAT))); + type.addField(new Field("my_int", DataType.getArray(DataType.INT))); + type.addField(new Field("my_long", DataType.getArray(DataType.LONG))); + type.addField(new Field("my_raw", DataType.getArray(DataType.RAW))); + type.addField(new Field("my_string", DataType.getArray(DataType.STRING))); + type.addField(new Field("my_uri", DataType.getArray(DataType.URI))); + + assertFlat(new Field("foo", type), + new Field("foo.my_byte", DataType.getArray(DataType.BYTE)), + new Field("foo.my_double", DataType.getArray(DataType.DOUBLE)), + new Field("foo.my_float", DataType.getArray(DataType.FLOAT)), + new Field("foo.my_int", DataType.getArray(DataType.INT)), + new Field("foo.my_long", DataType.getArray(DataType.LONG)), + new Field("foo.my_raw", DataType.getArray(DataType.RAW)), + new Field("foo.my_string", DataType.getArray(DataType.STRING)), + new Field("foo.my_uri", DataType.getArray(DataType.URI))); + } + + @Test + public void requireThatStructWithArrayOfArrayFieldIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.getArray(DataType.getArray(DataType.BYTE)))); + type.addField(new Field("my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE)))); + type.addField(new Field("my_float", DataType.getArray(DataType.getArray(DataType.FLOAT)))); + type.addField(new Field("my_int", DataType.getArray(DataType.getArray(DataType.INT)))); + type.addField(new Field("my_long", DataType.getArray(DataType.getArray(DataType.LONG)))); + type.addField(new Field("my_raw", DataType.getArray(DataType.getArray(DataType.RAW)))); + type.addField(new Field("my_string", DataType.getArray(DataType.getArray(DataType.STRING)))); + type.addField(new Field("my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + + assertFlat(new Field("foo", type), + new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), + new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), + new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), + new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), + new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), + new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), + new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), + new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + } + + @Test + public void requireThatArrayOfStructWithArrayFieldIsFlattened() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("my_byte", DataType.getArray(DataType.BYTE))); + type.addField(new Field("my_double", DataType.getArray(DataType.DOUBLE))); + type.addField(new Field("my_float", DataType.getArray(DataType.FLOAT))); + type.addField(new Field("my_int", DataType.getArray(DataType.INT))); + type.addField(new Field("my_long", DataType.getArray(DataType.LONG))); + type.addField(new Field("my_raw", DataType.getArray(DataType.RAW))); + type.addField(new Field("my_string", DataType.getArray(DataType.STRING))); + type.addField(new Field("my_uri", DataType.getArray(DataType.URI))); + + assertFlat(new Field("foo", DataType.getArray(type)), + new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), + new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), + new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), + new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), + new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), + new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), + new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), + new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); + } + + private static void assertFlat(Field fieldToFlatten, Field... expectedFields) { + List actual = new LinkedList<>(IndexSchema.flattenField(fieldToFlatten)); + List expected = new LinkedList<>(Arrays.asList(expectedFields)); + Collections.sort(actual); + Collections.sort(expected); + for (Field field : actual) { + if (!expected.remove(field)) { + fail("Unexpected field: " + field); + } + } + assertTrue("Missing fields: " + expected, expected.isEmpty()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java new file mode 100644 index 00000000000..1c433237fd8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java @@ -0,0 +1,192 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * Tests inheritance + * + * @author bratseth + */ +public class InheritanceTestCase extends AbstractExportingTestCase { + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void requireThatIndexedStructFieldCanBeInherited() throws IOException, ParseException { + String dir = "src/test/derived/inheritstruct/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); + derive("inheritstruct", builder, builder.getSchema("child")); + assertCorrectConfigFiles("inheritstruct"); + } + + @Test + public void requireThatInheritFromNullIsCaught() throws IOException, ParseException { + try { + assertCorrectDeriving("inheritfromnull"); + } catch (IllegalArgumentException e) { + assertEquals("document inheritfromnull inherits from unavailable document foo", e.getMessage()); + } + } + + @Test + public void requireThatStructTypesAreInheritedThroughDiamond() throws IOException, ParseException { + String dir = "src/test/derived/inheritdiamond/"; + { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "mother.sd"); + builder.addSchemaFile(dir + "father.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); + derive("inheritdiamond", builder, builder.getSchema("child")); + assertCorrectConfigFiles("inheritdiamond"); + } + List files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); + File outDir = tmpDir.newFolder("out"); + for (int startIdx = 0; startIdx < files.size(); ++startIdx) { + var builder = new ApplicationBuilder(new TestProperties()); + for (int fileIdx = startIdx; fileIdx < startIdx + files.size(); ++fileIdx) { + String fileName = files.get(fileIdx % files.size()); + builder.addSchemaFile(dir + fileName); + } + builder.build(true); + DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); + DerivedConfiguration.exportDocuments(new DocumentManager(). + produce(builder.getModel(), b), outDir.getPath()); + DocumentmanagerConfig dc = b.build(); + assertEquals(5, dc.doctype().size()); + + assertNull(structType("child.body", dc)); + var childHeader = structType("child.header", dc); + assertEquals(childHeader.field(0).name(), "foo"); + assertEquals(childHeader.field(1).name(), "bar"); + assertEquals(childHeader.field(2).name(), "baz"); + assertEquals(childHeader.field(3).name(), "cox"); + + var root = documentType("document", dc); + var child = documentType("child", dc); + var mother = documentType("mother", dc); + var father = documentType("father", dc); + var grandparent = documentType("grandparent", dc); + + assertEquals(child.inherits(0).idx(), root.idx()); + assertEquals(child.inherits(1).idx(), mother.idx()); + assertEquals(child.inherits(2).idx(), father.idx()); + assertEquals(mother.inherits(0).idx(), root.idx()); + assertEquals(mother.inherits(1).idx(), grandparent.idx()); + } + } + + private DocumentmanagerConfig.Doctype.Structtype structType(String name, DocumentmanagerConfig dc) { + for (var dt : dc.doctype()) { + for (var st : dt.structtype()) { + if (name.equals(st.name())) return st; + } + } + return null; + } + + private DocumentmanagerConfig.Doctype documentType(String name, DocumentmanagerConfig dc) { + for (var dot : dc.doctype()) { + if (name.equals(dot.name())) return dot; + } + return null; + } + + @Test + public void requireThatStructTypesAreInheritedFromParent() throws IOException, ParseException { + String dir = "src/test/derived/inheritfromparent/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); + derive("inheritfromparent", builder, builder.getSchema("child")); + assertCorrectConfigFiles("inheritfromparent"); + } + + @Test + public void requireThatStructTypesAreInheritedFromGrandParent() throws IOException, ParseException { + String dir = "src/test/derived/inheritfromgrandparent/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "parent.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); + derive("inheritfromgrandparent", builder, builder.getSchema("child")); + assertCorrectConfigFiles("inheritfromgrandparent"); + } + + @Test + public void testInheritance() throws IOException, ParseException { + String dir = "src/test/derived/inheritance/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "grandparent.sd"); + builder.addSchemaFile(dir + "father.sd"); + builder.addSchemaFile(dir + "mother.sd"); + builder.addSchemaFile(dir + "child.sd"); + builder.build(true); + derive("inheritance", builder, builder.getSchema("child")); + assertCorrectConfigFiles("inheritance"); + } + + @Test + public void testIndexSettingInheritance() { + SDDocumentType parent = new SDDocumentType("parent"); + Schema parentSchema = new Schema("parent", MockApplicationPackage.createEmpty()); + parentSchema.addDocument(parent); + SDField prefixed = parent.addField("prefixed", DataType.STRING); + prefixed.parseIndexingScript("{ index }"); + prefixed.addIndex(new Index("prefixed", true)); + + SDDocumentType child = new SDDocumentType("child"); + child.inherit(parent); + Schema childSchema = new Schema("child", MockApplicationPackage.createEmpty()); + childSchema.addDocument(child); + + prefixed = (SDField)child.getField("prefixed"); + assertNotNull(prefixed); + assertEquals(new Index("prefixed", true), childSchema.getIndex("prefixed")); + } + + @Test + public void testInheritStructDiamondNew() throws IOException, ParseException { + String dir = "src/test/derived/declstruct/"; + List files = Arrays.asList("common.sd", "foo.sd", "bar.sd", "foobar.sd"); + var builder = new ApplicationBuilder(new TestProperties()); + for (String fileName : files) { + builder.addSchemaFile(dir + fileName); + } + builder.build(true); + derive("declstruct", builder, builder.getSchema("foobar")); + assertCorrectConfigFiles("declstruct"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IntegerAttributeToStringIndexTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IntegerAttributeToStringIndexTestCase.java new file mode 100644 index 00000000000..b5f222673ab --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/IntegerAttributeToStringIndexTestCase.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseth + */ +public class IntegerAttributeToStringIndexTestCase extends AbstractExportingTestCase { + @Test + public void testIt() throws IOException, ParseException { + assertCorrectDeriving("integerattributetostringindex"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java new file mode 100644 index 00000000000..c5090c88a1b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java @@ -0,0 +1,113 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Set; + +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class LiteralBoostTestCase extends AbstractExportingTestCase { + + /** + * Tests adding of literal boost constructs + */ + @Test + public void testLiteralBoost() { + Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + SDDocumentType document = new SDDocumentType("literalboost"); + schema.addDocument(document); + SDField field1 = document.addField("a", DataType.STRING); + field1.parseIndexingScript("{ index }"); + field1.setLiteralBoost(20); + RankProfile other = new RankProfile("other", schema, rankProfileRegistry); + rankProfileRegistry.add(other); + other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); + + new Processing().process(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), + true, false, Set.of()); + DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); + + // Check attribute fields + derived.getAttributeFields(); // TODO: assert content + + // Check il script addition + assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", + "clear_state | guard { input a | tokenize | index a_literal; }"), + schema); + + // Check index info addition + IndexInfo indexInfo = derived.getIndexInfo(); + assertTrue(indexInfo.hasCommand("a", "literal-boost")); + } + + /** + * Tests adding a literal boost in a non-default rank profile only + */ + @Test + public void testNonDefaultRankLiteralBoost() { + Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + SDDocumentType document = new SDDocumentType("literalboost"); + schema.addDocument(document); + SDField field1 = document.addField("a", DataType.STRING); + field1.parseIndexingScript("{ index }"); + RankProfile other = new RankProfile("other", schema, rankProfileRegistry); + rankProfileRegistry.add(other); + other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); + + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); + + // Check il script addition + assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", + "clear_state | guard { input a | tokenize | index a_literal; }"), + schema); + + // Check index info addition + IndexInfo indexInfo = derived.getIndexInfo(); + assertTrue(indexInfo.hasCommand("a","literal-boost")); + } + + /** Tests literal boosts in two fields going to the same index */ + @Test + public void testTwoLiteralBoostFields() { + Schema schema = new Schema("msb", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + SDDocumentType document = new SDDocumentType("msb"); + schema.addDocument(document); + SDField field1 = document.addField("title", DataType.STRING); + field1.parseIndexingScript("{ summary | index }"); + field1.setLiteralBoost(20); + SDField field2 = document.addField("body", DataType.STRING); + field2.parseIndexingScript("{ summary | index }"); + field2.setLiteralBoost(20); + + schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); + new DerivedConfiguration(schema, rankProfileRegistry); + assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", + "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", + "clear_state | guard { input title | tokenize | index title_literal; }", + "clear_state | guard { input body | tokenize | index body_literal; }"), + schema); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/LowercaseTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/LowercaseTestCase.java new file mode 100644 index 00000000000..f234a9cc324 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/LowercaseTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseth + */ +public class LowercaseTestCase extends AbstractExportingTestCase { + + @Test + public void testDeriving() throws IOException, ParseException { + assertCorrectDeriving("lowercase"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MailTestCase.java new file mode 100644 index 00000000000..c48c44554ed --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/MailTestCase.java @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; +import java.io.IOException; + +/** + * Tests streaming configuration deriving + * + * @author bratseth + */ +public class MailTestCase extends AbstractExportingTestCase { + + @Test + public void testMail() throws IOException, ParseException { + String dir = "src/test/derived/mail/"; + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile(dir + "mail.sd"); + assertCorrectDeriving(sb, dir, new TestableDeployLogger()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MatchSettingsResolvingTestCase.java new file mode 100755 index 00000000000..a7df862134a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/MatchSettingsResolvingTestCase.java @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author arnej + */ +public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase { + + @Test + public void testSimpleDefaults() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_def", new TestProperties()); + } + + @Test + public void testSimpleWithStructSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_wss", + new TestProperties()); + } + + @Test + public void testSimpleWithFieldSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_wfs", new TestProperties()); + } + + @Test + public void testSimpleStructAndFieldSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_simple_wss_wfs", new TestProperties()); + } + + @Test + public void testMapDefaults() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_def", new TestProperties()); + } + + @Test + public void testMapWithStructSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_wss", new TestProperties()); + } + + @Test + public void testMapWithFieldSettings() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_wfs", new TestProperties()); + } + + @Test + public void testMapAfter() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_after", new TestProperties()); + } + + + @Test + public void testMapInStruct() throws IOException, ParseException { + assertCorrectDeriving("matchsettings_map_in_struct", new TestProperties()); + } + + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/MultiStructTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MultiStructTestCase.java new file mode 100644 index 00000000000..66b3698b38c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/MultiStructTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import org.junit.Test; + +/** + * Tests deriving a configuration with structs in multiple .sd files + * + * @author arnej + */ +public class MultiStructTestCase extends AbstractExportingTestCase { + + @Test + public void testDocTypeConfigs() throws Exception { + var logger = new TestableDeployLogger(); + var props = new TestProperties(); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory + ("src/test/derived/multi_struct/", new MockFileRegistry(), logger, props); + derive("multi_struct", builder, builder.getSchema("shop")); + assertCorrectConfigFiles("multi_struct"); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MultipleSummariesTestCase.java new file mode 100644 index 00000000000..d434673e43a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/MultipleSummariesTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving a configuration with multiple summaries + * + * @author bratseth + */ +public class MultipleSummariesTestCase extends AbstractExportingTestCase { + + @Test + public void testMultipleSummariesNew() throws IOException, ParseException { + assertCorrectDeriving("multiplesummaries", new TestProperties()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NameCollisionTestCase.java new file mode 100644 index 00000000000..689ff9814cc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NameCollisionTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DocumentTypeManager; + +import org.junit.Test; +import static org.junit.Assert.assertThrows; + +/** + * Verifies that a struct in a document type is preferred over another document type + * of the same name. + * + * @author bratseth + */ +public class NameCollisionTestCase extends AbstractExportingTestCase { + + @Test + public void testNameCollision() throws Exception { + assertCorrectDeriving("namecollision", "collisionstruct", + new TestProperties(), + new TestableDeployLogger()); + DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NativeRankTypeDefinitionsTestCase.java new file mode 100644 index 00000000000..f628420556a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NativeRankTypeDefinitionsTestCase.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.RankType; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Testing stuff related to native rank type definitions + * + * @author geirst + */ +public class NativeRankTypeDefinitionsTestCase extends AbstractSchemaTestCase { + @Test + public void testTables() { + assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable"); + assertEquals(NativeTable.Type.OCCURRENCE_COUNT.getName(), "occurrenceCountTable"); + assertEquals(NativeTable.Type.PROXIMITY.getName(), "proximityTable"); + assertEquals(NativeTable.Type.REVERSE_PROXIMITY.getName(), "reverseProximityTable"); + assertEquals(NativeTable.Type.WEIGHT.getName(), "weightTable"); + } + @Test + public void testDefinitions() { + NativeRankTypeDefinitionSet defs = new NativeRankTypeDefinitionSet("default"); + + NativeRankTypeDefinition rank; + Iterator tables; + + assertEquals(4, defs.types().size()); + + { + rank = defs.getRankTypeDefinition(RankType.EMPTY); + assertNotNull(rank); + assertEquals(RankType.EMPTY, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + rank = defs.getRankTypeDefinition(RankType.ABOUT); + assertNotNull(rank); + assertEquals(RankType.ABOUT, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + rank = defs.getRankTypeDefinition(RankType.IDENTITY); + assertNotNull(rank); + assertEquals(RankType.IDENTITY, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + rank = defs.getRankTypeDefinition(RankType.TAGS); + assertNotNull(rank); + assertEquals(RankType.TAGS, rank.getType()); + tables = rank.rankSettingIterator(); + assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"), tables.next()); + assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)"), tables.next()); + assertFalse(tables.hasNext()); + } + + { + assertEquals(RankType.ABOUT, defs.getRankTypeDefinition(RankType.DEFAULT).getType()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java new file mode 100644 index 00000000000..baee7bec2a2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class NearestNeighborTestCase extends AbstractExportingTestCase { + + @Test + public void testNearestNeighbor() throws IOException, ParseException { + try { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("nearestneighbor"); + + CompiledQueryProfileRegistry queryProfiles = + QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5 + queryProfiles.getComponent("default")); + fail("This should fail when q_vec is parsed as a tensor"); + } catch (IllegalArgumentException e) { + // success + assertEquals("Could not set 'ranking.features.query(q_vec)' to '[1,2,3,4,5,6]'", e.getMessage()); + } catch (RuntimeException e) { + e.printStackTrace(); + throw e; + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java new file mode 100644 index 00000000000..6e584099331 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import com.yahoo.component.ComponentId; + +import static org.junit.Assert.assertEquals; + +public class NeuralNetTestCase extends AbstractExportingTestCase { + + @Test + public void testNeuralNet() throws IOException, ParseException { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("neuralnet"); + // Verify that query profiles end up correct when passed through the same intermediate forms as a full system + CompiledQueryProfileRegistry queryProfiles = + QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + assertNeuralNetQuery(c, queryProfiles.getComponent("default")); + } + + @Test + public void testNeuralNet_noQueryProfiles() throws IOException, ParseException { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("neuralnet_noqueryprofile"); + } + + private void assertNeuralNetQuery(DerivedConfiguration c, CompiledQueryProfile defaultprofile) { + Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]", defaultprofile); + assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", + q.properties().get("ranking.features.query(b_1)").toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NuwaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NuwaTestCase.java new file mode 100644 index 00000000000..210c8a9bdd4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NuwaTestCase.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +/** + * A real world mlr ranking model, useful to benchmark memory and cpu usage. + * + * @author bratseth + */ +public class NuwaTestCase extends AbstractExportingTestCase { + + @Test + @Ignore + public void testNuwa() throws IOException, ParseException { + System.gc(); + long freeBytesBefore = Runtime.getRuntime().freeMemory(); + long totalBytesBefore = Runtime.getRuntime().totalMemory(); + + DerivedConfiguration configuration = assertCorrectDeriving("nuwa"); + + System.gc(); + long freeBytesAfter = Runtime.getRuntime().freeMemory(); + long totalBytesAfter = Runtime.getRuntime().totalMemory(); + long additionalAllocated = totalBytesAfter - totalBytesBefore; + System.out.println("Consumed " + ((freeBytesBefore - (freeBytesAfter - additionalAllocated) ) / 1000000) + " Mb"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/OrderIlscriptsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/OrderIlscriptsTestCase.java new file mode 100755 index 00000000000..8af0d0a21d3 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/OrderIlscriptsTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author Einar M R Rosenvinge + */ +public class OrderIlscriptsTestCase extends AbstractExportingTestCase { + + @Test + public void testOrderIlscripts() throws IOException, ParseException { + assertCorrectDeriving("orderilscripts"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/PrefixExactAttributeTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/PrefixExactAttributeTestCase.java new file mode 100644 index 00000000000..fdcb71432e4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/PrefixExactAttributeTestCase.java @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving of various field types + * + * @author bratseth + */ +public class PrefixExactAttributeTestCase extends AbstractExportingTestCase { + + @Test + public void testTypes() throws IOException, ParseException { + assertCorrectDeriving("prefixexactattribute"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java new file mode 100644 index 00000000000..a83db0caf5a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests a search definition with various rank profiles having different settings + * + * @author bratseth + */ +public class RankProfilesTestCase extends AbstractExportingTestCase { + @Test + public void testRankProfiles() throws IOException, ParseException { + assertCorrectDeriving("rankprofiles", null, new TestProperties(), new TestableDeployLogger()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/RankPropertiesTestCase.java new file mode 100644 index 00000000000..8db880e56fe --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/RankPropertiesTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseth + */ +public class RankPropertiesTestCase extends AbstractExportingTestCase { + + @Test + public void testRankProperties() throws IOException, ParseException { + assertCorrectDeriving("rankproperties"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFieldsTestCase.java new file mode 100644 index 00000000000..99d0cf8bf6d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFieldsTestCase.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author geirst + */ +public class ReferenceFieldsTestCase extends AbstractExportingTestCase { + + @Test + public void configs_related_to_reference_fields_are_derived() throws IOException, ParseException { + assertCorrectDeriving("reference_fields", "ad", new TestableDeployLogger()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFromSeveralTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFromSeveralTestCase.java new file mode 100644 index 00000000000..ff4506a7f57 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFromSeveralTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import org.junit.Test; + +/** + * Tests deriving a configuration with references from multiple .sd files + * + * @author arnej + */ +public class ReferenceFromSeveralTestCase extends AbstractExportingTestCase { + + @Test + public void testDocManConfigs() throws Exception { + var logger = new TestableDeployLogger(); + var props = new TestProperties(); + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory + ("src/test/derived/reference_from_several/", new MockFileRegistry(), logger, props); + derive("reference_from_several", builder, builder.getSchema("foo")); + assertCorrectConfigFiles("reference_from_several"); + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaInheritanceTestCase.java new file mode 100644 index 00000000000..1b5d55158b5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaInheritanceTestCase.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.io.IOUtils; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * @author bratseth + */ +public class SchemaInheritanceTestCase extends AbstractExportingTestCase { + + @Test + public void testIt() throws IOException, ParseException { + try { + ApplicationBuilder builder = ApplicationBuilder.createFromDirectory("src/test/derived/schemainheritance/", + new MockFileRegistry(), + new TestableDeployLogger(), + new TestProperties()); + derive("schemainheritance", builder, builder.getSchema("child")); + assertCorrectConfigFiles("schemainheritance"); + } + finally { + IOUtils.recursiveDeleteDir(new File("src/test/derived/schemainheritance/models.generated/")); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java new file mode 100644 index 00000000000..e672763f13c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java @@ -0,0 +1,146 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + * @author bjorncs + */ +public class SchemaOrdererTestCase extends AbstractSchemaTestCase { + + private static Map createSchemas() { + Map schemas = new HashMap<>(); + + Schema grandParent = createSchema("grandParent", schemas); + + Schema mother = createSchema("mother", schemas); + inherit(mother, grandParent); + + Schema father = createSchema("father", schemas); + inherit(father, grandParent); + createDocumentReference(father, mother, "wife_ref"); + + Schema daugther = createSchema("daughter", schemas); + inherit(daugther, father); + inherit(daugther, mother); + + Schema son = createSchema("son", schemas); + inherit(son, father); + inherit(son, mother); + + Schema product = createSchema("product", schemas); + + Schema pc = createSchema("pc", schemas); + inherit(pc, product); + Schema pcAccessory = createSchema("accessory-pc", schemas); + inherit(pcAccessory, product); + createDocumentReference(pcAccessory, pc, "pc_ref"); + + createSchema("alone", schemas); + + return schemas; + } + + private static Schema createSchema(String name, Map schemas) { + Schema schema = new Schema(name, MockApplicationPackage.createEmpty()); + SDDocumentType document = new SDDocumentType(name); + document.setDocumentReferences(new DocumentReferences(emptyMap())); + schema.addDocument(document); + schemas.put(schema.getName(), schema); + return schema; + } + + private static void inherit(Schema inheritee, Schema inherited) { + inheritee.getDocument().inherit(inherited.getDocument()); + } + + private static void assertOrder(List expectedSearchOrder, List inputNames) { + Map schemas = createSchemas(); + List inputSchemas = inputNames.stream() + .map(schemas::get) + .map(Objects::requireNonNull) + .collect(toList()); + List actualSearchOrder = new SearchOrderer() + .order(inputSchemas) + .stream() + .map(Schema::getName) + .collect(toList()); + assertEquals(expectedSearchOrder, actualSearchOrder); + } + + @SuppressWarnings("deprecation") + private static void createDocumentReference(Schema from, Schema to, String refFieldName) { + SDDocumentType fromDocument = from.getDocument(); + SDField refField = new TemporarySDField(fromDocument, refFieldName, NewDocumentReferenceDataType.forDocumentName(to.getName())); + fromDocument.addField(refField); + Map originalMap = fromDocument.getDocumentReferences().get().referenceMap(); + HashMap modifiedMap = new HashMap<>(originalMap); + modifiedMap.put(refFieldName, new DocumentReference(refField, to)); + fromDocument.setDocumentReferences(new DocumentReferences(modifiedMap)); + } + + + @Test + public void testPerfectOrderingIsKept() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); + } + + @Test + public void testOneLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); + } + + @Test + public void testMultiLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); + } + + @Test + public void testAloneIsKeptInPlaceWithMultiLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); + } + + @Test + public void testPartialMultiLevelReordering() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); + } + + @Test + public void testMultilevelReorderingAccrossHierarchies() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); + } + + @Test + public void referees_are_ordered_before_referrer() { + assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"), + Arrays.asList("accessory-pc", "daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); + } + + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SimpleInheritTestCase.java new file mode 100644 index 00000000000..d8b39bfd978 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SimpleInheritTestCase.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * Tests really simple inheriting + */ +public class SimpleInheritTestCase extends AbstractExportingTestCase { + + @Test + public void testEmptyChild() throws IOException, ParseException { + String name = "emptychild"; + final String expectedResultsDirName = "src/test/derived/" + name + "/"; + + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(expectedResultsDirName + "parent.sd"); + builder.addSchemaFile(expectedResultsDirName + "child.sd"); + builder.build(true); + + Schema schema = builder.getSchema("child"); + + String toDirName = "temp/" + name; + File toDir = new File(toDirName); + toDir.mkdirs(); + deleteContent(toDir); + + DerivedConfiguration config = new DerivedConfiguration(schema, builder.getRankProfileRegistry()); + config.export(toDirName); + + checkDir(toDirName, expectedResultsDirName); + } + + private void checkDir(String toDirName, String expectedResultsDirName) throws IOException { + File[] files = new File(expectedResultsDirName).listFiles(); + for (File file : files) { + if ( ! file.getName().endsWith(".cfg")) continue; + assertEqualFiles(file.getPath(), toDirName + "/" + file.getName(), false); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SliceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SliceTestCase.java new file mode 100644 index 00000000000..2aad47dae6c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SliceTestCase.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.component.ComponentId; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SliceTestCase extends AbstractExportingTestCase { + + @Test + public void testSlice() throws IOException, ParseException { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("slice"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SortingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SortingTestCase.java new file mode 100644 index 00000000000..0c091a7a367 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SortingTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests sort settings + * + * @author baldersheim + */ +public class SortingTestCase extends AbstractExportingTestCase { + + @Test + public void testDocumentDerivingNewParser() throws IOException, ParseException { + assertCorrectDeriving("sorting", new TestProperties()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/StreamingStructTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/StreamingStructTestCase.java new file mode 100755 index 00000000000..6f27930e239 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/StreamingStructTestCase.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests streaming search configuration deriving for structs + * + * @author bratseth + */ +public class StreamingStructTestCase extends AbstractExportingTestCase { + + @Test + public void testStreamingStruct() throws IOException, ParseException { + assertCorrectDeriving("streamingstruct"); + } + + @Test + public void testStreamingStructExplicitDefaultSummaryClass() throws IOException, ParseException { + assertCorrectDeriving("streamingstructdefault"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/StructAnyOrderTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/StructAnyOrderTestCase.java new file mode 100755 index 00000000000..865b5da87cf --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/StructAnyOrderTestCase.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author Einar M R Rosenvinge + */ +public class StructAnyOrderTestCase extends AbstractExportingTestCase { + @Test + public void testStructAnyOrder() throws IOException, ParseException { + assertCorrectDeriving("structanyorder"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/StructInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/StructInheritanceTestCase.java new file mode 100644 index 00000000000..092e64420e8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/StructInheritanceTestCase.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + + +import com.yahoo.schema.ApplicationBuilder; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; + + +import org.junit.rules.TemporaryFolder; + +/** + * Tests struct inheritance + * + * @author arnej + */ +public class StructInheritanceTestCase extends AbstractExportingTestCase { + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void requireThatStructCanInherit() throws IOException, ParseException { + String dir = "src/test/derived/structinheritance/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "simple.sd"); + builder.build(false); + derive("structinheritance", builder, builder.getSchema("simple")); + assertCorrectConfigFiles("structinheritance"); + } + + @Test + public void requireThatRedeclareIsNotAllowed() throws IOException, ParseException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("cannot inherit from base and redeclare field name"); + String dir = "src/test/derived/structinheritance/"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(dir + "bad.sd"); + builder.build(true); + derive("structinheritance", builder, builder.getSchema("bad")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java new file mode 100644 index 00000000000..f0fc58b97e5 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java @@ -0,0 +1,187 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.schema.*; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.Processing; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Set; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +/** + * Tests summary map extraction + * + * @author bratseth + */ +public class SummaryMapTestCase extends AbstractSchemaTestCase { + @Test + public void testDeriving() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); + SummaryMap summaryMap = new SummaryMap(schema); + + Iterator transforms = summaryMap.resultTransforms().values().iterator(); + FieldResultTransform transform = transforms.next(); + assertEquals("dyndesc", transform.getFieldName()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); + + transform = transforms.next(); + assertEquals("dynlong", transform.getFieldName()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); + + transform = transforms.next(); + assertEquals("dyndesc2", transform.getFieldName()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); + + transform = transforms.next(); + assertEquals("measurement", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); + + transform = transforms.next(); + assertEquals("rankfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); + + transform = transforms.next(); + assertEquals("summaryfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); + + transform = transforms.next(); + assertEquals("popsiness", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); + + transform = transforms.next(); + assertEquals("popularity", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); + + transform = transforms.next(); + assertEquals("access", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); + + assertFalse(transforms.hasNext()); + } + @Test + public void testPositionDeriving() { + Schema schema = new Schema("store", MockApplicationPackage.createEmpty()); + SDDocumentType document = new SDDocumentType("store"); + schema.addDocument(document); + String fieldName = "location"; + SDField field = document.addField(fieldName, PositionDataType.INSTANCE); + field.parseIndexingScript("{ attribute | summary }"); + new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), + true, false, Set.of()); + SummaryMap summaryMap = new SummaryMap(schema); + + Iterator transforms = summaryMap.resultTransforms().values().iterator(); + + FieldResultTransform transform = transforms.next(); + + assertEquals(fieldName, transform.getFieldName()); + assertEquals(SummaryTransform.GEOPOS, transform.getTransform()); + + transform = transforms.next(); + assertEquals("rankfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); + + transform = transforms.next(); + assertEquals("summaryfeatures", transform.getFieldName()); + assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); + + transform = transforms.next(); + assertEquals("location_zcurve", transform.getFieldName()); + assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + + assertFalse(transforms.hasNext()); + + SummarymapConfig.Builder scb = new SummarymapConfig.Builder(); + summaryMap.getConfig(scb); + SummarymapConfig c = scb.build(); + + assertEquals(-1, c.defaultoutputclass()); + assertEquals(c.override().size(), 4); + + assertEquals(c.override(0).field(), fieldName); + assertEquals(c.override(0).command(), "geopos"); + assertEquals(c.override(0).arguments(), PositionDataType.getZCurveFieldName(fieldName)); + + assertEquals(c.override(1).field(), "rankfeatures"); + assertEquals(c.override(1).command(), "rankfeatures"); + assertEquals(c.override(1).arguments(), ""); + + assertEquals(c.override(2).field(), "summaryfeatures"); + assertEquals(c.override(2).command(), "summaryfeatures"); + assertEquals(c.override(2).arguments(), ""); + + assertEquals(c.override(3).field(), "location_zcurve"); + assertEquals(c.override(3).command(), "attribute"); + assertEquals(c.override(3).arguments(), "location_zcurve"); + } + + @Test + public void testFailOnSummaryFieldSourceCollision() { + try { + ApplicationBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); + } catch (Exception e) { + assertTrue(e.getMessage().matches(".*equally named field.*")); + } + } + + @Test + public void source_field_is_passed_as_argument_in_matched_elements_filter_transforms() throws ParseException { + assertOverride(joinLines("field my_field type map {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + "}"), "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()); + + assertOverride(joinLines("field my_field type map {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}"), "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName()); + } + + @Test + public void commands_that_are_dynamic_and_require_the_query() { + assertTrue(SummaryMap.isDynamicCommand("dynamicteaser")); + assertTrue(SummaryMap.isDynamicCommand(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName())); + assertTrue(SummaryMap.isDynamicCommand(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName())); + assertFalse(SummaryMap.isDynamicCommand(SummaryTransform.ATTRIBUTE.getName())); + } + + private void assertOverride(String fieldContent, String expFieldName, String expCommand) throws ParseException { + var summaryMap = new SummaryMap(buildSearch(fieldContent)); + var cfgBuilder = new SummarymapConfig.Builder(); + summaryMap.getConfig(cfgBuilder); + var cfg = new SummarymapConfig(cfgBuilder); + var override = cfg.override(0); + assertEquals(expFieldName, override.field()); + assertEquals(expCommand, override.command()); + assertEquals(expFieldName, override.arguments()); + } + + private Schema buildSearch(String field) throws ParseException { + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + field, + " }", + "}")); + builder.build(true); + return builder.getSchema(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java new file mode 100644 index 00000000000..341d3ef7d43 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -0,0 +1,182 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.config.search.SummaryConfig; +import org.junit.Test; + +import java.io.IOException; +import java.util.Iterator; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests summary extraction + * + * @author bratseth + */ +public class SummaryTestCase extends AbstractSchemaTestCase { + + @Test + public void deriveRawAsBase64() throws ParseException { + String sd = joinLines( + "schema s {", + " raw-as-base64-in-summary", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); + SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.RAW, summary.fields().get("raw_field").getType()); + } + + @Test + public void deriveRawAsLegacy() throws ParseException { + String sd = joinLines( + "schema s {", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); + SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.DATA, summary.fields().get("raw_field").getType()); + } + + @Test + public void testDeriving() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); + SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); + assertEquals("default", summary.getName()); + + Iterator fields = summary.fields().values().iterator(); + + SummaryClassField field; + + assertEquals(13, summary.fields().size()); + + field = fields.next(); + assertEquals("exactemento", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("exact", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("title", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("description", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("dyndesc", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("longdesc", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("longstat", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("dynlong", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("dyndesc2", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + + field = fields.next(); + assertEquals("measurement", field.getName()); + assertEquals(SummaryClassField.Type.INTEGER, field.getType()); + + field = fields.next(); + assertEquals("rankfeatures", field.getName()); + assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); + + field = fields.next(); + assertEquals("summaryfeatures", field.getName()); + assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); + + field = fields.next(); + assertEquals("documentid", field.getName()); + assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); + } + + @Test + public void reference_fields_can_be_part_of_summary_classes() throws ParseException { + Schema adSchema = buildCampaignAdModel(); + + SummaryClass defaultClass = new SummaryClass(adSchema, adSchema.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.fields().get("campaign_ref").getType()); + assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.fields().get("other_campaign_ref").getType()); + + SummaryClass myClass = new SummaryClass(adSchema, adSchema.getSummary("my_summary"), new BaseDeployLogger()); + assertNull(myClass.fields().get("campaign_ref")); + assertEquals(SummaryClassField.Type.LONGSTRING, myClass.fields().get("other_campaign_ref").getType()); + } + + private static Schema buildCampaignAdModel() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema("search campaign { document campaign {} }"); + builder.addSchema(joinLines("search ad {", + " document ad {", + " field campaign_ref type reference {", + " indexing: summary | attribute", + " }", + " field other_campaign_ref type reference {", + " indexing: summary | attribute", + " }", + " }", + " document-summary my_summary {", + " summary other_campaign_ref type reference {}", + " }", + "}")); + builder.build(true); + return builder.getSchema("ad"); + } + + @Test + public void omit_summary_features_specified_for_document_summary() throws ParseException { + String sd = joinLines( + "schema test {", + " document test {", + " field foo type string { indexing: summary }", + " }", + " document-summary bar {", + " summary foo type string {}", + " omit-summary-features", + " }", + " document-summary baz {", + " summary foo type string {}", + " }", + "}"); + var search = ApplicationBuilder.createFromString(sd).getSchema(); + assertOmitSummaryFeatures(true, search, "bar"); + assertOmitSummaryFeatures(false, search, "baz"); + } + + private void assertOmitSummaryFeatures(boolean expected, Schema schema, String summaryName) { + var summary = new SummaryClass(schema, schema.getSummary(summaryName), new BaseDeployLogger()); + var config = new SummaryConfig.Classes(summary.getSummaryClassConfig()); + assertEquals(expected, config.omitsummaryfeatures()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TestableDeployLogger.java b/config-model/src/test/java/com/yahoo/schema/derived/TestableDeployLogger.java new file mode 100644 index 00000000000..bcde0f6b544 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/TestableDeployLogger.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.DeployLogger; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bratseth + */ +public class TestableDeployLogger implements DeployLogger { + + private static final Logger log = Logger.getLogger("DeployLogger"); + + public List warnings = new ArrayList<>(); + public List info = new ArrayList<>(); + + @Override + public final void log(Level level, String message) { + log.log(level, message); + if (level.equals(Level.WARNING)) + warnings.add(message); + if (level.equals(Level.INFO)) + info.add(message); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TokenizationTestCase.java new file mode 100755 index 00000000000..ac6acf172e9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/TokenizationTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseh + */ +public class TokenizationTestCase extends AbstractExportingTestCase { + + @Test + public void testTokenizationScripts() throws IOException, ParseException { + assertCorrectDeriving("tokenization"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TwoStreamingStructsTestCase.java new file mode 100644 index 00000000000..72411fa1770 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/TwoStreamingStructsTestCase.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test structs for streaming with another unrelated .sd present + * + * @author arnej27959 + */ +public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { + + @Test + public void testTwoStreamingStructsExporting() throws ParseException, IOException { + + String root = "src/test/derived/twostreamingstructs"; + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchemaFile(root + "/streamingstruct.sd"); + builder.addSchemaFile(root + "/whatever.sd"); + builder.build(true); + assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); + + builder = new ApplicationBuilder(); + builder.addSchemaFile(root + "/streamingstruct.sd"); + builder.addSchemaFile(root + "/whatever.sd"); + builder.build(true); + assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java new file mode 100644 index 00000000000..84a561924ca --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.Assert.assertFalse; +/** + * Tests automatic type conversion using multifield indices + * + * @author bratseth + */ +public class TypeConversionTestCase extends AbstractSchemaTestCase { + + /** Tests that exact-string stuff is not spilled over to the default index */ + @Test + public void testExactStringToStringTypeConversion() { + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); + RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); + SDDocumentType document = new SDDocumentType("test"); + schema.addDocument(document); + SDField a = new SDField(document, "a", DataType.STRING); + a.parseIndexingScript("{ index }"); + document.addField(a); + + new Processing().process(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), + true, false, Set.of()); + DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); + IndexInfo indexInfo = derived.getIndexInfo(); + assertFalse(indexInfo.hasCommand("default", "compact-to-term")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TypesTestCase.java new file mode 100644 index 00000000000..7443ef01c95 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/TypesTestCase.java @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests deriving of various field types + * + * @author bratseth + */ +public class TypesTestCase extends AbstractExportingTestCase { + + @Test + public void testTypes() throws IOException, ParseException { + assertCorrectDeriving("types"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java new file mode 100644 index 00000000000..c59f82a2c12 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class VsmFieldsTestCase { + + @SuppressWarnings("deprecation") + @Test + public void reference_type_field_is_unsearchable() { + Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + var sdoc = new SDDocumentType("test"); + schema.addDocument(sdoc); + SDField refField = new TemporarySDField(sdoc, "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type")); + refField.parseIndexingScript("{ summary }"); + schema.getDocument().addField(refField); + + VsmFields vsmFields = new VsmFields(schema); + VsmfieldsConfig.Builder cfgBuilder = new VsmfieldsConfig.Builder(); + vsmFields.getConfig(cfgBuilder); + VsmfieldsConfig cfg = cfgBuilder.build(); + + assertEquals(1, cfg.fieldspec().size()); + VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0); + assertEquals("ref_field", fieldSpec.name()); + assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.NONE, fieldSpec.searchmethod()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java new file mode 100644 index 00000000000..ea3f207df91 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java @@ -0,0 +1,244 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.document; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ComplexAttributeFieldUtilsTestCase { + + private static class FixtureBase { + + private final ImmutableSDField field; + + FixtureBase(String fieldName, String sdContent) throws ParseException { + Schema schema = ApplicationBuilder.createFromString(sdContent).getSchema(); + field = schema.getConcreteField(fieldName); + } + + public ImmutableSDField field() { + return field; + } + + boolean isSupportedComplexField() { + return ComplexAttributeFieldUtils.isSupportedComplexField(field()); + } + + boolean isArrayOfSimpleStruct() { + return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field()); + } + + boolean isMapOfSimpleStruct() { + return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field()); + } + + boolean isMapOfPrimitiveType() { + return ComplexAttributeFieldUtils.isMapOfPrimitiveType(field()); + } + + boolean isComplexFieldWithOnlyStructFieldAttributes() { + return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field()); + } + } + + private static class Fixture extends FixtureBase { + + Fixture(String fieldName, String sdFieldContent) throws ParseException { + super(fieldName, joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + sdFieldContent, + " }", + "}")); + } + } + + private static class ComplexFixture extends FixtureBase { + + ComplexFixture(String fieldName, String sdFieldContent) throws ParseException { + super(fieldName, joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weights type array {}", + " }", + sdFieldContent, + " }", + "}")); + } + } + + @Test + public void array_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { + Fixture f = new Fixture("elem_array", + joinLines("field elem_array type array {", + " indexing: summary", + " struct-field name { indexing: attribute }", + " struct-field weight { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + + @Test + public void array_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { + Fixture f = new Fixture("elem_array", + joinLines("field elem_array type array {", + " indexing: summary", + " struct-field weight { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + + @Test + public void map_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { + Fixture f = new Fixture("elem_map", + joinLines("field elem_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.name { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + + @Test + public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { + { + Fixture f = new Fixture("elem_map", + joinLines("field elem_map type map {", + " indexing: summary", + " struct-field value.name { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + Fixture f = new Fixture("elem_map", + joinLines("field elem_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + + @Test + public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { + Fixture f = new Fixture("str_map", + joinLines("field str_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + + @Test + public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { + { + Fixture f = new Fixture("int_map", + joinLines("field int_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + Fixture f = new Fixture("int_map", + joinLines("field int_map type map {", + " indexing: summary", + " struct-field value { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isMapOfPrimitiveType()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + + @Test + public void unsupported_complex_field_is_tagged_as_such() throws ParseException { + { + ComplexFixture f = new ComplexFixture("elem_array", + joinLines("field elem_array type array {", + " struct-field name { indexing: attribute }", + " struct-field weights { indexing: attribute }", + "}")); + assertFalse(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + ComplexFixture f = new ComplexFixture("elem_map", + joinLines("field elem_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.weights { indexing: attribute }", + "}")); + assertFalse(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + + @Test + public void only_struct_field_attributes_are_considered_when_tagging_a_complex_field() throws ParseException { + { + ComplexFixture f = new ComplexFixture("elem_array", + joinLines("field elem_array type array {", + " struct-field name { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertTrue(f.isArrayOfSimpleStruct()); + assertFalse(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + { + ComplexFixture f = new ComplexFixture("elem_map", + joinLines("field elem_map type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value.name { indexing: attribute }", + "}")); + assertTrue(f.isSupportedComplexField()); + assertFalse(f.isArrayOfSimpleStruct()); + assertTrue(f.isMapOfSimpleStruct()); + assertFalse(f.isMapOfPrimitiveType()); + assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/schema/document/HnswIndexParamsTestCase.java new file mode 100644 index 00000000000..8ef51369ecb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/document/HnswIndexParamsTestCase.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.document; + +import java.util.Optional; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class HnswIndexParamsTestCase { + + @Test + public void override_from() throws Exception { + var empty = new HnswIndexParams(); + var builder = new HnswIndexParams.Builder(); + builder.setMaxLinksPerNode(7); + builder.setMultiThreadedIndexing(false); + var one = builder.build(); + builder.setNeighborsToExploreAtInsert(42); + var three = builder.build(); + builder.setMaxLinksPerNode(17); + builder.setNeighborsToExploreAtInsert(500); + builder.setMultiThreadedIndexing(true); + var four = builder.build(); + + assertThat(empty.maxLinksPerNode(), is(16)); + assertThat(empty.neighborsToExploreAtInsert(), is(200)); + assertThat(empty.multiThreadedIndexing(), is(true)); + + assertThat(one.maxLinksPerNode(), is(7)); + assertThat(one.multiThreadedIndexing(), is(false)); + assertThat(three.neighborsToExploreAtInsert(), is(42)); + + assertThat(four.maxLinksPerNode(), is(17)); + assertThat(four.neighborsToExploreAtInsert(), is(500)); + assertThat(four.multiThreadedIndexing(), is(true)); + + var five = four.overrideFrom(Optional.of(empty)); + assertThat(five.maxLinksPerNode(), is(17)); + assertThat(five.neighborsToExploreAtInsert(), is(500)); + assertThat(five.multiThreadedIndexing(), is(true)); + + var six = four.overrideFrom(Optional.of(one)); + assertThat(six.maxLinksPerNode(), is(7)); + assertThat(six.neighborsToExploreAtInsert(), is(500)); + // This is explicitly set to false in 'one' + assertThat(six.multiThreadedIndexing(), is(false)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/parser/ConvertIntermediateTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/ConvertIntermediateTestCase.java new file mode 100644 index 00000000000..516c259013f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/parser/ConvertIntermediateTestCase.java @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.document.DocumentTypeManager; +import static com.yahoo.config.model.test.TestUtil.joinLines; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; + +/** + * @author arnej + */ +public class ConvertIntermediateTestCase { + + @Test + public void can_convert_minimal_schema() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " }", + "}"); + var collection = new IntermediateCollection(); + ParsedSchema schema = collection.addSchemaFromString(input); + assertEquals("foo", schema.getDocument().name()); + var docMan = new DocumentTypeManager(); + var converter = new ConvertSchemaCollection(collection, docMan); + converter.convertTypes(); + var dt = docMan.getDocumentType("foo"); + assertTrue(dt != null); + } + + @Test + public void can_convert_schema_files() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); + assertEquals(collection.getParsedSchemas().size(), 3); + var docMan = new DocumentTypeManager(); + var converter = new ConvertSchemaCollection(collection, docMan); + converter.convertTypes(); + var dt = docMan.getDocumentType("child"); + assertTrue(dt != null); + dt = docMan.getDocumentType("parent"); + assertTrue(dt != null); + dt = docMan.getDocumentType("grandparent"); + assertTrue(dt != null); + } + + @Test + public void can_convert_structs_and_annotations() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/converter/child.sd"); + collection.addSchemaFromFile("src/test/converter/other.sd"); + collection.addSchemaFromFile("src/test/converter/parent.sd"); + collection.addSchemaFromFile("src/test/converter/grandparent.sd"); + var docMan = new DocumentTypeManager(); + var converter = new ConvertSchemaCollection(collection, docMan); + converter.convertTypes(); + var dt = docMan.getDocumentType("child"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + dt = docMan.getDocumentType("parent"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + dt = docMan.getDocumentType("grandparent"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + dt = docMan.getDocumentType("other"); + assertTrue(dt != null); + for (var parent : dt.getInheritedTypes()) { + System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); + } + for (var field : dt.fieldSetAll()) { + System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java new file mode 100644 index 00000000000..c4ee1d27c8c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java @@ -0,0 +1,236 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.io.reader.NamedReader; +import static com.yahoo.config.model.test.TestUtil.joinLines; + +import java.nio.charset.StandardCharsets; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.List; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; + +/** + * @author arnej + */ +public class IntermediateCollectionTestCase { + + @Test + public void can_add_minimal_schema() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " }", + "}"); + var collection = new IntermediateCollection(); + ParsedSchema schema = collection.addSchemaFromString(input); + assertEquals("foo", schema.name()); + assertTrue(schema.hasDocument()); + assertEquals("foo", schema.getDocument().name()); + } + + @Test + public void names_may_differ() throws Exception { + String input = joinLines + ("schema foo_search {", + " document foo {", + " }", + "}"); + var collection = new IntermediateCollection(); + ParsedSchema schema = collection.addSchemaFromString(input); + assertEquals("foo_search", schema.name()); + assertTrue(schema.hasDocument()); + assertEquals("foo", schema.getDocument().name()); + } + + @Test + public void can_add_schema_files() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); + var schemes = collection.getParsedSchemas(); + assertEquals(schemes.size(), 3); + var schema = schemes.get("child"); + assertTrue(schema != null); + assertEquals(schema.name(), "child"); + schema = schemes.get("parent"); + assertTrue(schema != null); + assertEquals(schema.name(), "parent"); + schema = schemes.get("grandparent"); + assertTrue(schema != null); + assertEquals(schema.name(), "grandparent"); + } + + NamedReader readerOf(String fileName) throws Exception { + File f = new File(fileName); + FileReader fr = new FileReader(f, StandardCharsets.UTF_8); + BufferedReader br = new BufferedReader(fr); + return new NamedReader(fileName, br); + } + + @Test + public void can_add_schemas() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromReader(readerOf("src/test/derived/deriver/child.sd")); + collection.addSchemaFromReader(readerOf("src/test/derived/deriver/grandparent.sd")); + collection.addSchemaFromReader(readerOf("src/test/derived/deriver/parent.sd")); + var schemes = collection.getParsedSchemas(); + assertEquals(schemes.size(), 3); + var schema = schemes.get("child"); + assertTrue(schema != null); + assertEquals(schema.name(), "child"); + schema = schemes.get("parent"); + assertTrue(schema != null); + assertEquals(schema.name(), "parent"); + schema = schemes.get("grandparent"); + assertTrue(schema != null); + assertEquals(schema.name(), "grandparent"); + } + + ParsedRankProfile get(List all, String name) { + for (var rp : all) { + if (rp.name().equals(name)) return rp; + } + return null; + } + + @Test + public void can_add_extra_rank_profiles() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); + collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); + collection.addRankProfileFile("test", readerOf("src/test/derived/rankprofilemodularity/test/subdirectory/outside_schema2.profile")); + var schemes = collection.getParsedSchemas(); + assertEquals(schemes.size(), 1); + var schema = schemes.get("test"); + assertTrue(schema != null); + assertEquals(schema.name(), "test"); + var rankProfiles = schema.getRankProfiles(); + assertEquals(rankProfiles.size(), 7); + var outside = get(rankProfiles, "outside_schema1"); + assertTrue(outside != null); + assertEquals(outside.name(), "outside_schema1"); + var functions = outside.getFunctions(); + assertEquals(functions.size(), 1); + assertEquals(functions.get(0).name(), "fo1"); + outside = get(rankProfiles, "outside_schema2"); + assertTrue(outside != null); + assertEquals(outside.name(), "outside_schema2"); + functions = outside.getFunctions(); + assertEquals(functions.size(), 1); + assertEquals(functions.get(0).name(), "fo2"); + } + + @Test + public void name_mismatch_throws() throws Exception { + var collection = new IntermediateCollection(); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.addSchemaFromReader(readerOf("src/test/cfg/application/sdfilenametest/schemas/notmusic.sd"))); + assertEquals("The file containing schema 'music' must be named 'music.sd', was 'notmusic.sd'", + ex.getMessage()); + } + + @Test + public void bad_parse_throws() throws Exception { + var collection = new IntermediateCollection(); + var ex = assertThrows(ParseException.class, () -> + collection.addSchemaFromFile("src/test/examples/badparse.sd")); + assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/badparse.sd: Encountered")); + ex = assertThrows(ParseException.class, () -> + collection.addSchemaFromReader(readerOf("src/test/examples/badparse.sd"))); + assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/badparse.sd: Encountered")); + collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); + collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); + ex = assertThrows(ParseException.class, () -> + collection.addRankProfileFile("test", "src/test/examples/badparse.sd")); + assertTrue(ex.getMessage().startsWith("Failed parsing rank-profile from src/test/examples/badparse.sd: Encountered")); + } + + @Test + public void can_resolve_document_inheritance() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); + collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); + collection.resolveInternalConnections(); + var schemes = collection.getParsedSchemas(); + assertEquals(schemes.size(), 3); + var childDoc = schemes.get("child").getDocument(); + var inherits = childDoc.getResolvedInherits(); + assertEquals(inherits.size(), 1); + var parentDoc = inherits.get(0); + assertEquals(parentDoc.name(), "parent"); + inherits = parentDoc.getResolvedInherits(); + assertEquals(inherits.size(), 1); + assertEquals(inherits.get(0).name(), "grandparent"); + } + + @Test + public void can_detect_schema_inheritance_cycles() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo inherits bar { document foo {} }"); + collection.addSchemaFromString("schema bar inherits qux { document bar {} }"); + collection.addSchemaFromString("schema qux inherits foo { document qux {} }"); + assertEquals(collection.getParsedSchemas().size(), 3); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for schemas: ")); + } + + @Test + public void can_detect_document_inheritance_cycles() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo { document foo inherits bar {} }"); + collection.addSchemaFromString("schema bar { document bar inherits qux {} }"); + collection.addSchemaFromString("schema qux { document qux inherits foo {} }"); + assertEquals(collection.getParsedSchemas().size(), 3); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + System.err.println("ex: "+ex.getMessage()); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); + } + + @Test + public void can_detect_missing_doc() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo { document foo inherits bar {} }"); + collection.addSchemaFromString("schema qux { document qux inherits foo {} }"); + assertEquals(collection.getParsedSchemas().size(), 2); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + assertEquals("document foo inherits from unavailable document bar", ex.getMessage()); + } + + @Test + public void can_detect_document_reference_cycle() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo { document foo { field oneref type reference {} } }"); + collection.addSchemaFromString("schema bar { document bar { field tworef type reference {} } }"); + assertEquals(collection.getParsedSchemas().size(), 2); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + System.err.println("ex: "+ex.getMessage()); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); + } + + @Test + public void can_detect_cycles_with_reference() throws Exception { + var collection = new IntermediateCollection(); + collection.addSchemaFromString("schema foo { document foodoc inherits bardoc {} }"); + collection.addSchemaFromString("schema bar { document bardoc { field myref type reference { } } }"); + collection.addSchemaFromString("schema qux inherits foo { document qux inherits foodoc {} }"); + assertEquals(collection.getParsedSchemas().size(), 3); + var ex = assertThrows(IllegalArgumentException.class, () -> + collection.resolveInternalConnections()); + System.err.println("ex: "+ex.getMessage()); + assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); + } + + +} diff --git a/config-model/src/test/java/com/yahoo/schema/parser/ParsedDocumentTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/ParsedDocumentTestCase.java new file mode 100644 index 00000000000..9245b64b09e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/parser/ParsedDocumentTestCase.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +/** + * @author arnej + */ +public class ParsedDocumentTestCase { + + @Test + public void fields_can_be_added_once() throws Exception { + var doc = new ParsedDocument("foo"); + var stringType = ParsedType.fromName("string"); + doc.addField(new ParsedField("bar1", stringType)); + doc.addField(new ParsedField("zap", stringType)); + doc.addField(new ParsedField("bar2", stringType)); + doc.addField(new ParsedField("bar3", stringType)); + var e = assertThrows(IllegalArgumentException.class, () -> + doc.addField(new ParsedField("zap", stringType))); + System.err.println("As expected: "+e); + assertEquals("document 'foo' error: Duplicate (case insensitively) field 'zap' in document type 'foo'", e.getMessage()); + e = assertThrows(IllegalArgumentException.class, () -> + doc.addField(new ParsedField("ZAP", stringType))); + assertEquals("document 'foo' error: Duplicate (case insensitively) field 'ZAP' in document type 'foo'", e.getMessage()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java new file mode 100644 index 00000000000..d4e4f1dbb88 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -0,0 +1,276 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.parser; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.io.IOUtils; +import static com.yahoo.config.model.test.TestUtil.joinLines; + +import java.io.File; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; + +/** + * @author arnej + */ +public class SchemaParserTestCase { + + ParsedSchema parseString(String input) throws Exception { + var deployLogger = new BaseDeployLogger(); + var modelProperties = new TestProperties(); + var stream = new SimpleCharStream(input); + try { + var parser = new SchemaParser(stream, deployLogger, modelProperties); + return parser.schema(); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(pe.getMessage())); + } + } + + ParsedSchema parseFile(String fileName) throws Exception { + File file = new File(fileName); + return parseString(IOUtils.readFile(file)); + } + + @Test + public void minimal_schema_can_be_parsed() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " }", + "}"); + ParsedSchema schema = parseString(input); + assertEquals("foo", schema.name()); + assertTrue(schema.hasDocument()); + assertEquals("foo", schema.getDocument().name()); + } + + @Test + public void document_only_can_be_parsed() throws Exception { + String input = joinLines + ("document bar {", + "}"); + ParsedSchema schema = parseString(input); + assertEquals("bar", schema.name()); + assertTrue(schema.hasDocument()); + assertEquals("bar", schema.getDocument().name()); + } + + @Test + public void multiple_documents_disallowed() { + String input = joinLines + ("schema foo {", + " document foo {", + " }", + " document foo2 {", + " }", + "}"); + var e = assertThrows(IllegalArgumentException.class, () -> parseString(input)); + assertEquals("schema 'foo' error: already has document 'foo' so cannot add document 'foo2'", e.getMessage()); + } + + @Test + public void backwards_path_is_disallowed() { + assertEquals("'..' is not allowed in path", + assertThrows(IllegalArgumentException.class, + () -> parseString("schema foo {\n" + + " constant my_constant_tensor {\n" + + " file: foo/../bar\n" + + " type: tensor(x{},y{})\n" + + " }\n" + + "}\n")).getMessage()); + } + + void checkFileParses(String fileName) throws Exception { + var schema = parseFile(fileName); + assertNotNull(schema); + assertNotNull(schema.name()); + assertNotEquals("", schema.name()); + } + + // TODO: Many (all)? of the files below are parsed from other tests and can be removed from here + @Test + public void parse_various_old_sdfiles() throws Exception { + checkFileParses("src/test/cfg/search/data/travel/schemas/TTData.sd"); + checkFileParses("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); + checkFileParses("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); + checkFileParses("src/test/configmodel/types/other_doc.sd"); + checkFileParses("src/test/configmodel/types/types.sd"); + checkFileParses("src/test/configmodel/types/type_with_doc_field.sd"); + checkFileParses("src/test/derived/advanced/advanced.sd"); + checkFileParses("src/test/derived/annotationsimplicitstruct/annotationsimplicitstruct.sd"); + checkFileParses("src/test/derived/annotationsinheritance2/annotationsinheritance2.sd"); + checkFileParses("src/test/derived/annotationsinheritance/annotationsinheritance.sd"); + checkFileParses("src/test/derived/annotationsoutsideofdocument/annotationsoutsideofdocument.sd"); + checkFileParses("src/test/derived/annotationspolymorphy/annotationspolymorphy.sd"); + checkFileParses("src/test/derived/annotationsreference2/annotationsreference2.sd"); + checkFileParses("src/test/derived/annotationsreference/annotationsreference.sd"); + checkFileParses("src/test/derived/annotationssimple/annotationssimple.sd"); + checkFileParses("src/test/derived/annotationsstruct/annotationsstruct.sd"); + checkFileParses("src/test/derived/annotationsstructarray/annotationsstructarray.sd"); + checkFileParses("src/test/derived/array_of_struct_attribute/test.sd"); + checkFileParses("src/test/derived/arrays/arrays.sd"); + checkFileParses("src/test/derived/attributeprefetch/attributeprefetch.sd"); + checkFileParses("src/test/derived/attributerank/attributerank.sd"); + checkFileParses("src/test/derived/attributes/attributes.sd"); + checkFileParses("src/test/derived/combinedattributeandindexsearch/combinedattributeandindexsearch.sd"); + checkFileParses("src/test/derived/complex/complex.sd"); + checkFileParses("src/test/derived/deriver/child.sd"); + checkFileParses("src/test/derived/deriver/grandparent.sd"); + checkFileParses("src/test/derived/deriver/parent.sd"); + checkFileParses("src/test/derived/emptychild/child.sd"); + checkFileParses("src/test/derived/emptychild/parent.sd"); + checkFileParses("src/test/derived/emptydefault/emptydefault.sd"); + checkFileParses("src/test/derived/exactmatch/exactmatch.sd"); + checkFileParses("src/test/derived/fieldset/test.sd"); + checkFileParses("src/test/derived/flickr/flickrphotos.sd"); + checkFileParses("src/test/derived/function_arguments/test.sd"); + checkFileParses("src/test/derived/function_arguments_with_expressions/test.sd"); + checkFileParses("src/test/derived/gemini2/gemini.sd"); + checkFileParses("src/test/derived/hnsw_index/test.sd"); + checkFileParses("src/test/derived/id/id.sd"); + checkFileParses("src/test/derived/importedfields/child.sd"); + checkFileParses("src/test/derived/importedfields/grandparent.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/child_a.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/child_b.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/child_c.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/parent.sd"); + checkFileParses("src/test/derived/importedfields/parent_a.sd"); + checkFileParses("src/test/derived/importedfields/parent_b.sd"); + checkFileParses("src/test/derived/imported_position_field/child.sd"); + checkFileParses("src/test/derived/imported_position_field/parent.sd"); + checkFileParses("src/test/derived/imported_position_field_summary/child.sd"); + checkFileParses("src/test/derived/imported_position_field_summary/parent.sd"); + checkFileParses("src/test/derived/imported_struct_fields/child.sd"); + checkFileParses("src/test/derived/imported_struct_fields/parent.sd"); + checkFileParses("src/test/derived/indexinfo_fieldsets/indexinfo_fieldsets.sd"); + checkFileParses("src/test/derived/indexinfo_lowercase/indexinfo_lowercase.sd"); + checkFileParses("src/test/derived/indexschema/indexschema.sd"); + checkFileParses("src/test/derived/indexswitches/indexswitches.sd"); + checkFileParses("src/test/derived/inheritance/child.sd"); + checkFileParses("src/test/derived/inheritance/father.sd"); + checkFileParses("src/test/derived/inheritance/grandparent.sd"); + checkFileParses("src/test/derived/inheritance/mother.sd"); + checkFileParses("src/test/derived/inheritdiamond/child.sd"); + checkFileParses("src/test/derived/inheritdiamond/father.sd"); + checkFileParses("src/test/derived/inheritdiamond/grandparent.sd"); + checkFileParses("src/test/derived/inheritdiamond/mother.sd"); + checkFileParses("src/test/derived/inheritfromgrandparent/child.sd"); + checkFileParses("src/test/derived/inheritfromgrandparent/grandparent.sd"); + checkFileParses("src/test/derived/inheritfromgrandparent/parent.sd"); + checkFileParses("src/test/derived/inheritfromnull/inheritfromnull.sd"); + checkFileParses("src/test/derived/inheritfromparent/child.sd"); + checkFileParses("src/test/derived/inheritfromparent/parent.sd"); + checkFileParses("src/test/derived/inheritstruct/child.sd"); + checkFileParses("src/test/derived/inheritstruct/parent.sd"); + checkFileParses("src/test/derived/integerattributetostringindex/integerattributetostringindex.sd"); + checkFileParses("src/test/derived/language/language.sd"); + checkFileParses("src/test/derived/lowercase/lowercase.sd"); + checkFileParses("src/test/derived/mail/mail.sd"); + checkFileParses("src/test/derived/map_attribute/test.sd"); + checkFileParses("src/test/derived/map_of_struct_attribute/test.sd"); + checkFileParses("src/test/derived/mlr/mlr.sd"); + checkFileParses("src/test/derived/multiplesummaries/multiplesummaries.sd"); + checkFileParses("src/test/derived/music3/music3.sd"); + checkFileParses("src/test/derived/music/music.sd"); + checkFileParses("src/test/derived/namecollision/collision.sd"); + checkFileParses("src/test/derived/namecollision/collisionstruct.sd"); + checkFileParses("src/test/derived/nearestneighbor/test.sd"); + checkFileParses("src/test/derived/newrank/newrank.sd"); + checkFileParses("src/test/derived/nuwa/newsindex.sd"); + checkFileParses("src/test/derived/orderilscripts/orderilscripts.sd"); + checkFileParses("src/test/derived/position_array/position_array.sd"); + checkFileParses("src/test/derived/position_attribute/position_attribute.sd"); + checkFileParses("src/test/derived/position_extra/position_extra.sd"); + checkFileParses("src/test/derived/position_nosummary/position_nosummary.sd"); + checkFileParses("src/test/derived/position_summary/position_summary.sd"); + checkFileParses("src/test/derived/predicate_attribute/predicate_attribute.sd"); + checkFileParses("src/test/derived/prefixexactattribute/prefixexactattribute.sd"); + checkFileParses("src/test/derived/rankingexpression/rankexpression.sd"); + checkFileParses("src/test/derived/rankprofileinheritance/child.sd"); + checkFileParses("src/test/derived/rankprofileinheritance/parent1.sd"); + checkFileParses("src/test/derived/rankprofileinheritance/parent2.sd"); + checkFileParses("src/test/derived/rankprofilemodularity/test.sd"); + checkFileParses("src/test/derived/rankprofiles/rankprofiles.sd"); + checkFileParses("src/test/derived/rankproperties/rankproperties.sd"); + checkFileParses("src/test/derived/ranktypes/ranktypes.sd"); + checkFileParses("src/test/derived/reference_fields/ad.sd"); + checkFileParses("src/test/derived/reference_fields/campaign.sd"); + checkFileParses("src/test/derived/renamedfeatures/foo.sd"); + checkFileParses("src/test/derived/reserved_position/reserved_position.sd"); + checkFileParses("src/test/derived/schemainheritance/child.sd"); + checkFileParses("src/test/derived/schemainheritance/importedschema.sd"); + checkFileParses("src/test/derived/schemainheritance/parent.sd"); + checkFileParses("src/test/derived/slice/test.sd"); + checkFileParses("src/test/derived/streamingjuniper/streamingjuniper.sd"); + checkFileParses("src/test/derived/streamingstructdefault/streamingstructdefault.sd"); + checkFileParses("src/test/derived/streamingstruct/streamingstruct.sd"); + checkFileParses("src/test/derived/structandfieldset/test.sd"); + checkFileParses("src/test/derived/structanyorder/structanyorder.sd"); + checkFileParses("src/test/derived/structinheritance/bad.sd"); + checkFileParses("src/test/derived/structinheritance/simple.sd"); + checkFileParses("src/test/derived/tensor2/first.sd"); + checkFileParses("src/test/derived/tensor2/second.sd"); + checkFileParses("src/test/derived/tensor/tensor.sd"); + checkFileParses("src/test/derived/tokenization/tokenization.sd"); + checkFileParses("src/test/derived/twostreamingstructs/streamingstruct.sd"); + checkFileParses("src/test/derived/twostreamingstructs/whatever.sd"); + checkFileParses("src/test/derived/types/types.sd"); + checkFileParses("src/test/derived/uri_array/uri_array.sd"); + checkFileParses("src/test/derived/uri_wset/uri_wset.sd"); + checkFileParses("src/test/examples/arrays.sd"); + checkFileParses("src/test/examples/arraysweightedsets.sd"); + checkFileParses("src/test/examples/attributeposition.sd"); + checkFileParses("src/test/examples/attributesettings.sd"); + checkFileParses("src/test/examples/attributesexactmatch.sd"); + checkFileParses("src/test/examples/casing.sd"); + checkFileParses("src/test/examples/comment.sd"); + checkFileParses("src/test/examples/documentidinsummary.sd"); + checkFileParses("src/test/examples/fieldoftypedocument.sd"); + checkFileParses("src/test/examples/implicitsummaries_attribute.sd"); + checkFileParses("src/test/examples/implicitsummaryfields.sd"); + checkFileParses("src/test/examples/incorrectrankingexpressionfileref.sd"); + checkFileParses("src/test/examples/indexing_extra.sd"); + checkFileParses("src/test/examples/indexing_modify_field_no_output.sd"); + checkFileParses("src/test/examples/indexing.sd"); + checkFileParses("src/test/examples/indexrewrite.sd"); + checkFileParses("src/test/examples/indexsettings.sd"); + checkFileParses("src/test/examples/integerindex2attribute.sd"); + checkFileParses("src/test/examples/invalidimplicitsummarysource.sd"); + checkFileParses("src/test/examples/multiplesummaries.sd"); + checkFileParses("src/test/examples/music.sd"); + checkFileParses("src/test/examples/nextgen/boldedsummaryfields.sd"); + checkFileParses("src/test/examples/nextgen/dynamicsummaryfields.sd"); + checkFileParses("src/test/examples/nextgen/extrafield.sd"); + checkFileParses("src/test/examples/nextgen/implicitstructtypes.sd"); + checkFileParses("src/test/examples/nextgen/simple.sd"); + checkFileParses("src/test/examples/nextgen/summaryfield.sd"); + checkFileParses("src/test/examples/nextgen/toggleon.sd"); + checkFileParses("src/test/examples/nextgen/untransformedsummaryfields.sd"); + checkFileParses("src/test/examples/ngram.sd"); + checkFileParses("src/test/examples/outsidedoc.sd"); + checkFileParses("src/test/examples/outsidesummary.sd"); + checkFileParses("src/test/examples/position_array.sd"); + checkFileParses("src/test/examples/position_attribute.sd"); + checkFileParses("src/test/examples/position_base.sd"); + checkFileParses("src/test/examples/position_extra.sd"); + checkFileParses("src/test/examples/position_index.sd"); + checkFileParses("src/test/examples/position_inherited.sd"); + checkFileParses("src/test/examples/position_summary.sd"); + checkFileParses("src/test/examples/rankmodifier/literal.sd"); + checkFileParses("src/test/examples/rankpropvars.sd"); + checkFileParses("src/test/examples/reserved_words_as_field_names.sd"); + checkFileParses("src/test/examples/simple.sd"); + checkFileParses("src/test/examples/stemmingdefault.sd"); + checkFileParses("src/test/examples/stemmingsetting.sd"); + checkFileParses("src/test/examples/strange.sd"); + checkFileParses("src/test/examples/struct.sd"); + checkFileParses("src/test/examples/summaryfieldcollision.sd"); + checkFileParses("src/test/examples/weightedset-summaryto.sd"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java new file mode 100644 index 00000000000..0d64dd5c953 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * @author bjorncs + */ +public class AddAttributeTransformToSummaryOfImportedFieldsTest { + + private static final String IMPORTED_FIELD_NAME = "imported_myfield"; + private static final String DOCUMENT_NAME = "mydoc"; + private static final String SUMMARY_NAME = "mysummary"; + + @Test + public void attribute_summary_transform_applied_to_summary_field_of_imported_field() { + Schema schema = createSearchWithDocument(DOCUMENT_NAME); + schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME)); + schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, schema)); + + AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( + schema, null, null, null); + processor.process(true, false); + SummaryField summaryField = schema.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); + SummaryTransform actualTransform = summaryField.getTransform(); + assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); + } + + private static Schema createSearch(String documentType) { + return new Schema(documentType, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + } + + private static Schema createSearchWithDocument(String documentName) { + Schema schema = createSearch(documentName); + SDDocumentType document = new SDDocumentType(documentName, schema); + schema.addDocument(document); + return schema; + } + + private static ImportedFields createSingleImportedField(String fieldName) { + Schema targetSchema = createSearchWithDocument("target_doc"); + var doc = targetSchema.getDocument(); + SDField targetField = new SDField(doc, "target_field", DataType.INT); + DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); + ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); + return new ImportedFields(Collections.singletonMap(fieldName, importedField)); + } + + private static DocumentSummary createDocumentSummary(String fieldName, Schema schema) { + DocumentSummary summary = new DocumentSummary("mysummary", schema); + summary.add(new SummaryField(fieldName, DataType.INT)); + return summary; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java new file mode 100644 index 00000000000..103d08b39a8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java @@ -0,0 +1,260 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class AdjustPositionSummaryFieldsTestCase { + + @Test + public void test_pos_summary() { + SearchModel model = new SearchModel(false); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "pos_zcurve"); + model.assertSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos_zcurve"); + model.assertSummaryField("my_pos.distance", DataType.INT, SummaryTransform.DISTANCE, "pos_zcurve"); + } + + @Test + public void test_imported_pos_summary() { + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.resolve(); + model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "my_pos_zcurve"); + model.assertSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "my_pos_zcurve"); + model.assertSummaryField("my_pos.distance", DataType.INT, SummaryTransform.DISTANCE, "my_pos_zcurve"); + } + + @Test + public void test_imported_pos_summary_bad_source() { + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + // SummaryFieldsMustHaveValidSource processing not run in this test. + model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.NONE, "pos"); + model.assertNoSummaryField("my_pos.position"); + model.assertNoSummaryField("my_pos.distance"); + } + + @Test + public void test_imported_pos_summary_bad_datatype() { + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", DataType.getArray(PositionDataType.INSTANCE), null, "pos"); + model.resolve(); + model.assertSummaryField("my_pos", DataType.getArray(PositionDataType.INSTANCE), SummaryTransform.NONE, "pos"); + model.assertNoSummaryField("my_pos.position"); + model.assertNoSummaryField("my_pos.distance"); + } + + @Test + public void test_pos_summary_no_attr_no_rename() { + SearchModel model = new SearchModel(false, false, false); + model.addSummaryField("pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + model.assertSummaryField("pos", PositionDataType.INSTANCE, SummaryTransform.NONE, "pos"); + model.assertNoSummaryField("pos.position"); + model.assertNoSummaryField("pos.distance"); + } + + @Test + public void test_pos_default_summary_no_attr_no_rename() { + SearchModel model = new SearchModel(false, false, false); + model.resolve(); + assertNull(model.childSchema.getSummary("default")); // ImplicitSummaries processing not run in this test + } + + @Test + public void test_pos_summary_no_rename() { + SearchModel model = new SearchModel(false, true, false); + model.addSummaryField("pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + model.assertSummaryField("pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "pos_zcurve"); + model.assertSummaryField("pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos_zcurve"); + model.assertSummaryField("pos.distance", DataType.INT, SummaryTransform.DISTANCE, "pos_zcurve"); + } + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void test_pos_summary_no_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos': No position attribute 'pos_zcurve'"); + SearchModel model = new SearchModel(false, false, false); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + } + + @Test + public void test_pos_summary_bad_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos': No position attribute 'pos_zcurve'"); + SearchModel model = new SearchModel(false, false, true); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); + model.resolve(); + } + + @Test + public void test_imported_pos_summary_no_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_pos_zcurve': " + + "Field 'pos_zcurve' via reference field 'ref': Not found"); + SearchModel model = new SearchModel(true, false, false); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.resolve(); + } + + @Test + public void test_imported_pos_summary_bad_attr() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos': " + + "No position attribute 'my_pos_zcurve'"); + SearchModel model = new SearchModel(true, false, true); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.resolve(); + } + + @Test + public void test_my_pos_position_summary_bad_datatype() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " + + "exists with type 'datatype string (code: 2)', should be of type 'datatype Array (code: -1486737430)"); + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.addSummaryField("my_pos.position", DataType.STRING, null, "pos"); + model.resolve(); + } + + @Test + public void test_my_pos_position_summary_bad_transform() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " + + "has summary transform 'none', should have transform 'positions'"); + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.addSummaryField("my_pos.position", DataType.getArray(DataType.STRING), null, "pos"); + model.resolve(); + } + + @Test + public void test_my_pos_position_summary_bad_source() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " + + "has source '[source field 'pos']', should have source 'source field 'my_pos_zcurve''"); + SearchModel model = new SearchModel(); + model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); + model.addSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos"); + model.resolve(); + } + + static class SearchModel extends ParentChildSearchModel { + + SearchModel() { + this(true); + } + + SearchModel(boolean importedPos) { + this(importedPos, true, false); + } + + SearchModel(boolean importedPos, boolean setupPosAttr, boolean setupBadAttr) { + super(); + if (importedPos) { + createPositionField(parentSchema, setupPosAttr, setupBadAttr); + } + addRefField(childSchema, parentSchema, "ref"); + if (importedPos) { + addImportedField("my_pos", "ref", "pos"); + } else { + createPositionField(childSchema, setupPosAttr, setupBadAttr); + } + } + + private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) { + String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }"; + var doc = schema.getDocument(); + doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript)); + if (setupBadAttr) { + doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }")); + } + } + + void addSummaryField(String fieldName, DataType dataType, SummaryTransform transform, String source) { + addSummaryField("my_summary", fieldName, dataType, transform, source); + } + + public void addSummaryField(String summaryName, String fieldName, DataType dataType, SummaryTransform transform, String source) { + DocumentSummary summary = childSchema.getSummary(summaryName); + if (summary == null) { + summary = new DocumentSummary(summaryName, childSchema); + childSchema.addSummary(summary); + } + SummaryField summaryField = new SummaryField(fieldName, dataType); + if (source != null) { + summaryField.addSource(source); + } + if (transform != null) { + summaryField.setTransform(transform); + } + summary.add(summaryField); + } + + public void assertNoSummaryField(String fieldName) { + assertNoSummaryField("my_summary", fieldName); + } + + public void assertNoSummaryField(String summaryName, String fieldName) { + DocumentSummary summary = childSchema.getSummary(summaryName); + assertNotNull(summary); + SummaryField summaryField = summary.getSummaryField(fieldName); + assertNull(summaryField); + } + + public void assertSummaryField(String fieldName, DataType dataType, SummaryTransform transform, String source) { + assertSummaryField("my_summary", fieldName, dataType, transform, source); + } + + public void assertSummaryField(String summaryName, String fieldName, DataType dataType, SummaryTransform transform, String source) { + DocumentSummary summary = childSchema.getSummary(summaryName); + assertNotNull(summary); + SummaryField summaryField = summary.getSummaryField(fieldName); + assertNotNull(summaryField); + assertEquals(dataType, summaryField.getDataType()); + assertEquals(transform, summaryField.getTransform()); + if (source == null) { + assertEquals(0, summaryField.getSourceCount()); + } else { + assertEquals(1, summaryField.getSourceCount()); + assertEquals(source, summaryField.getSingleSource()); + } + } + + public void resolve() { + resolve(parentSchema); + resolve(childSchema); + } + + private static void resolve(Schema schema) { + new CreatePositionZCurve(schema, null, null, null).process(true, false); + assertNotNull(schema.temporaryImportedFields().get()); + assertFalse(schema.importedFields().isPresent()); + new ImportedFieldsResolver(schema, null, null, null).process(true, false); + assertNotNull(schema.importedFields().get()); + new AdjustPositionSummaryFields(schema, null, null, null).process(true, false); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java new file mode 100644 index 00000000000..82650598f29 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.IndexingScript; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen Hult + */ +public abstract class AssertIndexingScript { + + public static void assertIndexing(List expected, Schema schema) { + assertIndexing(expected, new IndexingScript(schema).expressions()); + } + + public static void assertIndexing(List expected, IndexingScript script) { + assertIndexing(expected, script.expressions()); + } + + public static void assertIndexing(List expected, Iterable actual) { + List parsedExpected = new LinkedList<>(); + for (String str : expected) { + try { + parsedExpected.add(Expression.fromString(str).toString()); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + for (Expression actualExp : actual) { + String str = actualExp.toString(); + assertTrue("Unexpected: " + str, parsedExpected.remove(str)); + } + assertTrue("Missing: " + parsedExpected.toString(), parsedExpected.isEmpty()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java new file mode 100644 index 00000000000..0b4d7c3a2b6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; + +import java.io.IOException; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen Hult + */ +public abstract class AssertSearchBuilder { + + public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { + assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); + } + + public static void assertBuildFails(String searchDefinitionFileName, String expectedException) + throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile(searchDefinitionFileName); + fail(searchDefinitionFileName); + } catch (IllegalArgumentException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java new file mode 100644 index 00000000000..40ebe458c74 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +/** + * Attributes should be implicitly exact-match in some cases + * @author vegardh + * + */ +public class AttributesExactMatchTestCase extends AbstractSchemaTestCase { + @Test + public void testAttributesExactMatch() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); + assertEquals(schema.getConcreteField("color").getMatching().getType(), MatchType.EXACT); + assertEquals(schema.getConcreteField("artist").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("drummer").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("guitarist").getMatching().getType(), MatchType.TEXT); + assertEquals(schema.getConcreteField("saxophonist_arr").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("flutist").getMatching().getType(), MatchType.TEXT); + + assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.WORD)); + assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.WORD)); + assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.WORD)); + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java new file mode 100644 index 00000000000..c37bc8085c7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class BoldingTestCase extends AbstractSchemaTestCase { + + private final String boldonnonstring = + "search boldnonstring {\n" + + " document boldnonstring {\n" + + " field title type string {\n" + + " indexing: summary | index\n" + + " }\n" + + "\n" + + " field year4 type int {\n" + + " indexing: summary | attribute\n" + + " bolding: on\n" + + " }\n" + + " }\n" + + "}\n"; + + @Test + public void testBoldOnNonString() throws ParseException { + try { + ApplicationBuilder.createFromString(boldonnonstring); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("'bolding: on' for non-text field 'year4' (datatype int (code: 0)) is not allowed", + e.getMessage()); + } + } + + private final String boldonarray = + "search boldonarray {\n" + + " document boldonarray {\n" + + " field myarray type array {\n" + + " indexing: summary | index\n" + + " bolding: on\n" + + " }\n" + + " }\n" + + "}\n"; + + @Test + public void testBoldOnArray() throws ParseException { + try { + ApplicationBuilder.createFromString(boldonarray); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("'bolding: on' for non-text field 'myarray' (datatype Array (code: -1486737430)) is not allowed", + e.getMessage()); + } + } + +} + + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java new file mode 100644 index 00000000000..287cc6559d1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class BoolAttributeValidatorTestCase { + + @Test + public void array_of_bool_attribute_is_not_supported() throws ParseException { + try { + createFromString(getSd("field b type array { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", + e.getMessage()); + } + } + + @Test + public void weigtedset_of_bool_attribute_is_not_supported() throws ParseException { + try { + createFromString(getSd("field b type weightedset { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", + e.getMessage()); + } + } + + private String getSd(String field) { + return joinLines( + "schema test {", + " document test {", + " " + field, + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java new file mode 100644 index 00000000000..1956b87a689 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java @@ -0,0 +1,250 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.schema.processing; + +import com.yahoo.config.model.test.TestUtil; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * Test configuration of dictionary control. + * + * @author baldersheim + */ +public class DictionaryTestCase { + private static AttributesConfig getConfig(Schema schema) { + AttributeFields attributes = new AttributeFields(schema); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 130000, true); + return builder.build(); + } + private Schema createSearch(String def) throws ParseException { + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); + return sb.getSchema(); + } + @Test + public void testDefaultDictionarySettings() throws ParseException { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field n1 type int {", + " indexing: summary | attribute", + " }", + " }", + "}"); + Schema schema = createSearch(def); + assertNull(schema.getAttribute("s1").getDictionary()); + assertNull(schema.getAttribute("n1").getDictionary()); + assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, + getConfig(schema).attribute().get(0).dictionary().type()); + assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, + getConfig(schema).attribute().get(1).dictionary().type()); + assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, + getConfig(schema).attribute().get(0).dictionary().match()); + assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, + getConfig(schema).attribute().get(1).dictionary().match()); + } + + Schema verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException + { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field n1 type " + type + " {", + " indexing: summary | attribute", + " attribute:fast-search", + TestUtil.joinLines(cfg), + " }", + " }", + "}"); + Schema schema = createSearch(def); + AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig = toCfg(expected); + assertEquals(expected, schema.getAttribute("n1").getDictionary().getType()); + assertEquals(expectedConfig, getConfig(schema).attribute().get(0).dictionary().type()); + return schema; + } + + AttributesConfig.Attribute.Dictionary.Type.Enum toCfg(Dictionary.Type v) { + return (v == Dictionary.Type.HASH) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.HASH + : (v == Dictionary.Type.BTREE) + ? AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE + : AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE_AND_HASH; + } + AttributesConfig.Attribute.Dictionary.Match.Enum toCfg(Case v) { + return (v == Case.CASED) + ? AttributesConfig.Attribute.Dictionary.Match.Enum.CASED + : AttributesConfig.Attribute.Dictionary.Match.Enum.UNCASED; + } + + void verifyStringDictionaryControl(Dictionary.Type expectedType, Case expectedCase, Case matchCasing, + String ... cfg) throws ParseException + { + Schema schema = verifyDictionaryControl(expectedType, "string", cfg); + ImmutableSDField f = schema.getField("n1"); + AttributesConfig.Attribute.Dictionary.Match.Enum expectedCaseCfg = toCfg(expectedCase); + assertEquals(matchCasing, f.getMatching().getCase()); + assertEquals(expectedCase, schema.getAttribute("n1").getDictionary().getMatch()); + assertEquals(expectedCaseCfg, getConfig(schema).attribute().get(0).dictionary().match()); + } + + @Test + public void testCasedBtreeSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:cased"); + } + + @Test + public void testNumericBtreeSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:btree"); + } + @Test + public void testNumericHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.HASH, "int", "dictionary:hash"); + } + @Test + public void testNumericBtreeAndHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "int", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNumericArrayBtreeAndHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "array", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNumericWSetBtreeAndHashSettings() throws ParseException { + verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "weightedset", "dictionary:btree", "dictionary:hash"); + } + @Test + public void testStringBtreeSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary:btree"); + } + @Test + public void testStringBtreeUnCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary { btree\nuncased\n}"); + } + @Test + public void testStringBtreeCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.CASED, Case.CASED, "dictionary { btree\ncased\n}", "match:cased"); + } + @Test + public void testStringHashSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary:hash"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashUnCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nuncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': hash dictionary require cased match", e.getMessage()); + } + } + @Test + public void testStringHashBothCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}", "match:cased"); + } + @Test + public void testStringHashCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } + } + @Test + public void testStringBtreeHashSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary{hash\nbtree\n}"); + } + @Test + public void testStringBtreeHashUnCasedSettings() throws ParseException { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nbtree\nuncased\n}"); + } + @Test + public void testStringBtreeHashCasedSettings() throws ParseException { + try { + verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.CASED, Case.CASED, "dictionary { btree\nhash\ncased\n}"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); + } + } + @Test + public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { + String def = TestUtil.joinLines( + "schema test {", + " document test {", + " field n1 type bool {", + " indexing: summary | attribute", + " dictionary:btree", + " }", + " }", + "}"); + try { + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); + fail("Controlling dictionary for non-numeric fields are not yet supported."); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': You can only specify 'dictionary:' for numeric or string fields", e.getMessage()); + } + } + @Test + public void testNonFastSearchNumericFieldsFailsDictionaryControl() throws ParseException { + String def = TestUtil.joinLines( + "schema test {", + " document test {", + " field n1 type int {", + " indexing: summary | attribute", + " dictionary:btree", + " }", + " }", + "}"); + try { + ApplicationBuilder sb = ApplicationBuilder.createFromString(def); + fail("Controlling dictionary for non-fast-search fields are not allowed."); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); + } + } + + @Test + public void testCasingForNonFastSearch() throws ParseException { + String def = TestUtil.joinLines( + "schema test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field s2 type string {", + " indexing: attribute | summary", + " match:uncased", + " }", + " field s3 type string {", + " indexing: attribute | summary", + " match:cased", + " }", + " }", + "}"); + Schema schema = createSearch(def); + assertEquals(Case.UNCASED, schema.getAttribute("s1").getCase()); + assertEquals(Case.UNCASED, schema.getAttribute("s2").getCase()); + assertEquals(Case.CASED, schema.getAttribute("s3").getCase()); + assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(schema).attribute().get(0).match()); + assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(schema).attribute().get(1).match()); + assertEquals(AttributesConfig.Attribute.Match.CASED, getConfig(schema).attribute().get(2).match()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java new file mode 100644 index 00000000000..64b0a437b1d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +/** + * @author lesters + */ +public class DisallowComplexMapAndWsetKeyTypesTestCase { + + @Test(expected = IllegalArgumentException.class) + public void requireThatComplexTypesForMapKeysFail() throws ParseException { + testFieldType("map"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatComplexTypesForWsetFail() throws ParseException { + testFieldType("weightedset"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatNestedComplexTypesForMapFail() throws ParseException { + testFieldType("array>"); + } + + @Test + public void requireThatNestedComplexValuesForMapSucceed() throws ParseException { + testFieldType("array>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatNestedComplexTypesForWsetFail() throws ParseException { + testFieldType("array>"); + } + + @Test(expected = IllegalArgumentException.class) + public void requireThatDeepNestedComplexTypesForMapFail() throws ParseException { + testFieldType("map>"); + } + + private void testFieldType(String fieldType) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " struct mystruct {}\n" + + " field a type " + fieldType + " {}\n" + + " }\n" + + "}\n"); + builder.build(true); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java new file mode 100644 index 00000000000..b249b407c7b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.test.TestUtil; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * @author bjorncs + */ +public class FastAccessValidatorTest { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_on_incompatible_use_of_fastaccess() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema( + TestUtil.joinLines( + "schema parent {", + " document parent {", + " field int_field type int { indexing: attribute }", + " }", + "}")); + builder.addSchema( + TestUtil.joinLines( + "schema test {", + " document test { ", + " field int_attribute type int { ", + " indexing: attribute ", + " attribute: fast-access", + " }", + " field predicate_attribute type predicate {", + " indexing: attribute ", + " attribute: fast-access", + " }", + " field tensor_attribute type tensor(x[5]) {", + " indexing: attribute ", + " attribute: fast-access", + " }", + " field reference_attribute type reference {", + " indexing: attribute ", + " attribute: fast-access", + " }", + " }", + "}")); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema 'test': The following attributes have a type that is incompatible " + + "with fast-access: predicate_attribute, tensor_attribute, reference_attribute. " + + "Predicate, tensor and reference attributes are incompatible with fast-access."); + builder.build(true); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java new file mode 100644 index 00000000000..594124c9500 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { + + @Test + public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertEquals(2, docType.getFieldCount()); + } + + @Test + public void testRequireThatSummaryFieldsAreIncluded() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("cox")); + assertNotNull(docType.getField("mytags")); + assertNotNull(docType.getField("alltags")); + assertEquals(5, docType.getFieldCount()); + } + + @Test + public void testRequireThatBoldedSummaryFieldsAreIncluded() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("baz")); + assertNotNull(docType.getField("cox")); + assertEquals(4, docType.getFieldCount()); + } + + @Test + public void testRequireThatUntransformedSummaryFieldsAreIgnored() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertNotNull(docType.getField("baz")); + assertEquals(3, docType.getFieldCount()); + } + + @Test + public void testRequireThatDynamicSummaryFieldsAreIgnored() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertNotNull(docType.getField("foo")); + assertNotNull(docType.getField("bar")); + assertEquals(2, docType.getFieldCount()); + } + + @Test + public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { + ApplicationBuilder sb = new ApplicationBuilder(); + sb.addSchemaFile("src/test/examples/nextgen/simple.sd"); + sb.build(true); + assertNotNull(sb.getSchema()); + new DerivedConfiguration(sb.getSchema(), sb.getRankProfileRegistry()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java new file mode 100644 index 00000000000..111ed266d74 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.*; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; +public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { + @Test + public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + assertStruct(docType, PositionDataType.INSTANCE); + } + @Test + public void testRequireThatImplicitStructsAreUsed() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); + assertNotNull(schema); + + SDDocumentType docType = schema.getDocument(); + assertNotNull(docType); + + assertField(docType, "doc_str", DataType.STRING); + assertField(docType, "doc_str_sum", DataType.STRING); + assertField(docType, "doc_uri", DataType.URI); + assertField(docType, "docsum_str", DataType.STRING); + } + + @SuppressWarnings({ "UnusedDeclaration" }) + private static void assertStruct(SDDocumentType docType, StructDataType expectedStruct) { + // TODO: When structs are refactored from a static register to a member of the owning document types, this test + // TODO: must be changed to retrieve struct type from the provided document type. + StructDataType structType = (StructDataType) docType.getType(expectedStruct.getName()).getStruct(); + assertNotNull(structType); + for (Field expectedField : expectedStruct.getFields()) { + Field field = structType.getField(expectedField.getName()); + assertNotNull(field); + assertEquals(expectedField.getDataType(), field.getDataType()); + } + assertEquals(expectedStruct.getFieldCount(), structType.getFieldCount()); + } + + private static void assertField(SDDocumentType docType, String fieldName, DataType type) { + Field field = getSecretField(docType, fieldName); // TODO: get rid of this stupidity + assertNotNull(field); + assertEquals(type, field.getDataType()); + assertTrue(field instanceof SDField); + } + + private static Field getSecretField(SDDocumentType docType, String fieldName) { + for (Field field : docType.fieldSet()) { + if (field.getName().equals(fieldName)) { + return field; + } + } + return null; + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java new file mode 100644 index 00000000000..50deb5d5b42 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author Simon Thoresen Hult + */ +public class ImplicitSummariesTestCase { + + @Test + public void requireThatSummaryFromAttributeDoesNotWarn() throws IOException, ParseException { + LogHandler log = new LogHandler(); + Logger.getLogger("").addHandler(log); + + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); + assertNotNull(schema); + assertTrue(log.records.isEmpty()); + } + + private static class LogHandler extends Handler { + + final List records = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + if (record.getLevel() == Level.WARNING || + record.getLevel() == Level.SEVERE) + { + records.add(record); + } + } + + @Override + public void flush() { + + } + + @Override + public void close() throws SecurityException { + + } + } + + @Test + public void attribute_combiner_transform_is_set_on_array_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array").getTransform()); + } + + @Test + public void attribute_combiner_transform_is_set_on_map_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("str_elem_map").getTransform()); + } + + @Test + public void attribute_combiner_transform_is_not_set_when_map_of_struct_has_some_struct_field_attributes() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); + assertEquals(SummaryTransform.NONE, schema.getSummaryField("int_elem_map").getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java new file mode 100644 index 00000000000..f32c9079d36 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImplicitSummaryFieldsTestCase extends AbstractSchemaTestCase { + + @Test + public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); + assertNotNull(schema); + + DocumentSummary docsum = schema.getSummary("default"); + assertNotNull(docsum); + assertNotNull(docsum.getSummaryField("rankfeatures")); + assertNotNull(docsum.getSummaryField("summaryfeatures")); + assertEquals(2, docsum.getSummaryFields().size()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java new file mode 100644 index 00000000000..5baa64d06d4 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java @@ -0,0 +1,152 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableImportedSDField; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; +import com.yahoo.tensor.TensorType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; + +/** + * @author geirst + */ +public class ImportedFieldsResolverTestCase { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + private void resolve_imported_field(String fieldName, String targetFieldName) { + SearchModel model = new SearchModel(); + model.addImportedField(fieldName, "ref", targetFieldName).resolve(); + + assertEquals(1, model.importedFields.fields().size()); + ImportedField myField = model.importedFields.fields().get(fieldName); + assertNotNull(myField); + assertEquals(fieldName, myField.fieldName()); + assertSame(model.childSchema.getConcreteField("ref"), myField.reference().referenceField()); + assertSame(model.parentSchema, myField.reference().targetSearch()); + ImmutableSDField targetField = model.parentSchema.getField(targetFieldName); + if (targetField instanceof SDField) { + assertSame(targetField, myField.targetField()); + } else { + assertSame(getImportedField(targetField), getImportedField(myField.targetField())); + } + } + + private static ImportedField getImportedField(ImmutableSDField field) { + return ((ImmutableImportedSDField) field).getImportedField(); + } + + @Test + public void valid_imported_fields_are_resolved() { + resolve_imported_field("my_attribute_field", "attribute_field"); + resolve_imported_field("my_tensor_field", "tensor_field"); + resolve_imported_field("my_ancient_field", "ancient_field"); + } + + @Test + public void resolver_fails_if_document_reference_is_not_found() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_attribute_field': " + + "Reference field 'not_ref' not found"); + new SearchModel().addImportedField("my_attribute_field", "not_ref", "budget").resolve(); + } + + @Test + public void resolver_fails_if_referenced_field_is_not_found() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_attribute_field': " + + "Field 'not_existing' via reference field 'ref': Not found"); + new SearchModel().addImportedField("my_attribute_field", "ref", "not_existing").resolve(); + } + + @Test + public void resolver_fails_if_imported_field_is_not_an_attribute() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'child', import field 'my_not_attribute': " + + "Field 'not_attribute' via reference field 'ref': Is not an attribute field. Only attribute fields supported"); + new SearchModel().addImportedField("my_not_attribute", "ref", "not_attribute").resolve(); + } + + @Test + public void resolver_fails_if_imported_field_is_indexing() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema 'child', import field 'my_attribute_and_index': " + + "Field 'attribute_and_index' via reference field 'ref': Is an index field. Not supported"); + new SearchModel() + .addImportedField("my_attribute_and_index", "ref", "attribute_and_index") + .resolve(); + } + + @Test + public void resolver_fails_if_imported_field_is_of_type_predicate() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema 'child', import field 'my_predicate_field': " + + "Field 'predicate_field' via reference field 'ref': Is of type 'predicate'. Not supported"); + new SearchModel().addImportedField("my_predicate_field", "ref", "predicate_field").resolve(); + } + + static class SearchModel extends ParentChildSearchModel { + + public final Schema grandParentSchema; + public ImportedFields importedFields; + + public SearchModel() { + super(); + grandParentSchema = createSearch("grandparent"); + var grandParentDoc = grandParentSchema.getDocument(); + grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }")); + var parentDoc = parentSchema.getDocument(); + parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }")); + parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }")); + parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT)); + parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); + parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }")); + addRefField(parentSchema, grandParentSchema, "ref"); + addImportedField(parentSchema, "ancient_field", "ref", "ancient_field"); + + addRefField(childSchema, parentSchema, "ref"); + } + + + protected SearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) { + return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName); + } + + protected SearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) { + super.addImportedField(schema, fieldName, referenceFieldName, targetFieldName); + return this; + } + + public void resolve() { + resolve(grandParentSchema); + resolve(parentSchema); + importedFields = resolve(childSchema); + } + + private static ImportedFields resolve(Schema schema) { + assertNotNull(schema.temporaryImportedFields().get()); + assertFalse(schema.importedFields().isPresent()); + new ImportedFieldsResolver(schema, null, null, null).process(true, false); + assertNotNull(schema.importedFields().get()); + return schema.importedFields().get(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java new file mode 100644 index 00000000000..ab702154527 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java @@ -0,0 +1,529 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @author geirst + */ +public class ImportedFieldsTestCase { + + @Test + public void fields_can_be_imported_from_referenced_document_types() throws ParseException { + Schema schema = buildAdSearch(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference { indexing: attribute }", + " field person_ref type reference { indexing: attribute }", + " }", + " import field campaign_ref.budget as my_budget {}", + " import field person_ref.name as my_name {}", + "}")); + assertEquals(2, schema.importedFields().get().fields().size()); + assertSearchContainsImportedField("my_budget", "campaign_ref", "campaign", "budget", schema); + assertSearchContainsImportedField("my_name", "person_ref", "person", "name", schema); + } + + @SuppressWarnings("deprecation") + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void field_reference_spec_must_include_dot() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Illegal field reference spec 'campaignrefbudget': Does not include a single '.'"); + buildAdSearch(joinLines( + "search ad {", + " document ad {}", + " import field campaignrefbudget as budget {}", + "}")); + } + + @Test + public void fail_duplicate_import() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'ad', import field as 'my_budget': Field already imported"); + Schema schema = buildAdSearch(joinLines( + "schema ad {", + " document ad {", + " field campaign_ref type reference { indexing: attribute }", + " }", + " import field campaign_ref.budget as my_budget {}", + " import field campaign_ref.budget as my_budget {}", + "}")); + } + + private static Schema buildAdSearch(String sdContent) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "schema campaign {", + " document campaign {", + " field budget type int { indexing: attribute }", + " }", + "}")); + builder.addSchema(joinLines( + "schema person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + builder.addSchema(sdContent); + builder.build(true); + return builder.getSchema("ad"); + } + + private static void checkStructImport(AncestorStructSdBuilder parentBuilder) throws ParseException { + Schema schema = buildChildSearch(parentBuilder.build(), new ChildStructSdBuilder().build()); + checkImportedStructFields(schema, parentBuilder); + } + + private static void checkNestedStructImport(AncestorStructSdBuilder grandParentBuilder) throws ParseException { + Schema schema = buildChildSearch(grandParentBuilder.build(), + new IntermediateParentStructSdBuilder().build(), + new ChildStructSdBuilder().build()); + checkImportedStructFields(schema, grandParentBuilder); + } + + private static void checkImportedStructFields(Schema schema, AncestorStructSdBuilder ancestorBuilder) { + assertEquals(3, schema.importedFields().get().fields().size()); + checkImportedField("my_elem_array.name", "parent_ref", "parent", "elem_array.name", schema, ancestorBuilder.elem_array_name_attr); + checkImportedField("my_elem_array.weight", "parent_ref", "parent", "elem_array.weight", schema, ancestorBuilder.elem_array_weight_attr); + checkImportedField("my_elem_map.key", "parent_ref", "parent", "elem_map.key", schema, ancestorBuilder.elem_map_key_attr); + checkImportedField("my_elem_map.value.name", "parent_ref", "parent", "elem_map.value.name", schema, ancestorBuilder.elem_map_value_name_attr); + checkImportedField("my_elem_map.value.weight", "parent_ref", "parent", "elem_map.value.weight", schema, ancestorBuilder.elem_map_value_weight_attr); + checkImportedField("my_str_int_map.key", "parent_ref", "parent", "str_int_map.key", schema, ancestorBuilder.str_int_map_key_attr); + checkImportedField("my_str_int_map.value", "parent_ref", "parent", "str_int_map.value", schema, ancestorBuilder.str_int_map_value_attr); + checkImportedField("my_elem_array", "parent_ref", "parent", "elem_array", schema, true); + checkImportedField("my_elem_map", "parent_ref", "parent", "elem_map", schema, true); + checkImportedField("my_str_int_map", "parent_ref", "parent", "str_int_map", schema, true); + } + + @Test + public void check_struct_import() throws ParseException { + checkStructImport(new ParentStructSdBuilder()); + checkStructImport(new ParentStructSdBuilder().elem_array_weight_attr(false).elem_map_value_weight_attr(false)); + checkStructImport(new ParentStructSdBuilder().elem_array_name_attr(false).elem_map_value_name_attr(false)); + } + + @Test + public void check_nested_struct_import() throws ParseException { + checkNestedStructImport(new GrandParentStructSdBuilder()); + checkNestedStructImport(new GrandParentStructSdBuilder().elem_array_weight_attr(false).elem_map_value_weight_attr(false)); + checkNestedStructImport(new GrandParentStructSdBuilder().elem_array_name_attr(false).elem_map_value_name_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_array_of_struct_attributes() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_elem_array': Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field."); + checkStructImport(new ParentStructSdBuilder().elem_array_name_attr(false).elem_array_weight_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_struct_key_attribute() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_elem_map' (nested to 'my_elem_map.key'): Field 'elem_map.key' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); + checkStructImport(new ParentStructSdBuilder().elem_map_key_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_struct_value_attributes() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_elem_map' (nested to 'my_elem_map.value'): Field 'elem_map.value' via reference field 'parent_ref': Is not a struct containing an attribute field."); + checkStructImport(new ParentStructSdBuilder().elem_map_value_name_attr(false).elem_map_value_weight_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_primitive_key_attribute() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_str_int_map' (nested to 'my_str_int_map.key'): Field 'str_int_map.key' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); + checkStructImport(new ParentStructSdBuilder().str_int_map_key_attr(false)); + } + + @Test + public void check_illegal_struct_import_missing_map_of_primitive_value_attribute() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_str_int_map' (nested to 'my_str_int_map.value'): Field 'str_int_map.value' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); + checkStructImport(new ParentStructSdBuilder().str_int_map_value_attr(false)); + } + + private static class NamedSdBuilder { + protected String name; + private String fieldPrefix; + + public NamedSdBuilder(String name, String fieldPrefix) { + this.name = name; + this.fieldPrefix = fieldPrefix; + } + + protected String prefixedFieldName(String name) { + return fieldPrefix + name; + } + } + + private static class AncestorStructSdBuilder extends NamedSdBuilder { + private boolean elem_array_name_attr; + private boolean elem_array_weight_attr; + private boolean elem_map_key_attr; + private boolean elem_map_value_name_attr; + private boolean elem_map_value_weight_attr; + private boolean str_int_map_key_attr; + private boolean str_int_map_value_attr; + + public AncestorStructSdBuilder(String name, String fieldPrefix) { + super(name, fieldPrefix); + elem_array_name_attr = true; + elem_array_weight_attr = true; + elem_map_key_attr = true; + elem_map_value_name_attr = true; + elem_map_value_weight_attr = true; + str_int_map_key_attr = true; + str_int_map_value_attr = true; + } + + public AncestorStructSdBuilder elem_array_name_attr(boolean v) { elem_array_name_attr = v; return this; } + public AncestorStructSdBuilder elem_array_weight_attr(boolean v) { elem_array_weight_attr = v; return this; } + public AncestorStructSdBuilder elem_map_key_attr(boolean v) { elem_map_key_attr = v; return this; } + public AncestorStructSdBuilder elem_map_value_name_attr(boolean v) { elem_map_value_name_attr = v; return this; } + public AncestorStructSdBuilder elem_map_value_weight_attr(boolean v) { elem_map_value_weight_attr = v; return this; } + public AncestorStructSdBuilder str_int_map_key_attr(boolean v) { str_int_map_key_attr = v; return this; } + public AncestorStructSdBuilder str_int_map_value_attr(boolean v) { str_int_map_value_attr = v; return this; } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + " field " + prefixedFieldName("elem_array") + " type array {", + " indexing: summary", + " struct-field name {", + structFieldSpec(elem_array_name_attr), + " }", + " struct-field weight {", + structFieldSpec(elem_array_weight_attr), + " }", + " }", + " field " + prefixedFieldName("elem_map") + " type map {", + " indexing: summary", + " struct-field key {", + structFieldSpec(elem_map_key_attr), + " }", + " struct-field value.name {", + structFieldSpec(elem_map_value_name_attr), + " }", + " struct-field value.weight {", + structFieldSpec(elem_map_value_weight_attr), + " }", + " }", + " field " + prefixedFieldName("str_int_map") + " type map {", + " indexing: summary", + " struct-field key {", + structFieldSpec(str_int_map_key_attr), + " }", + " struct-field value {", + structFieldSpec(str_int_map_value_attr), + " }", + " }", + " }", + "}"); + } + + private static String structFieldSpec(boolean isAttribute) { + return isAttribute ? " indexing: attribute" : ""; + } + } + + private static class ParentStructSdBuilder extends AncestorStructSdBuilder { + ParentStructSdBuilder() { + super("parent", ""); + } + } + + private static class GrandParentStructSdBuilder extends AncestorStructSdBuilder { + GrandParentStructSdBuilder() { + super("grandparent", "gp_"); + } + } + + private static class DescendantSdBuilder extends NamedSdBuilder { + protected String parentName; + private String parentFieldPrefix; + + public DescendantSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { + super(name, fieldPrefix); + this.parentName = parentName; + this.parentFieldPrefix = parentFieldPrefix; + } + + protected String parentRef() { + return parentName + "_ref"; + } + + protected String importParentField(String fieldName) { + return " import field " + parentRef() + "." + parentFieldPrefix + fieldName + " as " + prefixedFieldName(fieldName) + " {}"; + } + } + + private static class DescendantStructSdBuilder extends DescendantSdBuilder { + public DescendantStructSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { + super(name, fieldPrefix, parentName, parentFieldPrefix); + } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + " field " + parentRef() + " type reference<" + parentName + "> {", + " indexing: attribute | summary", + " }", + " }", + importParentField("elem_array"), + importParentField("elem_map"), + importParentField("str_int_map"), + "}"); + } + } + + private static class ChildStructSdBuilder extends DescendantStructSdBuilder { + public ChildStructSdBuilder() { + super("child", "my_", "parent", ""); + } + } + + private static class IntermediateParentStructSdBuilder extends DescendantStructSdBuilder { + public IntermediateParentStructSdBuilder() { + super("parent", "", "grandparent", "gp_"); + } + } + + private static Schema buildChildSearch(String parentSdContent, String sdContent) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); + return builder.getSchema("child"); + } + + private static Schema buildChildSearch(String grandParentSdContent, String parentSdContent, String sdContent) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(grandParentSdContent); + builder.addSchema(parentSdContent); + builder.addSchema(sdContent); + builder.build(true); + return builder.getSchema("child"); + } + + private static class AncestorPosSdBuilder extends NamedSdBuilder { + public AncestorPosSdBuilder(String name, String fieldPrefix) { + super(name, fieldPrefix); + } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + "field " + prefixedFieldName("pos") + " type position {", + "indexing: attribute | summary", + " }", + " }", + "}"); + } + } + + private static class ParentPosSdBuilder extends AncestorPosSdBuilder { + public ParentPosSdBuilder() { super("parent", ""); } + } + + private static class GrandParentPosSdBuilder extends AncestorPosSdBuilder { + public GrandParentPosSdBuilder() { super("grandparent", "gp_"); } + } + + private static class DescendantPosSdBuilder extends DescendantSdBuilder { + private boolean import_pos_zcurve_before; + + public DescendantPosSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { + super(name, fieldPrefix, parentName, parentFieldPrefix); + import_pos_zcurve_before = false; + } + + DescendantPosSdBuilder import_pos_zcurve_before(boolean v) { import_pos_zcurve_before = v; return this; } + + public String build() { + return joinLines("search " + name + " {", + " document " + name + " {", + " field " + parentRef() + " type reference<" + parentName + "> {", + " indexing: attribute | summary", + " }", + " }", + importPosZCurve(import_pos_zcurve_before), + importParentField("pos"), + "}"); + } + + private static String importPosZCurve(boolean doImport) { + return doImport ? "import field parent_ref.pos_zcurve as my_pos_zcurve {}" : ""; + } + } + + private static class ChildPosSdBuilder extends DescendantPosSdBuilder { + public ChildPosSdBuilder() { + super("child", "my_", "parent", ""); + } + } + + private static class IntermediateParentPosSdBuilder extends DescendantPosSdBuilder { + public IntermediateParentPosSdBuilder() { + super("parent", "", "grandparent", "gp_"); + } + } + + private static void checkPosImport(ParentPosSdBuilder parentBuilder, DescendantPosSdBuilder childBuilder) throws ParseException { + Schema schema = buildChildSearch(parentBuilder.build(), childBuilder.build()); + checkImportedPosFields(schema); + } + + private static void checkNestedPosImport(GrandParentPosSdBuilder grandParentBuilder, DescendantPosSdBuilder childBuilder) throws ParseException { + Schema schema = buildChildSearch(grandParentBuilder.build(), new IntermediateParentPosSdBuilder().build(), childBuilder.build()); + checkImportedPosFields(schema); + } + + private static void checkImportedPosFields(Schema schema) { + assertEquals(2, schema.importedFields().get().fields().size()); + assertSearchContainsImportedField("my_pos_zcurve", "parent_ref", "parent", "pos_zcurve", schema); + assertSearchContainsImportedField("my_pos", "parent_ref", "parent", "pos", schema); + } + + @Test + public void check_pos_import() throws ParseException { + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder()); + } + + @Test + public void check_nested_pos_import() throws ParseException { + checkNestedPosImport(new GrandParentPosSdBuilder(), new ChildPosSdBuilder()); + } + + @Test + public void check_pos_import_after_pos_zcurve_import() throws ParseException { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("For schema 'child', import field 'my_pos_zcurve': Field 'pos_zcurve' via reference field 'parent_ref': Field already imported"); + checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_before(true)); + } + + private static ImportedField getImportedField(String name, Schema schema) { + if (name.contains(".")) { + assertNull(schema.importedFields().get().fields().get(name)); + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + ImportedField superField = schema.importedFields().get().fields().get(superFieldName); + if (superField != null && superField instanceof ImportedComplexField) { + return ((ImportedComplexField)superField).getNestedField(subFieldName); + } + return null; + } + return schema.importedFields().get().fields().get(name); + } + + private static void assertSearchNotContainsImportedField(String fieldName, Schema schema) { + ImportedField importedField = getImportedField(fieldName, schema); + assertNull(importedField); + } + + private static void assertSearchContainsImportedField(String fieldName, + String referenceFieldName, + String referenceDocType, + String targetFieldName, + Schema schema) { + ImportedField importedField = getImportedField(fieldName, schema); + assertNotNull(importedField); + assertEquals(fieldName, importedField.fieldName()); + assertEquals(referenceFieldName, importedField.reference().referenceField().getName()); + assertEquals(referenceDocType, importedField.reference().targetSearch().getName()); + assertEquals(targetFieldName, importedField.targetField().getName()); + } + + private static void checkImportedField(String fieldName, String referenceFieldName, String referenceDocType, + String targetFieldName, Schema schema, boolean present) { + if (present) { + assertSearchContainsImportedField(fieldName, referenceFieldName, referenceDocType, targetFieldName, schema); + } else { + assertSearchNotContainsImportedField(fieldName, schema); + } + } + + @Test + public void field_with_struct_field_attributes_can_be_imported_from_parents_that_use_inheritance() throws ParseException { + var builder = buildParentsUsingInheritance(); + + assertParentContainsEntriesAttributes(builder.getSchema("parent_a")); + assertParentContainsEntriesAttributes(builder.getSchema("parent_b")); + + var child = builder.getSchema("child"); + checkImportedField("entries_from_a", "ref_parent_a", "parent_a", "entries", child, true); + checkImportedField("entries_from_a.key", "ref_parent_a", "parent_a", "entries.key", child, true); + checkImportedField("entries_from_a.value", "ref_parent_a", "parent_a", "entries.value", child, true); + + checkImportedField("entries_from_b", "ref_parent_b", "parent_b", "entries", child, true); + checkImportedField("entries_from_b.key", "ref_parent_b", "parent_b", "entries.key", child, true); + checkImportedField("entries_from_b.value", "ref_parent_b", "parent_b", "entries.value", child, true); + } + + private void assertParentContainsEntriesAttributes(Schema parent) { + var attrs = new AttributeFields(parent); + assertTrue(attrs.containsAttribute("entries.key")); + assertTrue(attrs.containsAttribute("entries.value")); + } + + private ApplicationBuilder buildParentsUsingInheritance() throws ParseException { + var builder = new ApplicationBuilder(); + builder.addSchema(joinLines("schema parent_a {", + "document parent_a {", + " struct Entry {", + " field key type string {}", + " field value type string {}", + " }", + " field entries type array {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + " }", + "}", + "}")); + + builder.addSchema(joinLines("schema parent_b {", + "document parent_b inherits parent_a {", + "}", + "}")); + + builder.addSchema(joinLines("schema child {", + "document child {", + " field ref_parent_a type reference {", + " indexing: attribute", + " }", + " field ref_parent_b type reference {", + " indexing: attribute", + " }", + "}", + "import field ref_parent_a.entries as entries_from_a {}", + "import field ref_parent_b.entries as entries_from_b {}", + "}")); + + builder.build(true); + return builder; + } + + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java new file mode 100644 index 00000000000..71c79feedc1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingInputsTestCase { + + @Test + public void requireThatExtraFieldInputExtraFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", + "For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + + "to field 'bar' which does not exist in document type " + + "'indexing_extra_field_input_extra_field', and is not a mutable attribute."); + } + + @Test + public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", + "For schema 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + + "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit', and is not a mutable attribute."); + } + + @Test + public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", + "For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + + "'foo' which does not exist in document type 'indexing_extra_field_input_null', and is not a mutable attribute."); + } + + @Test + public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", + "For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + + "'foo' which does not exist in document type 'indexing_extra_field_input_self', and is not a mutable attribute."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java new file mode 100644 index 00000000000..687549f920e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + + +/** + * @author Simon Thoresen Hult + */ +public class IndexingOutputsTestCase { + + @Test + public void requireThatOutputOtherFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_output_other_field.sd", + "For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void requireThatOutputConflictThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_output_conflict.sd", + "For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + + "to assign conflicting values to field 'bar'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java new file mode 100644 index 00000000000..76cb6a5505c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -0,0 +1,200 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.schema.Index; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.util.Arrays; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; +import java.util.Set; + +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; +import static org.junit.Assert.assertEquals; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { + + @Test + public void testSetLanguageRewriting() { + assertIndexingScript("{ input test | set_language; }", + createField("test", DataType.STRING, "{ set_language }")); + } + + @Test + public void testSummaryRewriting() { + assertIndexingScript("{ input test | summary test; }", + createField("test", DataType.STRING, "{ summary }")); + } + + @Test + public void testDynamicSummaryRewriting() { + SDField field = createField("test", DataType.STRING, "{ summary }"); + field.addSummaryField(createDynamicSummaryField(field, "dyn")); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn; }", field); + } + + @Test + public void testSummaryRewritingWithIndexing() { + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", + createField("test", DataType.STRING, "{ summary | index }")); + } + + @Test + public void testDynamicAndStaticSummariesRewritingWithIndexing() { + SDField field = createField("test", DataType.STRING, "{ summary | index }"); + field.addSummaryField(createDynamicSummaryField(field, "dyn")); + field.addSummaryField(createStaticSummaryField(field, "test")); + field.addSummaryField(createStaticSummaryField(field, "other")); + field.addSummaryField(createDynamicSummaryField(field, "dyn2")); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn | summary dyn2 | summary other | " + + "summary test | index test; }", field); + } + + @Test + public void testIntSummaryRewriting() { + assertIndexingScript("{ input test | summary test | attribute test; }", + createField("test", DataType.INT, "{ summary | index }")); + } + + @Test + public void testStringAttributeSummaryRewriting() { + assertIndexingScript("{ input test | summary test | attribute test; }", + createField("test", DataType.STRING, "{ summary | attribute }")); + } + + @Test + public void testMultiblockTokenize() { + SDField field = createField("test", DataType.STRING, + "{ input test | tokenize | { summary test; }; }"); + assertIndexingScript("{ input test | tokenize | { summary test; }; }", field); + } + + @Test + public void requireThatOutputDefaultsToCurrentField() { + assertIndexingScript("{ input test | attribute test; }", + createField("test", DataType.STRING, "{ attribute; }")); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | index test; }", + createField("test", DataType.STRING, "{ index; }")); + assertIndexingScript("{ input test | summary test; }", + createField("test", DataType.STRING, "{ summary; }")); + } + + @Test + public void testTokenizeComparisonDisregardsConfig() { + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", + createField("test", DataType.STRING, "{ summary | tokenize | index; }")); + } + + @Test + public void testDerivingFromSimple() throws Exception { + assertIndexing(Arrays.asList("clear_state | guard { input access | attribute access; }", + "clear_state | guard { input category | split \";\" | attribute category_arr; }", + "clear_state | guard { input category | tokenize | index category; }", + "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", + "clear_state | guard { input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", + "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }", + "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | summary dyndesc | index description; }", + "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", + "clear_state | guard { input longdesc | tokenize normalize stem:\"BEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }", + "clear_state | guard { input measurement | attribute measurement | summary measurement; }", + "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", + "clear_state | guard { input popularity | attribute popularity; }", + "clear_state | guard { input popularity * input measurement | attribute popsiness; }", + "clear_state | guard { input smallattribute | attribute smallattribute; }", + "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", + "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), + ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); + } + + @Test + public void testIndexRewrite() throws Exception { + assertIndexing( + Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + + " tokenize | index title; }", + "clear_state | guard { input title_src | summary title_s; }"), + ApplicationBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); + } + + @Test + public void requireThatPredicateFieldsGetOptimization() { + assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | attribute test; }; }", + createPredicateField( + "test", DataType.PREDICATE, "{ attribute; }", 10, OptionalLong.empty(), OptionalLong.empty())); + assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | summary test | attribute test; }; }", + createPredicateField( + "test", DataType.PREDICATE, "{ summary | attribute ; }", 10, OptionalLong.empty(), OptionalLong.empty())); + assertIndexingScript( + "{ 2 | set_var arity | 0L | set_var lower_bound | 1023L | set_var upper_bound | " + + "{ input test | optimize_predicate | attribute test; }; }", + createPredicateField("test", DataType.PREDICATE, "{ attribute; }", 2, OptionalLong.of(0L), OptionalLong.of(1023L))); + } + + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { + assertEquals(expectedScript, + processField(unprocessedField).toString()); + } + + private static ScriptExpression processField(SDField unprocessedField) { + SDDocumentType sdoc = new SDDocumentType("test"); + sdoc.addField(unprocessedField); + Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); + schema.addDocument(sdoc); + new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), + new QueryProfiles(), true, false, Set.of()); + return unprocessedField.getIndexingScript(); + } + + private static SDField createField(String name, DataType type, String script) { + SDField field = new SDField(null, name, type); + field.parseIndexingScript(script); + return field; + } + + private static SDField createPredicateField( + String name, DataType type, String script, int arity, OptionalLong lower_bound, OptionalLong upper_bound) { + SDField field = new SDField(null, name, type); + field.parseIndexingScript(script); + Index index = new Index("foo"); + index.setBooleanIndexDefiniton(new BooleanIndexDefinition( + OptionalInt.of(arity), lower_bound, upper_bound, OptionalDouble.empty())); + field.addIndex(index); + return field; + } + + private static SummaryField createDynamicSummaryField(SDField field, String name) { + return createSummaryField(field, name, true); + } + + private static SummaryField createStaticSummaryField(SDField field, String name) { + return createSummaryField(field, name, false); + } + + private static SummaryField createSummaryField(SDField field, String name, boolean dynamic) { + SummaryField summaryField = new SummaryField(name, field.getDataType()); + if (dynamic) { + summaryField.setTransform(SummaryTransform.DYNAMICTEASER); + } + summaryField.addDestination("default"); + summaryField.addSource(field.getName()); + return summaryField; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java new file mode 100644 index 00000000000..4da6880aa26 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AbstractExportingTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingValidationTestCase extends AbstractExportingTestCase { + + @Test + public void testAttributeChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_attribute_changed.sd", + "For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + + "Attempting to assign conflicting values to field 'foo'."); + } + + @Test + public void testAttributeOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_attribute_other.sd", + "For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testIndexChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_index_changed.sd", + "For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + + "Attempting to assign conflicting values to field 'foo'."); + } + + @Test + public void testIndexOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_index_other.sd", + "For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testSummaryChanged() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_summary_changed.sd", + "For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + + "to assign conflicting values to field 'foo'."); + } + + @Test + public void testSummaryOther() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_summary_other.sd", + "For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + + "attempts to write to a field other than 'foo'."); + } + + @Test + public void testExtraField() throws IOException, ParseException { + assertIndexing( + Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", + "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), + ApplicationBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); + } + + @Test + public void requireThatMultilineOutputConflictThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_multiline_output_conflict.sd", + "For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + + "Attempting to assign conflicting values to field 'cox'."); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java new file mode 100644 index 00000000000..2784fe69b28 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; + +/** + * @author Simon Thoresen Hult + */ +public class IndexingValuesTestCase { + + @Test + public void requireThatModifyFieldNoOutputDoesNotThrow() throws IOException, ParseException { + assertBuilds("src/test/examples/indexing_modify_field_no_output.sd"); + } + + @Test + public void requireThatInputOtherFieldThrows() throws IOException, ParseException { + assertBuildFails("src/test/examples/indexing_input_other_field.sd", + "For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + + "attempts to modify the value of the document field 'bar'. " + + "Use a field outside the document block instead."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java new file mode 100644 index 00000000000..f36effab146 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author baldersheim + */ +public class IntegerIndex2AttributeTestCase extends AbstractSchemaTestCase { + + @Test + public void testIntegerIndex2Attribute() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); + new IntegerIndex2Attribute(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + + SDField f; + f = schema.getConcreteField("s1"); + assertTrue(f.getAttributes().isEmpty()); + assertTrue(f.existsIndex("s1")); + f = schema.getConcreteField("s2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue(f.existsIndex("s2")); + + f = schema.getConcreteField("as1"); + assertTrue(f.getAttributes().isEmpty()); + assertTrue(f.existsIndex("as1")); + f = schema.getConcreteField("as2"); + assertEquals(f.getAttributes().size(), 1); + assertTrue(f.existsIndex("as2")); + + f = schema.getConcreteField("i1"); + assertEquals(f.getAttributes().size(), 1); + assertFalse(f.existsIndex("i1")); + + f = schema.getConcreteField("i2"); + assertEquals(f.getAttributes().size(), 1); + assertFalse(f.existsIndex("i2")); + + f = schema.getConcreteField("ai1"); + assertEquals(schema.getConcreteField("ai1").getAttributes().size(), 1); + assertFalse(schema.getConcreteField("ai1").existsIndex("ai1")); + f = schema.getConcreteField("ai2"); + assertEquals(f.getAttributes().size(), 1); + assertFalse(f.existsIndex("ai2")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java new file mode 100644 index 00000000000..530b6a95ce8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import org.junit.Test; + +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; + +public class MatchPhaseSettingsValidatorTestCase { + + private static String getMessagePrefix() { + return "In search definition 'test', rank-profile 'default': match-phase attribute 'foo' "; + } + + @Test + public void requireThatAttributeMustExists() throws Exception { + assertBuildFails("src/test/examples/matchphase/non_existing_attribute.sd", + getMessagePrefix() + "does not exists"); + } + + @Test + public void requireThatAttributeMustBeNumeric() throws Exception { + assertBuildFails("src/test/examples/matchphase/wrong_data_type_attribute.sd", + getMessagePrefix() + "must be single value numeric, but it is 'string'"); + } + + @Test + public void requireThatAttributeMustBeSingleValue() throws Exception { + assertBuildFails("src/test/examples/matchphase/wrong_collection_type_attribute.sd", + getMessagePrefix() + "must be single value numeric, but it is 'Array'"); + } + + @Test + public void requireThatAttributeMustHaveFastSearch() throws Exception { + assertBuildFails("src/test/examples/matchphase/non_fast_search_attribute.sd", + getMessagePrefix() + "must be fast-search, but it is not"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java new file mode 100644 index 00000000000..c401376ac3a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java @@ -0,0 +1,192 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class MatchedElementsOnlyResolverTestCase { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void complex_field_with_some_struct_field_attributes_gets_default_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type map {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type map {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type array {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field name { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); + } + + @Test + public void complex_field_with_only_struct_field_attributes_gets_attribute_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type map {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type map {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field key { indexing: attribute }", + " struct-field value.name { indexing: attribute }", + " struct-field value.weight { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + + assertSummaryField(joinLines("field my_field type array {", + " indexing: summary", + " summary: matched-elements-only", + " struct-field name { indexing: attribute }", + " struct-field weight { indexing: attribute }", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + + @Test + public void explicit_complex_summary_field_can_use_filter_transform_with_reference_to_source_field() throws ParseException { + String documentSummary = joinLines("document-summary my_summary {", + " summary my_filter_field type map {", + " source: my_field", + " matched-elements-only", + " }", + "}"); + { + var search = buildSearch(joinLines("field my_field type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + "}"), + documentSummary); + assertSummaryField(search.getSummaryField("my_filter_field"), + SummaryTransform.MATCHED_ELEMENTS_FILTER, "my_field"); + assertSummaryField(search.getSummaryField("my_field"), + SummaryTransform.NONE, "my_field"); + } + { + var search = buildSearch(joinLines("field my_field type map {", + " indexing: summary", + " struct-field key { indexing: attribute }", + " struct-field value { indexing: attribute }", + "}"), + documentSummary); + assertSummaryField(search.getSummaryField("my_filter_field"), + SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field"); + assertSummaryField(search.getSummaryField("my_field"), + SummaryTransform.ATTRIBUTECOMBINER, "my_field"); + } + } + + @Test + public void primitive_array_attribute_field_gets_attribute_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type array {", + " indexing: attribute | summary", + " summary: matched-elements-only", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + + @Test + public void primitive_weighted_set_attribute_field_gets_attribute_transform() throws ParseException { + assertSummaryField(joinLines("field my_field type weightedset {", + " indexing: attribute | summary", + " summary: matched-elements-only", + "}"), + "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } + + @Test + public void explicit_summary_field_can_use_filter_transform_with_reference_to_attribute_source_field() throws ParseException { + String documentSummary = joinLines("document-summary my_summary {", + " summary my_filter_field type array {", + " source: my_field", + " matched-elements-only", + " }", + "}"); + + var search = buildSearch(joinLines( + "field my_field type array {", + " indexing: attribute | summary", + "}"), + documentSummary); + assertSummaryField(search.getSummaryField("my_filter_field"), + SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field"); + assertSummaryField(search.getSummaryField("my_field"), + SummaryTransform.ATTRIBUTE, "my_field"); + } + + @Test + public void unsupported_field_type_throws() throws ParseException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For schema 'test', document summary 'default', summary field 'my_field': " + + "'matched-elements-only' is not supported for this field type. " + + "Supported field types are: array of primitive, weighted set of primitive, " + + "array of simple struct, map of primitive type to simple struct, " + + "and map of primitive type to primitive type"); + buildSearch(joinLines("field my_field type string {", + " indexing: summary", + " summary: matched-elements-only", + "}")); + } + + private void assertSummaryField(String fieldContent, String fieldName, SummaryTransform expTransform) throws ParseException { + var search = buildSearch(fieldContent); + assertSummaryField(search.getSummaryField(fieldName), expTransform, fieldName); + } + + private void assertSummaryField(SummaryField field, SummaryTransform expTransform, String expSourceField) { + assertEquals(expTransform, field.getTransform()); + assertEquals(expSourceField, field.getSingleSource()); + } + + private Schema buildSearch(String field) throws ParseException { + return buildSearch(field, ""); + } + + private Schema buildSearch(String field, String summary) throws ParseException { + var builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(joinLines("search test {", + " document test {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + field, + " }", + summary, + "}")); + builder.build(true); + return builder.getSchema(); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java new file mode 100644 index 00000000000..912e6fcf030 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class NGramTestCase extends AbstractSchemaTestCase { + + @Test + public void testNGram() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/ngram.sd"); + assertNotNull(schema); + + SDField gram1 = schema.getConcreteField("gram_1"); + assertEquals(MatchType.GRAM, gram1.getMatching().getType()); + assertEquals(1, gram1.getMatching().getGramSize()); + + SDField gram2 = schema.getConcreteField("gram_2"); + assertEquals(MatchType.GRAM, gram2.getMatching().getType()); + assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly + + SDField gram3= schema.getConcreteField("gram_3"); + assertEquals(MatchType.GRAM,gram3.getMatching().getType()); + assertEquals(3, gram3.getMatching().getGramSize()); + + assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); + assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); + assertEquals("input gram_3 | ngram 3 | index gram_3", gram3.getIndexingScript().iterator().next().toString()); + + assertFalse(gram1.getNormalizing().doRemoveAccents()); + assertEquals(Stemming.NONE, gram1.getStemming()); + + List queryCommands = gram1.getQueryCommands(); + assertEquals(2, queryCommands.size()); + assertEquals("ngram 1", queryCommands.get(1)); + } + + @Test + public void testInvalidNGramSetting1() throws IOException, ParseException { + try { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram-size can only be set when the matching mode is 'gram'", e.getMessage()); + } + } + + @Test + public void testInvalidNGramSetting2() throws IOException, ParseException { + try { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram-size can only be set when the matching mode is 'gram'", e.getMessage()); + } + } + + @Test + public void testInvalidNGramSetting3() throws IOException, ParseException { + try { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); + fail("Should cause an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("gram matching is not supported with attributes, use 'index' in indexing", e.getMessage()); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java new file mode 100644 index 00000000000..a291dda24b9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -0,0 +1,119 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.util.Optional; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.schema.ApplicationBuilder.createFromStrings; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PagedAttributeValidatorTestCase { + + @Test + public void dense_tensor_attribute_supports_paged_setting() throws ParseException { + assertPagedSupported("tensor(x[2],y[2])"); + } + + @Test + public void primitive_attribute_types_support_paged_setting() throws ParseException { + assertPagedSupported("int"); + assertPagedSupported("array"); + assertPagedSupported("weightedset"); + + assertPagedSupported("string"); + assertPagedSupported("array"); + assertPagedSupported("weightedset"); + } + + @Test + public void struct_field_attributes_support_paged_setting() throws ParseException { + var sd = joinLines("schema test {", + " document test {", + " struct elem {", + " field first type int {}", + " field second type string {}", + " }", + " field foo type array {", + " indexing: summary", + " struct-field first {", + " indexing: attribute", + " attribute: paged", + " }", + " struct-field second {", + " indexing: attribute", + " attribute: paged", + " }", + " }", + " }", + "}"); + + var appBuilder = createFromString(sd); + var field = appBuilder.getSchema().getField("foo"); + assertTrue(field.getStructField("first").getAttribute().isPaged()); + assertTrue(field.getStructField("second").getAttribute().isPaged()); + } + + private void assertPagedSupported(String fieldType) throws ParseException { + var appBuilder = createFromString(getSd(fieldType)); + var attribute = appBuilder.getSchema().getAttribute("foo"); + assertTrue(attribute.isPaged()); + } + + @Test + public void non_dense_tensor_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("tensor(x{},y[2])"); + } + + @Test + public void predicate_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("predicate"); + } + + @Test + public void reference_attribute_does_not_support_paged_setting() throws ParseException { + assertPagedSettingNotSupported("reference", Optional.of(getSd("parent", "int"))); + } + + private void assertPagedSettingNotSupported(String fieldType) throws ParseException { + assertPagedSettingNotSupported(fieldType, Optional.empty()); + } + + private void assertPagedSettingNotSupported(String fieldType, Optional parentSd) throws ParseException { + try { + if (parentSd.isPresent()) { + createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType)); + } else { + createFromString(getSd(fieldType)); + } + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types", + e.getMessage()); + } + } + + private String getSd(String fieldType) { + return getSd("test", fieldType); + } + + private String getSd(String docType, String fieldType) { + return joinLines( + "schema " + docType + " {", + " document " + docType + " {", + " field foo type " + fieldType + "{", + " indexing: attribute", + " attribute: paged", + " }", + " }", + "}"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java new file mode 100644 index 00000000000..e5636da57a0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; +import com.yahoo.schema.document.TemporarySDField; + +/* + * Fixture class used for ImportedFieldsResolverTestCase and AdjustPositionSummaryFieldsTestCase. + */ +public class ParentChildSearchModel { + + public Schema parentSchema; + public Schema childSchema; + + ParentChildSearchModel() { + parentSchema = createSearch("parent"); + childSchema = createSearch("child"); + } + + protected Schema createSearch(String name) { + Schema result = new Schema(name, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); + result.addDocument(new SDDocumentType(name)); + return result; + } + + protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + TemporarySDField result = new TemporarySDField(repo, name, dataType); + result.parseIndexingScript(indexingScript); + return result; + } + + @SuppressWarnings("deprecation") + protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) { + return new TemporarySDField(repo, fieldName, NewDocumentReferenceDataType.forDocumentName(parentType)); + } + + protected static void addRefField(Schema child, Schema parent, String fieldName) { + SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName); + child.getDocument().addField(refField); + child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(), + new DocumentReference(refField, parent)))); + } + + protected ParentChildSearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) { + return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName); + } + + protected ParentChildSearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) { + schema.temporaryImportedFields().get().add(new TemporaryImportedField(fieldName, referenceFieldName, targetFieldName)); + return this; + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java new file mode 100644 index 00000000000..6f0facf9541 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java @@ -0,0 +1,130 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.PositionDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Test Position processor. + * + * @author hmusum + */ +public class PositionTestCase { + + @Test + public void inherited_position_zcurve_field_is_not_added_to_document_fieldset() throws Exception { + ApplicationBuilder sb = ApplicationBuilder.createFromFiles(Arrays.asList( + "src/test/examples/position_base.sd", + "src/test/examples/position_inherited.sd")); + + Schema schema = sb.getSchema("position_inherited"); + FieldSet fieldSet = schema.getDocument().getFieldSets().builtInFieldSets().get(DocumentType.DOCUMENT); + assertFalse(fieldSet.getFieldNames().contains(PositionDataType.getZCurveFieldName("pos"))); + } + + @Test + public void requireThatPositionCanBeAttribute() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_attribute.sd"); + assertNull(schema.getAttribute("pos")); + assertNull(schema.getAttribute("pos.x")); + assertNull(schema.getAttribute("pos.y")); + + assertPositionAttribute(schema, "pos", Attribute.CollectionType.SINGLE); + assertPositionSummary(schema, "pos", false); + } + + @Test + public void requireThatPositionCanNotBeIndex() throws Exception { + try { + ApplicationBuilder.buildFromFile("src/test/examples/position_index.sd"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'position_index', field 'pos': Indexing of data type 'position' is not " + + "supported, replace 'index' statement with 'attribute'.", e.getMessage()); + } + } + + @Test + public void requireThatSummaryAloneDoesNotCreateZCurve() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_summary.sd"); + assertNull(schema.getAttribute("pos")); + assertNull(schema.getAttribute("pos.x")); + assertNull(schema.getAttribute("pos.y")); + assertNull(schema.getAttribute("pos.zcurve")); + + SummaryField summary = schema.getSummaryField("pos"); + assertNotNull(summary); + assertEquals(2, summary.getSourceCount()); + Iterator it = summary.getSources().iterator(); + assertEquals("pos.x", it.next().getName()); + assertEquals("pos.y", it.next().getName()); + assertEquals(SummaryTransform.NONE, summary.getTransform()); + + assertNull(schema.getSummaryField("pos_ext.distance")); + } + + @Test + public void requireThatExtraFieldCanBePositionAttribute() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_extra.sd"); + assertNull(schema.getAttribute("pos_ext")); + assertNull(schema.getAttribute("pos_ext.x")); + assertNull(schema.getAttribute("pos_ext.y")); + + assertPositionAttribute(schema, "pos_ext", Attribute.CollectionType.SINGLE); + assertPositionSummary(schema, "pos_ext", false); + } + + @Test + public void requireThatPositionArrayIsSupported() throws Exception { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_array.sd"); + assertNull(schema.getAttribute("pos")); + assertNull(schema.getAttribute("pos.x")); + assertNull(schema.getAttribute("pos.y")); + + assertPositionAttribute(schema, "pos", Attribute.CollectionType.ARRAY); + assertPositionSummary(schema, "pos", true); + } + + private static void assertPositionAttribute(Schema schema, String fieldName, Attribute.CollectionType type) { + Attribute attribute = schema.getAttribute(PositionDataType.getZCurveFieldName(fieldName)); + assertNotNull(attribute); + assertTrue(attribute.isPosition()); + assertEquals(attribute.getCollectionType(), type); + assertEquals(attribute.getType(), Attribute.Type.LONG); + } + + private static void assertPositionSummary(Schema schema, String fieldName, boolean isArray) { + assertSummaryField(schema, + fieldName, + PositionDataType.getZCurveFieldName(fieldName), + (isArray ? DataType.getArray(PositionDataType.INSTANCE) : PositionDataType.INSTANCE), + SummaryTransform.GEOPOS); + assertNull(schema.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); + assertNull(schema.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); + } + + private static void assertSummaryField(Schema schema, String fieldName, String sourceName, DataType dataType, + SummaryTransform transform) + { + SummaryField summary = schema.getSummaryField(fieldName); + assertNotNull(summary); + assertEquals(1, summary.getSourceCount()); + assertEquals(sourceName, summary.getSingleSource()); + assertEquals(dataType, summary.getDataType()); + assertEquals(transform, summary.getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java new file mode 100644 index 00000000000..69bf62be84b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * Tests for the field "rank {" shortcut + * @author vegardh + * + */ +public class RankModifierTestCase extends AbstractSchemaTestCase { + @Test + public void testLiteral() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java new file mode 100644 index 00000000000..e380b1ab9af --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.google.common.collect.ImmutableList; +import com.yahoo.config.application.api.ApplicationPackage; +import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; +import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; +import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Helper class for setting up and asserting over a Search instance with a rank profile given literally + * in the search definition language. + * + * @author geirst + */ +class RankProfileSearchFixture { + + private final ImmutableList importers = ImmutableList.of(new TensorFlowImporter(), + new OnnxImporter(), + new LightGBMImporter(), + new XGBoostImporter()); + private final RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private final QueryProfileRegistry queryProfileRegistry; + private final Schema schema; + private final Map compiledRankProfiles = new HashMap<>(); + private final ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + + RankProfileSearchFixture(String rankProfiles) throws ParseException { + this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); + } + + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, + String rankProfiles) throws ParseException { + this(applicationpackage, queryProfileRegistry, rankProfiles, null, null); + } + + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, + String rankProfiles, String constant, String field) + throws ParseException { + this.queryProfileRegistry = queryProfileRegistry; + ApplicationBuilder builder = new ApplicationBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + String sdContent = "search test {\n" + + " " + (constant != null ? constant : "") + "\n" + + " document test {\n" + + " " + (field != null ? field : "") + "\n" + + " }\n" + + rankProfiles + + "\n" + + "}"; + builder.addSchema(sdContent); + builder.build(true); + schema = builder.getSchema(); + } + + public void assertFirstPhaseExpression(String expExpression, String rankProfile) { + assertEquals(expExpression, compiledRankProfile(rankProfile).getFirstPhaseRanking().getRoot().toString()); + } + + public void assertSecondPhaseExpression(String expExpression, String rankProfile) { + assertEquals(expExpression, compiledRankProfile(rankProfile).getSecondPhaseRanking().getRoot().toString()); + } + + public void assertRankProperty(String expValue, String name, String rankProfile) { + List rankPropertyList = compiledRankProfile(rankProfile).getRankPropertyMap().get(name); + assertEquals(1, rankPropertyList.size()); + assertEquals(expValue, rankPropertyList.get(0).getValue()); + } + + public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { + assertEquals(expexctedExpression, + compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); + } + + public RankProfile compileRankProfile(String rankProfile) { + return compileRankProfile(rankProfile, Path.fromString("nonexistinng")); + } + + public RankProfile compileRankProfile(String rankProfile, Path applicationDir) { + RankProfile compiled = rankProfileRegistry.get(schema, rankProfile) + .compile(queryProfileRegistry, + new ImportedMlModels(applicationDir.toFile(), executor, importers)); + compiledRankProfiles.put(rankProfile, compiled); + return compiled; + } + + /** Returns the given uncompiled profile */ + public RankProfile rankProfile(String rankProfile) { + return rankProfileRegistry.get(schema, rankProfile); + } + + /** Returns the given compiled profile, or null if not compiled yet or not present at all */ + public RankProfile compiledRankProfile(String rankProfile) { + return compiledRankProfiles.get(rankProfile); + } + + public Schema search() { return schema; } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java new file mode 100644 index 00000000000..dab1d9e6e95 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile.RankProperty; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.fail; + +public class RankPropertyVariablesTestCase extends AbstractSchemaTestCase { + + @Test + public void testRankPropVariables() throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankpropvars.sd", + new BaseDeployLogger(), + rankProfileRegistry, + new QueryProfileRegistry()); + assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar1", "foo"); + assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar_2", "bar"); + assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvarOne23", "baz"); + assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$Testvar1", "1"); + assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$Testvar_4", "4"); + assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$testvarFour23", "234234.234"); + } + + private void assertRankPropEquals(List props, String key, String val) { + for (RankProperty prop : props) { + if (prop.getName().equals(key)) { + if (prop.getValue().equals(val)) { + return; + } + } + } + fail(key+":"+val+ " not found in rank properties."); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java new file mode 100644 index 00000000000..4b6a22fc81a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java @@ -0,0 +1,521 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * @author bratseth + */ +public class RankingExpressionTypeResolverTestCase { + + @Test + public void tensorFirstPhaseMustProduceDouble() throws Exception { + try { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: attribute(a)", + " }", + " }", + "}" + )); + builder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", + Exceptions.toMessageString(expected)); + } + } + + + @Test + public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { + try { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "schema test {", + " document test { ", + " field a type tensor(d0[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function my_func() {", + " expression: x_tensor*2.0", + " }", + " function inline other_func() {", + " expression: z_tensor+3.0", + " }", + " first-phase {", + " expression: reduce(attribute(a),sum,d0)+y_tensor+my_func+other_func", + " }", + " constants {", + " x_tensor {", // legacy form + " type: tensor(x{})", + " value: { {x:bar}:17 }", + " }", + " y_tensor tensor(y{}):{{y:foo}:42 }", + " z_tensor tensor(z{}):{qux:666}", + " }", + " }", + "}" + )); + builder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x{},y{},z{})", + Exceptions.toMessageString(expected)); + } + } + + + + @Test + public void tensorSecondPhaseMustProduceDouble() throws Exception { + try { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(attribute(a))", + " }", + " second-phase {", + " expression: attribute(a)", + " }", + " }", + "}" + )); + builder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { + try { + ApplicationBuilder schemaBuilder = new ApplicationBuilder(); + schemaBuilder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[5]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(if(1>0, attribute(a), attribute(b)))", + " }", + " }", + "}" + )); + schemaBuilder.build(true); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[10],y[5]) while the 'false' type is tensor(z[10])" + + "\n'true' branch: attribute(a)" + + "\n'false' branch: attribute(b)", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testFunctionInvocationTypes() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[3]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function macro1(attribute_to_use) {", + " expression: attribute(attribute_to_use)", + " }", + " summary-features {", + " macro1(a)", + " macro1(b)", + " }", + " }", + "}" + )); + builder.build(true); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), + summaryFeatures(profile).get("macro1(a)").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("macro1(b)").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testTensorFunctionInvocationTypes_Nested() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[1]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function return_a() {", + " expression: return_first(attribute(a), attribute(b))", + " }", + " function return_b() {", + " expression: return_second(attribute(a), attribute(b))", + " }", + " function return_first(e1, e2) {", + " expression: e1", + " }", + " function return_second(e1, e2) {", + " expression: return_first(e2, e1)", + " }", + " summary-features {", + " return_a", + " return_b", + " }", + " }", + "}" + )); + builder.build(true); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), + summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testAttributeInvocationViaBoundIdentifier() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search newsarticle {", + " document newsarticle {", + " field title type string {", + " indexing {", + " input title | index", + " }", + " weight: 30", + " }", + " field usstaticrank type int {", + " indexing: summary | attribute", + " }", + " field eustaticrank type int {", + " indexing: summary | attribute", + " }", + " }", + " rank-profile default {", + " macro newsboost() { ", + " expression: 200 * matches(title)", + " }", + " macro commonboost(mystaticrank) { ", + " expression: attribute(mystaticrank) + newsboost", + " }", + " macro commonfirstphase(mystaticrank) { ", + " expression: nativeFieldMatch(title) + commonboost(mystaticrank) ", + " }", + " first-phase { expression: commonfirstphase(usstaticrank) }", + " }", + " rank-profile eurank inherits default {", + " first-phase { expression: commonfirstphase(eustaticrank) }", + " }", + "}")); + builder.build(true); + RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "eurank"); + } + + @Test + public void testTensorFunctionInvocationTypes_NestedSameName() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field a type tensor(x[10],y[1]) {", + " indexing: attribute", + " }", + " field b type tensor(z[10]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function return_a() {", + " expression: return_first(attribute(a), attribute(b))", + " }", + " function return_b() {", + " expression: return_second(attribute(a), attribute(b))", + " }", + " function return_first(e1, e2) {", + " expression: just_return(e1)", + " }", + " function just_return(e1) {", + " expression: e1", + " }", + " function return_second(e1, e2) {", + " expression: return_first(e2+0, e1)", + " }", + " summary-features {", + " return_a", + " return_b", + " }", + " }", + "}" + )); + builder.build(true); + RankProfile profile = + builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); + assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), + summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); + assertEquals(TensorType.fromSpec("tensor(z[10])"), + summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void testTensorFunctionInvocationTypes_viaFuncWithExpr() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search test {", + " document test {", + " field t1 type tensor(y{}) { indexing: attribute | summary }", + " field t2 type tensor(x{}) { indexing: attribute | summary }", + " }", + " rank-profile test {", + " function my_func(t) { expression: sum(t, x) + 1 }", + " function test_func_via_func_with_expr() { expression: call_func_with_expr( attribute(t1), attribute(t2) ) }", + " function call_func_with_expr(a, b) { expression: my_func( a * b ) }", + " summary-features { test_func_via_func_with_expr }", + " }", + "}")); + builder.build(true); + RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "test"); + assertEquals(TensorType.fromSpec("tensor(y{})"), + summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); + } + + @Test + public void importedFieldsAreAvailable() throws Exception { + ApplicationBuilder builder = new ApplicationBuilder(); + builder.addSchema(joinLines( + "search parent {", + " document parent {", + " field a type tensor(x[5],y[1000]) {", + " indexing: attribute", + " }", + " }", + "}" + )); + builder.addSchema(joinLines( + "search child {", + " document child { ", + " field ref type reference {", + "indexing: attribute | summary", + " }", + " }", + " import field ref.a as imported_a {}", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(attribute(imported_a))", + " }", + " }", + "}" + )); + builder.build(true); + } + + @Test + public void undeclaredQueryFeaturesAreAccepted() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type double {" + + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: query(foo) + f() + sum(attribute(anyfield))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + String message = logger.findMessage("The following query features"); + assertNull(message); + } + + @Test + public void undeclaredQueryFeaturesAreNotAcceptedWhenStrict() throws Exception { + try { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type double {" + + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " strict: true" + + " first-phase {", + " expression: query(foo) + f() + sum(attribute(anyfield))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + } + catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank profile 'my_rank_profile': rank profile 'my_rank_profile' is strict but is missing a query profile type declaration of features [query(bar), query(baz), query(foo)]", + Exceptions.toMessageString(e)); + } + } + + @Test + public void undeclaredQueryFeaturesAreAcceptedWithWarningWhenUsingTensors() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type tensor(d[2]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: query(foo) + f() + sum(attribute(anyfield))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + String message = logger.findMessage("The following query features"); + assertNotNull(message); + assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared in query profile types and " + + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", + message); + } + + @Test + public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(logger); + QueryProfileType myType = new QueryProfileType("mytype"); + myType.addField(new FieldDescription("rank.feature.query(foo)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + myType.addField(new FieldDescription("rank.feature.query(bar)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + myType.addField(new FieldDescription("rank.feature.query(baz)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + builder.getQueryProfileRegistry().getTypeRegistry().register(myType); + builder.addSchema(joinLines( + "search test {", + " document test { ", + " field anyfield type tensor(d[2]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(query(foo) + f() + sum(attribute(anyfield)))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + )); + builder.build(true); + String message = logger.findMessage("The following query features"); + assertNull(message); + } + + private Map summaryFeatures(RankProfile profile) { + return profile.getSummaryFeatures().stream().collect(Collectors.toMap(f -> f.toString(), f -> f)); + } + + private static class InspectableDeployLogger implements DeployLogger { + + private List messages = new ArrayList<>(); + + @Override + public void log(Level level, String message) { + messages.add(level + ": " + message); + } + + /** Returns the first message containing the given string, or null if none */ + public String findMessage(String substring) { + return messages.stream().filter(message -> message.contains(substring)).findFirst().orElse(null); + } + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java new file mode 100644 index 00000000000..4df0a09ec2e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.schema.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author lesters + */ +public class RankingExpressionWithLightGBMTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); + + private final static String lightGBMExpression = + "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + + @Test + public void testNestedLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithOnnxTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); + searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithOnnxTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java new file mode 100644 index 00000000000..713e11fd608 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java @@ -0,0 +1,184 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.DocumentDatabase; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import org.junit.After; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class RankingExpressionWithOnnxModelTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/onnx-model/"); + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testOnnxModelFeature() throws Exception { + VespaModel model = loadModel(applicationDir); + assertTransformedFeature(model); + assertGeneratedConfig(model); + + Path storedApplicationDir = applicationDir.append("copy"); + try { + storedApplicationDir.toFile().mkdirs(); + IOUtils.copy(applicationDir.append("services.xml").toString(), storedApplicationDir.append("services.xml").toString()); + IOUtils.copyDirectory(applicationDir.append("schemas").toFile(), storedApplicationDir.append("schemas").toFile()); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + + VespaModel storedModel = loadModel(storedApplicationDir); + assertTransformedFeature(storedModel); + assertGeneratedConfig(storedModel); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDir.toFile()); + } + } + + private VespaModel loadModel(Path path) throws Exception { + FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); + DeployState state = new DeployState.Builder().applicationPackage(applicationPackage).build(); + return new VespaModel(state); + } + + private void assertGeneratedConfig(VespaModel vespaModel) { + DocumentDatabase db = ((IndexedSearchCluster)vespaModel.getSearchClusters().get(0)).getDocumentDbs().get(0); + + RankingConstantsConfig.Builder rankingConstantsConfigBuilder = new RankingConstantsConfig.Builder(); + db.getConfig(rankingConstantsConfigBuilder); + var rankingConstantsConfig = rankingConstantsConfigBuilder.build(); + assertEquals(1, rankingConstantsConfig.constant().size()); + assertEquals("my_constant", rankingConstantsConfig.constant(0).name()); + assertEquals("tensor(d0[2])", rankingConstantsConfig.constant(0).type()); + assertEquals("files/constant.json", rankingConstantsConfig.constant(0).fileref().value()); + + OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); + ((OnnxModelsConfig.Producer) db).getConfig(builder); + OnnxModelsConfig config = new OnnxModelsConfig(builder); + assertEquals(6, config.model().size()); + for (OnnxModelsConfig.Model model : config.model()) { + assertTrue(model.dry_run_on_setup()); + } + + OnnxModelsConfig.Model model = config.model(0); + assertEquals("my_model", model.name()); + assertEquals(3, model.input().size()); + assertEquals("second/input:0", model.input(0).name()); + assertEquals("constant(my_constant)", model.input(0).source()); + assertEquals("first_input", model.input(1).name()); + assertEquals("attribute(document_field)", model.input(1).source()); + assertEquals("third_input", model.input(2).name()); + assertEquals("rankingExpression(my_function)", model.input(2).source()); + assertEquals(3, model.output().size()); + assertEquals("path/to/output:0", model.output(0).name()); + assertEquals("out", model.output(0).as()); + assertEquals("path/to/output:1", model.output(1).name()); + assertEquals("path_to_output_1", model.output(1).as()); + assertEquals("path/to/output:2", model.output(2).name()); + assertEquals("path_to_output_2", model.output(2).as()); + + model = config.model(1); + assertEquals("dynamic_model", model.name()); + assertEquals(1, model.input().size()); + assertEquals(1, model.output().size()); + assertEquals("rankingExpression(my_function)", model.input(0).source()); + + model = config.model(2); + assertEquals("unbound_model", model.name()); + assertEquals(1, model.input().size()); + assertEquals(1, model.output().size()); + assertEquals("rankingExpression(my_function)", model.input(0).source()); + + model = config.model(3); + assertEquals("files_model_onnx", model.name()); + assertEquals(3, model.input().size()); + assertEquals(3, model.output().size()); + assertEquals("path/to/output:0", model.output(0).name()); + assertEquals("path_to_output_0", model.output(0).as()); + assertEquals("path/to/output:1", model.output(1).name()); + assertEquals("path_to_output_1", model.output(1).as()); + assertEquals("path/to/output:2", model.output(2).name()); + assertEquals("path_to_output_2", model.output(2).as()); + assertEquals("files_model_onnx", model.name()); + + model = config.model(4); + assertEquals("another_model", model.name()); + assertEquals("third_input", model.input(2).name()); + assertEquals("rankingExpression(another_function)", model.input(2).source()); + + model = config.model(5); + assertEquals("files_summary_model_onnx", model.name()); + assertEquals(3, model.input().size()); + assertEquals(3, model.output().size()); + } + + private void assertTransformedFeature(VespaModel model) { + DocumentDatabase db = ((IndexedSearchCluster)model.getSearchClusters().get(0)).getDocumentDbs().get(0); + RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); + ((RankProfilesConfig.Producer) db).getConfig(builder); + RankProfilesConfig config = new RankProfilesConfig(builder); + assertEquals(10, config.rankprofile().size()); + + assertEquals("test_model_config", config.rankprofile(2).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(2).fef().property(0).name()); + assertEquals("vespa.rank.firstphase", config.rankprofile(2).fef().property(2).name()); + assertEquals("rankingExpression(firstphase)", config.rankprofile(2).fef().property(2).value()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(2).fef().property(3).name()); + assertEquals("onnxModel(my_model).out{d0:1}", config.rankprofile(2).fef().property(3).value()); + + assertEquals("test_generated_model_config", config.rankprofile(3).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(3).fef().property(0).name()); + assertEquals("rankingExpression(first_input).rankingScript", config.rankprofile(3).fef().property(2).name()); + assertEquals("rankingExpression(second_input).rankingScript", config.rankprofile(3).fef().property(4).name()); + assertEquals("rankingExpression(third_input).rankingScript", config.rankprofile(3).fef().property(6).name()); + assertEquals("vespa.rank.firstphase", config.rankprofile(3).fef().property(8).name()); + assertEquals("rankingExpression(firstphase)", config.rankprofile(3).fef().property(8).value()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(3).fef().property(9).name()); + assertEquals("onnxModel(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value()); + + assertEquals("test_summary_features", config.rankprofile(4).name()); + assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(0).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(3).name()); + assertEquals("1", config.rankprofile(4).fef().property(3).value()); + assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(4).name()); + assertEquals("onnxModel(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(4).value()); + assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(5).name()); + assertEquals("onnxModel(another_model).out", config.rankprofile(4).fef().property(5).value()); + + assertEquals("test_dynamic_model", config.rankprofile(5).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(5).fef().property(0).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(5).fef().property(3).name()); + assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:1}", config.rankprofile(5).fef().property(3).value()); + + assertEquals("test_dynamic_model_2", config.rankprofile(6).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(6).fef().property(5).name()); + assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:2}", config.rankprofile(6).fef().property(5).value()); + + assertEquals("test_dynamic_model_with_transformer_tokens", config.rankprofile(7).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(7).fef().property(1).name()); + assertEquals("tensor(d0[1],d1[10])((if (d1 < 1.0 + rankingExpression(__token_length@-1993461420) + 1.0, 0.0, if (d1 < 1.0 + rankingExpression(__token_length@-1993461420) + 1.0 + rankingExpression(__token_length@-1993461420) + 1.0, 1.0, 0.0))))", config.rankprofile(7).fef().property(1).value()); + + assertEquals("test_unbound_model", config.rankprofile(8).name()); + assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(8).fef().property(0).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(8).fef().property(3).name()); + assertEquals("onnxModel(unbound_model).my_output{d0:0, d1:1}", config.rankprofile(8).fef().property(3).value()); + + + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java new file mode 100644 index 00000000000..94a51d25717 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java @@ -0,0 +1,417 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; +import org.junit.After; +import org.junit.Test; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +public class RankingExpressionWithOnnxTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/onnx/"); + + private final static String name = "mnist_softmax"; + private final static String vespaExpression = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_layer_Variable), f(a,b)(a * b)), sum, d2) * 1.0, constant(mnist_softmax_layer_Variable_1) * 1.0, f(a,b)(a + b))"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testOnnxReferenceWithConstantFeature() { + RankProfileSearchFixture search = fixtureWith("constant(mytensor)", + "onnx_vespa('mnist_softmax.onnx')", + "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", + null); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceWithQueryFeature() { + String queryProfile = ""; + String queryProfileType = + "" + + " " + + ""; + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, + queryProfile, + queryProfileType); + RankProfileSearchFixture search = fixtureWith("query(mytensor)", + "onnx_vespa('mnist_softmax.onnx')", + null, + null, + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceWithDocumentFeature() { + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); + RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", + "onnx_vespa('mnist_softmax.onnx')", + null, + "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + + @Test + public void testOnnxReferenceWithFeatureCombination() { + String queryProfile = ""; + String queryProfileType = + "" + + " " + + ""; + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, queryProfileType); + RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", + "onnx_vespa('mnist_softmax.onnx')", + "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", + "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", + "Placeholder", + application); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + + @Test + public void testNestedOnnxReference() { + RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", + "5 + sum(onnx_vespa('mnist_softmax.onnx'))"); + search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); + } + + @Test + public void testOnnxReferenceWithSpecifiedOutput() { + RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx', 'layer_add')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceWithSpecifiedOutputAndSignature() { + RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx', 'default.layer_add')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testOnnxReferenceMissingFunction() throws ParseException { + try { + RankProfileSearchFixture search = new RankProfileSearchFixture( + new StoringApplicationPackage(applicationDir), + new QueryProfileRegistry(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: onnx_vespa('mnist_softmax.onnx')" + + " }\n" + + " }"); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx_vespa(\"mnist_softmax.onnx\"): " + + "Model refers input 'Placeholder' of type tensor(d0[1],d1[784]) but this function is " + + "not present in rank profile 'my_profile'", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testOnnxReferenceWithWrongFunctionType() { + try { + RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d5[10])(0.0)", + "onnx_vespa('mnist_softmax.onnx')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx_vespa(\"mnist_softmax.onnx\"): " + + "Model refers input 'Placeholder'. The required type of this is tensor(d0[1],d1[784]), " + + "but this function returns tensor(d0[1],d5[10])", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testOnnxReferenceSpecifyingNonExistingOutput() { + try { + RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx', 'y')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + fail("Expecting exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + + "onnx_vespa(\"mnist_softmax.onnx\",\"y\"): " + + "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: default.layer_add", + Exceptions.toMessageString(expected)); + } + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", + "onnx_vespa(\"mnist_softmax.onnx\")"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[2],d1[784])(0.0)", + "onnx_vespa('mnist_softmax.onnx')", + null, + null, + "Placeholder", + storedApplication); + searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); + // Verify that the constants exists, but don't verify the content as we are not + // simulating file distribution in this test + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + @Test + public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { + String rankProfile = + " rank-profile my_profile {\n" + + " function Placeholder() {\n" + + " expression: tensor(d0[1],d1[784])(0.0)\n" + + " }\n" + + " function " + name + "_layer_Variable() {\n" + + " expression: tensor(d1[10],d2[784])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx_vespa('mnist_softmax.onnx')" + + " }\n" + + " }" + + " rank-profile my_profile_child inherits my_profile {\n" + + " }"; + + String vespaExpressionWithoutConstant = + "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), " + name + "_layer_Variable, f(a,b)(a * b)), sum, d2) * 1.0, constant(" + name + "_layer_Variable_1) * 1.0, f(a,b)(a + b))"; + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfile, new StoringApplicationPackage(applicationDir)); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.compileRankProfile("my_profile_child", applicationDir.append("models")); + search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); + search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); + + assertNull("Constant overridden by function is not added", + search.search().constants().get(name + "_Variable")); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); + searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); + searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); + searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); + searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); + assertNull("Constant overridden by function is not added", + searchFromStored.search().constants().get(name + "_Variable")); + } finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + @Test + public void testFunctionGeneration() { + final String name = "small_constants_and_functions"; + final String rankProfiles = + " rank-profile my_profile {\n" + + " function input() {\n" + + " expression: tensor(d0[3])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx_vespa('" + name + ".onnx')" + + " }\n" + + " }"; + final String functionName = "imported_ml_function_" + name + "_exp_output"; + final String expression = "join(" + functionName + ", reduce(join(join(reduce(" + functionName + ", sum, d0), tensor(d0[1])(1.0), f(a,b)(a * b)), constant(" + name + "_epsilon), f(a,b)(a + b)), sum, d0), f(a,b)(a / b))"; + final String functionExpression = "map(input, f(a)(exp(a)))"; + + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfiles, new StoringApplicationPackage(applicationDir)); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.assertFirstPhaseExpression(expression, "my_profile"); + search.assertFunction(functionExpression, functionName, "my_profile"); + } + + @Test + public void testImportingFromStoredExpressionsWithSmallConstantsAndInheritance() throws IOException { + final String name = "small_constants_and_functions"; + final String rankProfiles = + " rank-profile my_profile {\n" + + " function input() {\n" + + " expression: tensor(d0[3])(0.0)\n" + + " }\n" + + " first-phase {\n" + + " expression: onnx_vespa('" + name + ".onnx')" + + " }\n" + + " }" + + " rank-profile my_profile_child inherits my_profile {\n" + + " }"; + final String functionName = "imported_ml_function_" + name + "_exp_output"; + final String expression = "join(" + functionName + ", reduce(join(join(reduce(" + functionName + ", sum, d0), tensor(d0[1])(1.0), f(a,b)(a * b)), constant(" + name + "_epsilon), f(a,b)(a + b)), sum, d0), f(a,b)(a / b))"; + final String functionExpression = "map(input, f(a)(exp(a)))"; + + RankProfileSearchFixture search = uncompiledFixtureWith(rankProfiles, new StoringApplicationPackage(applicationDir)); + search.compileRankProfile("my_profile", applicationDir.append("models")); + search.compileRankProfile("my_profile_child", applicationDir.append("models")); + search.assertFirstPhaseExpression(expression, "my_profile"); + search.assertFirstPhaseExpression(expression, "my_profile_child"); + assertSmallConstant(name + "_epsilon", TensorType.fromSpec("tensor()"), search); + search.assertFunction(functionExpression, functionName, "my_profile"); + search.assertFunction(functionExpression, functionName, "my_profile_child"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfiles, storedApplication); + searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); + searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); + searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); + searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); + assertSmallConstant(name + "_epsilon", TensorType.fromSpec("tensor()"), search); + searchFromStored.assertFunction(functionExpression, functionName, "my_profile"); + searchFromStored.assertFunction(functionExpression, functionName, "my_profile_child"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private void assertSmallConstant(String name, TensorType type, RankProfileSearchFixture search) { + var value = search.compiledRankProfile("my_profile").constants().get(FeatureNames.asConstantFeature(name)); + assertNotNull(value); + assertEquals(type, value.type()); + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression) { + return fixtureWith(placeholderExpression, firstPhaseExpression, null, null, "Placeholder", + new StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression, + String constant, String field) { + return fixtureWith(placeholderExpression, firstPhaseExpression, constant, field, "Placeholder", + new StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture uncompiledFixtureWith(String rankProfile, StoringApplicationPackage application) { + try { + return new RankProfileSearchFixture(application, application.getQueryProfiles(), + rankProfile, null, null); + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + private RankProfileSearchFixture fixtureWith(String functionExpression, + String firstPhaseExpression, + String constant, + String field, + String functionName, + StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + + " }\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } + catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + static class StoringApplicationPackage extends MockApplicationPackage { + + StoringApplicationPackage(Path applicationPackageWritableRoot) { + this(applicationPackageWritableRoot, null, null); + } + + StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { + super(new File(applicationPackageWritableRoot.toString()), + null, null, List.of(), Map.of(), null, + null, null, false, queryProfile, queryProfileType); + } + + @Override + public ApplicationFile getFile(Path file) { + return new MockApplicationFile(file, Path.fromString(root().toString())); + } + + @Override + public List getFiles(Path path, String suffix) { + File[] files = getFileReference(path).listFiles(); + if (files == null) return List.of(); + List readers = new ArrayList<>(); + for (File file : files) { + if ( ! file.getName().endsWith(suffix)) continue; + try { + readers.add(new NamedReader(file.getName(), new FileReader(file))); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return readers; + } + + } + + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java new file mode 100644 index 00000000000..1f065bc7a20 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java @@ -0,0 +1,202 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class RankingExpressionWithTensorTestCase { + + @Test + public void requireThatSingleLineConstantMappedTensorCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{},y{}):{ {x:1,y:2}:1, {x:2,y:1}:2 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantIndexedTensorCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):{ {x:0}:1, {x:1}:2, {x:2}:3 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantIndexedTensorShortFormCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):[1, 2, 3]\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(constant(my_tensor))\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{},y{}):{{x:1,y:2}:1, {x:2,y:1}:2}\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatMultiLineConstantTensorAndTypeCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{},y{}):\n" + + " { {x:1,y:2}:1,\n" + + " {x:2,y:1}:2 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInSecondPhaseExpression() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " second-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{}):{ {x:1}:1 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertSecondPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInInheritedRankProfile() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile parent {\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " }\n" + + " }\n" + + " }\n" + + " rank-profile my_profile inherits parent {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " function my_macro() {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " first-phase {\n" + + " expression: 5.0 + my_macro\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x{}):{ {x:1}:1 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); + f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatCombinationOfConstantTensorsAndConstantValuesCanBeUsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: my_number_1 + sum(my_tensor) + my_number_2\n" + + " }\n" + + " constants {\n" + + " my_number_1 double: 3.0\n" + + " my_tensor tensor(x{}):{ {x:1}:1 }\n" + + " my_number_2 double: 5.0\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("3.0 + reduce(constant(my_tensor), sum) + 5.0", "my_profile"); + f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatInvalidTensorTypeSpecThrowsException() throws ParseException { + try { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " constants {\n" + + " my_tensor tensor(x):{ {x:1}:1 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertStartsWith("Type of constant(my_tensor): Illegal tensor type spec: A tensor type spec must be on the form", + e.getMessage()); + } + } + + private void assertStartsWith(String prefix, String string) { + assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java new file mode 100644 index 00000000000..f8086fb3bc6 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.expressiontransforms.TokenTransformer; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.tensor.Tensor; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +public class RankingExpressionWithTransformerTokensTestCase { + + @Test + public void testTokenInputIds() throws Exception { + String expected = "tensor(d0[1],d1[12]):[101,1,2,102,3,4,5,102,6,7,102,0]"; + String a = "tensor(d0[2]):[1,2]"; + String b = "tensor(d0[3]):[3,4,5]"; + String c = "tensor(d0[2]):[6,7]"; + String expression = "tokenInputIds(12, a, b, c)"; + Tensor result = evaluateExpression(expression, a, b, c); + assertEquals(Tensor.from(expected), result); + } + + @Test + public void testTokenTypeIds() throws Exception { + String expected = "tensor(d0[1],d1[10]):[0,0,0,0,1,1,1,1,0,0]"; + String a = "tensor(d0[2]):[1,2]"; + String b = "tensor(d0[3]):[3,4,5]"; + String expression = "tokenTypeIds(10, a, b)"; + Tensor result = evaluateExpression(expression, a, b); + assertEquals(Tensor.from(expected), result); + } + + @Test + public void testAttentionMask() throws Exception { + String expected = "tensor(d0[1],d1[10]):[1,1,1,1,1,1,1,1,0,0]"; + String a = "tensor(d0[2]):[1,2]"; + String b = "tensor(d0[3]):[3,4,5]"; + String expression = "tokenAttentionMask(10, a, b)"; + Tensor result = evaluateExpression(expression, a, b); + assertEquals(Tensor.from(expected), result); + } + + private Tensor evaluateExpression(String expression, String a, String b) throws Exception { + return evaluateExpression(expression, a, b, null, null); + } + + private Tensor evaluateExpression(String expression, String a, String b, String c) throws Exception { + return evaluateExpression(expression, a, b, c, null); + } + + private Tensor evaluateExpression(String expression, String a, String b, String c, String d) throws Exception { + MapContext context = new MapContext(); + if (a != null) context.put("a", new TensorValue(Tensor.from(a))); + if (b != null) context.put("b", new TensorValue(Tensor.from(b))); + if (c != null) context.put("c", new TensorValue(Tensor.from(c))); + if (d != null) context.put("d", new TensorValue(Tensor.from(d))); + var transformContext = createTransformContext(); + var rankingExpression = new RankingExpression(expression); + var transformed = new TokenTransformer().transform(rankingExpression, transformContext); + for (var entry : transformContext.rankProfile().getFunctions().entrySet()) { + context.put(entry.getKey(), entry.getValue().function().getBody().evaluate(context).asDouble()); + } + return transformed.evaluate(context).asTensor(); + } + + private RankProfileTransformContext createTransformContext() throws ParseException { + MockApplicationPackage application = (MockApplicationPackage) MockApplicationPackage.createEmpty(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfileRegistry = application.getQueryProfiles(); + String sdContent = "search test {\n" + + " document test {}\n" + + " rank-profile my_profile inherits default {}\n" + + "}"; + ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); + schemaBuilder.addSchema(sdContent); + schemaBuilder.build(true); + Schema schema = schemaBuilder.getSchema(); + RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); + return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java new file mode 100644 index 00000000000..e1b1473a59a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.schema.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author grace-lam + * @author bratseth + */ +public class RankingExpressionWithXGBoostTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/xgboost/"); + + private final static String vespaExpression = + "if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + " + + "if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testXGBoostReference() { + RankProfileSearchFixture search = fixtureWith("xgboost('xgboost.2.2.json')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + + @Test + public void testNestedXGBoostReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(xgboost('xgboost.2.2.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("xgboost('xgboost.2.2.json')"); + search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithOnnxTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("xgboost('xgboost.2.2.json')"); + searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithOnnxTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java new file mode 100644 index 00000000000..ace3788e49a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java @@ -0,0 +1,128 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class RankingExpressionsTestCase extends AbstractSchemaTestCase { + + private static Schema createSearch(String dir, ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + return ApplicationBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); + } + + @Test + public void testFunctions() throws IOException, ParseException { + ModelContext.Properties deployProperties = new TestProperties(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + Schema schema = createSearch("src/test/examples/rankingexpressionfunction", deployProperties, rankProfileRegistry); + RankProfile functionsRankProfile = rankProfileRegistry.get(schema, "macros"); + Map functions = functionsRankProfile.getFunctions(); + assertEquals(2, functions.get("titlematch$").function().arguments().size()); + assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); + assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals(0, functions.get("artistmatch").function().arguments().size()); + + RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), + new ImportedMlModels(), new AttributeFields(schema), deployProperties); + List> rankProperties = rawRankProfile.configProperties(); + assertEquals(6, rankProperties.size()); + + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); + + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); + + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); + + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); + + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); + } + + @Test(expected = IllegalArgumentException.class) + public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + Schema schema = createSearch("src/test/examples/rankingexpressioninfile", new TestProperties(), registry); + new DerivedConfiguration(schema, registry); // rank profile parsing happens during deriving + } + + private void verifyProfile(RankProfile profile, List expectedFunctions, List> rankProperties, + LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels models, + AttributeFields attributes, ModelContext.Properties properties) { + var functions = profile.getFunctions(); + assertEquals(expectedFunctions.size(), functions.size()); + for (String func : expectedFunctions) { + assertTrue(functions.containsKey(func)); + } + + RawRankProfile raw = new RawRankProfile(profile, largeExpressions, queryProfiles, models, attributes, properties); + assertEquals(rankProperties.size(), raw.configProperties().size()); + for (int i = 0; i < rankProperties.size(); i++) { + assertEquals(rankProperties.get(i).getFirst(), raw.configProperties().get(i).getFirst()); + assertEquals(rankProperties.get(i).getSecond(), raw.configProperties().get(i).getSecond()); + } + } + + private void verifySearch(Schema schema, RankProfileRegistry rankProfileRegistry, LargeRankExpressions largeExpressions, + QueryProfileRegistry queryProfiles, ImportedMlModels models, ModelContext.Properties properties) + { + AttributeFields attributes = new AttributeFields(schema); + + verifyProfile(rankProfileRegistry.get(schema, "base"), Arrays.asList("large_f", "large_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), + largeExpressions, queryProfiles, models, attributes, properties); + for (String child : Arrays.asList("child_a", "child_b")) { + verifyProfile(rankProfileRegistry.get(schema, child), Arrays.asList("large_f", "large_m", "large_local_f", "large_local_m"), + Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), + new Pair<>("rankingExpression(large_local_f).expressionName", child + ".large_local_f"), new Pair<>("rankingExpression(large_local_m).expressionName", child + ".large_local_m"), + new Pair<>("vespa.rank.firstphase", "rankingExpression(firstphase)"), new Pair<>("rankingExpression(firstphase).expressionName", child + ".firstphase")), + largeExpressions, queryProfiles, models, attributes, properties); + } + } + + @Test + public void testLargeInheritedFunctions() throws IOException, ParseException { + ModelContext.Properties properties = new TestProperties(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry(), 50); + QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); + ImportedMlModels models = new ImportedMlModels(); + Schema schema = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); + verifySearch(schema, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + // Need to verify that second derivation works as that will happen if same sd is used in multiple content clusters + verifySearch(schema, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java new file mode 100644 index 00000000000..57b4d928a52 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.parser.ParseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class ReferenceFieldTestCase { + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void reference_fields_are_parsed_from_search_definition() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(); + String campaignSdContent = + "schema campaign {\n" + + " document campaign {\n" + + " }\n" + + "}"; + String salespersonSdContent = + "schema salesperson {\n" + + " document salesperson {\n" + + " }\n" + + "}"; + String adSdContent = + "schema ad {\n" + + " document ad {\n" + + " field campaign_ref type reference { indexing: attribute }\n" + + " field salesperson_ref type reference { indexing: attribute }\n" + + " }\n" + + "}"; + builder.addSchema(campaignSdContent); + builder.addSchema(salespersonSdContent); + builder.addSchema(adSdContent); + builder.build(true); + Schema schema = builder.getSchema("ad"); + assertSearchContainsReferenceField("campaign_ref", "campaign", schema.getDocument()); + assertSearchContainsReferenceField("salesperson_ref", "salesperson", schema.getDocument()); + } + + @Test + public void cyclic_document_dependencies_are_detected() throws ParseException { + var builder = new ApplicationBuilder(new TestProperties()); + String campaignSdContent = + "schema campaign {\n" + + " document campaign {\n" + + " field ad_ref type reference { indexing: attribute }\n" + + " }\n" + + "}"; + String adSdContent = + "schema ad {\n" + + " document ad {\n" + + " field campaign_ref type reference { indexing: attribute }\n" + + " }\n" + + "}"; + builder.addSchema(campaignSdContent); + builder.addSchema(adSdContent); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("reference cycle for documents"); + builder.build(true); + } + + private static void assertSearchContainsReferenceField(String expectedFieldname, + String referencedDocType, + SDDocumentType documentType) { + Field field = documentType.getDocumentType().getField(expectedFieldname); + assertNotNull("Field does not exist in document type: " + expectedFieldname, field); + DataType dataType = field.getDataType(); + assertTrue(dataType instanceof NewDocumentReferenceDataType); + NewDocumentReferenceDataType refField = (NewDocumentReferenceDataType) dataType; + assertEquals(referencedDocType, refField.getTargetTypeName()); + assertTrue(! refField.isTemporary()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java new file mode 100644 index 00000000000..974d8c261ca --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.derived.AbstractExportingTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen Hult + */ +public class ReservedDocumentNamesTestCase extends AbstractExportingTestCase { + + @Test + public void requireThatPositionIsAReservedDocumentName() throws IOException, ParseException { + try { + assertCorrectDeriving("reserved_position"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'position': Document name 'position' is reserved.", e.getMessage()); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java new file mode 100644 index 00000000000..e405a105f3c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.util.logging.Level; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lesters + */ +public class ReservedRankingExpressionFunctionNamesTestCase { + + @Test + public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { + TestDeployLogger deployLogger = new TestDeployLogger(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(deployLogger, rankProfileRegistry); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test_rank_profile {\n" + + " function not_a_reserved_name(x) {\n" + + " expression: x + x\n" + + " }\n" + + " function sigmoid(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sigmoid(2) + not_a_reserved_name(1)\n" + + " }\n" + + " }\n" + + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + + " function sin(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sigmoid(2) + sin(1)\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + + assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile")); + assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile_2")); + assertTrue(deployLogger.log.contains("sin") && deployLogger.log.contains("test_rank_profile_2")); + assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile")); + assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile_2")); + + } + + public static class TestDeployLogger implements DeployLogger { + public String log = ""; + @Override + public void log(Level level, String message) { + log += message; + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java b/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java new file mode 100644 index 00000000000..03f9d7c5960 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +/** + * @author hmusum + */ +public class SchemaMustHaveDocumentTest { + + @Test + public void requireErrorWhenMissingDocument() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); + fail("SD without document"); + } catch (IllegalArgumentException e) { + if (!e.getMessage() + .contains("For schema 'imageconfig': A search specification must have an equally named document inside of it.")) { + throw e; + } + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java new file mode 100644 index 00000000000..76132a4d09f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; + +public class SummaryConsistencyTestCase { + + @Test + public void attribute_combiner_transform_is_set_when_source_is_array_of_struct_with_only_struct_field_attributes() throws ParseException { + String sd = joinLines( + "search structmemorysummary {", + " document structmemorysummary {", + " struct elem {", + " field name type string {}", + " field weight type int {}\n", + " }", + " field elem_array type array {", + " indexing: summary", + " struct-field name {", + " indexing: attribute", + " }", + " struct-field weight {", + " indexing: attribute", + " }", + " }", + " }", + " document-summary unfiltered {", + " summary elem_array_unfiltered type array {", + " source: elem_array", + " }", + " }", + "", + "}" + ); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java new file mode 100644 index 00000000000..d94815015d7 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; + +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTestCase { + + @Test + public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); + fail("This should throw and never get here"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); + } + } + + @Test + public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); + fail("This should throw and never get here"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); + } + } + + @Test + public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { + try { + ApplicationBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); + fail("This should throw and never get here"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); + } + } + + @Test + public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); + BaseDeployLogger deployLogger = new BaseDeployLogger(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + new SummaryFieldsMustHaveValidSource(schema, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); + assertEquals("documentid", schema.getSummary("withid").getSummaryField("w").getSingleSource()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java new file mode 100644 index 00000000000..67c77508e3b --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java @@ -0,0 +1,172 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + + +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author geirst + */ +public class TensorFieldTestCase { + + @Test + public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { + try { + createFromString(getSd("field f1 type array {}")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead.", + e.getMessage()); + } + } + + @Test + public void requireThatTensorFieldCannotBeIndexField() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that.", + e.getMessage()); + } + } + + @Test + public void requireThatIndexedTensorAttributeCannotBeFastSearch() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(x[3]) { indexing: attribute \n attribute: fast-search }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'.", e.getMessage()); + } + } + + @Test + public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertStartsWith("Field type: Illegal tensor type spec:", e.getMessage()); + } + } + + @Test + public void hnsw_index_is_default_turned_off() throws ParseException { + var attr = getAttributeFromSd("field t1 type tensor(x[64]) { indexing: attribute }", "t1"); + assertFalse(attr.hnswIndexParams().isPresent()); + } + + @Test + public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { + assertHnswIndexParams("", 16, 200); + assertHnswIndexParams("index: hnsw", 16, 200); + } + + @Test + public void hnsw_index_parameters_can_be_specified() throws ParseException { + assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); + assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); + assertHnswIndexParams(joinLines("index {", + " hnsw {", + " max-links-per-node: 32", + " neighbors-to-explore-at-insert: 300", + " }", + "}"), + 32, 300); + } + + @Test + public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { + try { + createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); + } + } + + @Test + public void tensor_with_hnsw_index_parameters_must_be_an_index() throws ParseException { + try { + createFromString(getSd(joinLines( + "field t1 type tensor(x[64]) {", + " indexing: attribute ", + " index {", + " hnsw { max-links-per-node: 32 }", + " }", + "}"))); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For schema 'test', field 't1': " + + "A tensor that specifies hnsw index parameters must also specify 'index' in 'indexing'", + e.getMessage()); + } + } + + @Test + public void tensors_with_at_least_one_mapped_dimension_can_be_direct() throws ParseException { + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); + } + + @Test + public void tensors_with_at_least_one_mapped_dimension_can_be_fast_rank() throws ParseException { + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank()); + assertTrue(getAttributeFromSd( + "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank()); + } + + private static String getSd(String field) { + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + + private Attribute getAttributeFromSd(String fieldSpec, String attrName) throws ParseException { + return createFromString(getSd(fieldSpec)).getSchema().getAttribute(attrName); + } + + private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(indexSpec); + var search = createFromString(sd).getSchema(); + var attr = search.getAttribute("t1"); + var params = attr.hnswIndexParams(); + assertTrue(params.isPresent()); + assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); + assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); + } + + private String getSdWithIndexSpec(String indexSpec) { + return getSd(joinLines("field t1 type tensor(x[64]) {", + " indexing: attribute | index", + " " + indexSpec, + "}")); + } + + private void assertStartsWith(String prefix, String string) { + assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java new file mode 100644 index 00000000000..aaf5f381c62 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java @@ -0,0 +1,234 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class TensorTransformTestCase extends AbstractSchemaTestCase { + + @Test + public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { + assertTransformedExpression("max(1.0,2.0)", + "max(1.0,2.0)"); + assertTransformedExpression("min(attribute(double_field),x)", + "min(attribute(double_field),x)"); + assertTransformedExpression("max(attribute(double_field),attribute(double_array_field))", + "max(attribute(double_field),attribute(double_array_field))"); + assertTransformedExpression("min(attribute(tensor_field_1),attribute(double_field))", + "min(attribute(tensor_field_1),attribute(double_field))"); + assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)", + "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)"); + assertTransformedExpression("min(constant(test_constant_tensor),1.0)", + "min(test_constant_tensor,1.0)"); + assertTransformedExpression("max(constant(base_constant_tensor),1.0)", + "max(base_constant_tensor,1.0)"); + assertTransformedExpression("min(constant(file_constant_tensor),1.0)", + "min(constant(file_constant_tensor),1.0)"); + assertTransformedExpression("max(query(q),1.0)", + "max(query(q),1.0)"); + assertTransformedExpression("max(query(n),1.0)", + "max(query(n),1.0)"); + } + + @Test + public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { + assertTransformedExpression("reduce(attribute(tensor_field_1),max,x)", + "max(attribute(tensor_field_1),x)"); + assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)", + "1 + max(attribute(tensor_field_1),x)"); + assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),reduce(attribute(tensor_field_1),sum,x))", + "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),reduce(attribute(tensor_field_1), sum, x))"); + assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)", + "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)"); + assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)", + "max(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),x)"); + assertTransformedExpression("max(reduce(attribute(tensor_field_1),max,x),x)", + "max(max(attribute(tensor_field_1),x),x)"); // will result in deploy error. + assertTransformedExpression("reduce(reduce(attribute(tensor_field_2),max,x),max,y)", + "max(max(attribute(tensor_field_2),x),y)"); + } + + @Test + public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { + assertTransformedExpression("reduce(constant(test_constant_tensor),max,x)", + "max(test_constant_tensor,x)"); + assertTransformedExpression("reduce(constant(base_constant_tensor),max,x)", + "max(base_constant_tensor,x)"); + assertTransformedExpression("reduce(constant(file_constant_tensor),min,x)", + "min(constant(file_constant_tensor),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { + assertTransformedExpression("reduce(attribute(double_field)+attribute(tensor_field_1),min,x)", + "min(attribute(double_field) + attribute(tensor_field_1),x)"); + assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)", + "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)"); + assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)", + "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); + assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", + "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertTransformedExpression("reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)", + "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { + assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)", + "max(tensorFromLabels(attribute(double_array_field)),double_array_field)"); + assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field),x),max,x)", + "max(tensorFromLabels(attribute(double_array_field),x),x)"); + assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)", + "max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)"); + assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)", + "max(tensorFromWeightedSet(attribute(weightedset_field),x),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { + assertTransformedExpression("reduce(query(q),max,x)", "max(query(q),x)"); + assertTransformedExpression("max(query(n),x)", "max(query(n),x)"); + } + + @Test + public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { + assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", + "max(returns_tensor,x)"); + assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", + "max(wraps_returns_tensor,x)"); + assertTransformedExpression("reduce(rankingExpression(tensor_inheriting),max,x)", + "max(tensor_inheriting,x)"); + assertTransformedExpression("reduce(rankingExpression(returns_tensor_with_arg@),max,x)", + "max(returns_tensor_with_arg(attribute(tensor_field_1)),x)"); + } + + private void assertTransformedExpression(String expected, String original) throws ParseException { + for (Pair rankPropertyExpression : buildSearch(original)) { + String rankProperty = rankPropertyExpression.getFirst(); + if (rankProperty.equals("rankingExpression(testexpression).rankingScript")) { + String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); + assertEquals(expected, rankExpression); + return; + } + } + fail("No 'rankingExpression(testexpression).rankingScript' property produced"); + } + + private List> buildSearch(String expression) throws ParseException { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); + builder.addSchema( + "search test {\n" + + " document test { \n" + + " field double_field type double { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field double_array_field type array { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field weightedset_field type weightedset { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_1 type tensor(x{}) { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " field tensor_field_2 type tensor(x[3],y[3]) { \n" + + " indexing: summary | attribute \n" + + " }\n" + + " }\n" + + " constant file_constant_tensor {\n" + + " file: constants/tensor.json\n" + + " type: tensor(x{})\n" + + " }\n" + + " rank-profile base {\n" + + " constants {\n" + + " base_constant_tensor tensor(x[1]):[0]\n"+ + " }\n" + + " function base_tensor() {\n" + + " expression: constant(base_constant_tensor)\n" + + " }\n" + + " }\n" + + " rank-profile test inherits base {\n" + + " constants {\n" + + " test_constant_tensor tensor(x[1]):[1]" + + " }\n" + + " function returns_tensor_with_arg(arg1) {\n" + + " expression: 2.0 * arg1\n" + + " }\n" + + " function wraps_returns_tensor() {\n" + + " expression: returns_tensor\n" + + " }\n" + + " function returns_tensor() {\n" + + " expression: attribute(tensor_field_2)\n" + + " }\n" + + " function tensor_inheriting() {\n" + + " expression: base_tensor\n" + + " }\n" + + " function testexpression() {\n" + + " expression: " + expression + "\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + Schema s = builder.getSchema(); + RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); + List> testRankProperties = new RawRankProfile(test, + new LargeRankExpressions(new MockFileRegistry()), + queryProfiles, + new ImportedMlModels(), + new AttributeFields(s), new TestProperties()).configProperties(); + return testRankProperties; + } + + private static QueryProfileRegistry setupQueryProfileTypes() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); + QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); + type.addField(new FieldDescription("ranking.features.query(q)", + FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); + type.addField(new FieldDescription("ranking.features.query(n)", + FieldType.fromString("integer", typeRegistry)), typeRegistry); + typeRegistry.register(type); + return registry; + } + + private String censorBindingHash(String s) { + StringBuilder b = new StringBuilder(); + boolean areInHash = false; + for (int i = 0; i < s.length() ; i++) { + char current = s.charAt(i); + if ( ! Character.isLetterOrDigit(current)) // end of hash + areInHash = false; + if ( ! areInHash) + b.append(current); + if (current == '@') // start of hash + areInHash = true; + } + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java new file mode 100644 index 00000000000..87bb2e96042 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Collections; + +/** + * @author bjorncs + */ +public class ValidateFieldTypesTest { + + private static final String IMPORTED_FIELD_NAME = "imported_myfield"; + private static final String DOCUMENT_NAME = "my_doc"; + + @SuppressWarnings("deprecation") + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void throws_exception_if_type_of_document_field_does_not_match_summary_field() { + Schema schema = createSearchWithDocument(DOCUMENT_NAME); + schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME, DataType.INT)); + schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, DataType.STRING, schema)); + + ValidateFieldTypes validator = new ValidateFieldTypes(schema, null, null, null); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "For schema '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); + validator.process(true, false); + } + + private static Schema createSearch(String documentType) { + return new Schema(documentType, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new TestableDeployLogger(), + new TestProperties()); + } + + private static Schema createSearchWithDocument(String documentName) { + Schema schema = createSearch(documentName); + SDDocumentType document = new SDDocumentType(documentName, schema); + schema.addDocument(document); + return schema; + } + + private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) { + Schema targetSchema = createSearchWithDocument("target_doc"); + SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); + DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); + ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); + return new ImportedFields(Collections.singletonMap(fieldName, importedField)); + } + + private static DocumentSummary createDocumentSummary(String fieldName, DataType dataType, Schema schema) { + DocumentSummary summary = new DocumentSummary("mysummary", schema); + summary.add(new SummaryField(fieldName, dataType)); + return summary; + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java new file mode 100644 index 00000000000..016e30e80af --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java @@ -0,0 +1,77 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.ml.ImportedModelTester; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * Tests adding Vespa ranking expression based models in the models/ dir + * + * @author bratseth + */ +public class VespaMlModelTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/vespa/"); + + private final String expectedRankConfig = + "constant(constant1).type : tensor(x[3])\n" + + "constant(constant1).value : tensor(x[3]):[0.5, 1.5, 2.5]\n" + + "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" + + "rankingExpression(foo1).input2.type : tensor(x[3])\n" + + "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" + + "rankingExpression(foo2).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1asLarge), max, x) * 3.0\n" + + "rankingExpression(foo2).input2.type : tensor(x[3])\n" + + "rankingExpression(foo2).input1.type : tensor(name{},x[3])\n"; + + /** The model name */ + private final String name = "example"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testGlobalVespaModel() throws IOException { + ImportedModelTester tester = new ImportedModelTester(name, applicationDir); + VespaModel model = tester.createVespaModel(); + tester.assertLargeConstant("constant1asLarge", model, Optional.of(3L)); + assertEquals(expectedRankConfig, rankConfigOf("example", model)); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedAppDir = applicationDir.append("copy"); + try { + storedAppDir.toFile().mkdirs(); + IOUtils.copy(applicationDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + ImportedModelTester storedTester = new ImportedModelTester(name, storedAppDir); + VespaModel storedModel = storedTester.createVespaModel(); + storedTester.assertLargeConstant("constant1asLarge", model, Optional.of(3L)); + assertEquals(expectedRankConfig, rankConfigOf("example", storedModel)); + } + finally { + IOUtils.recursiveDeleteDir(storedAppDir.toFile()); + } + } + + private String rankConfigOf(String rankProfileName, VespaModel model) { + StringBuilder b = new StringBuilder(); + RawRankProfile profile = model.rankProfileList().getRankProfiles().get(rankProfileName); + for (var property : profile.configProperties()) + b.append(property.getFirst()).append(" : ").append(property.getSecond()).append("\n"); + return b.toString(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java new file mode 100644 index 00000000000..2f62228cc3f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +/** @author bratseth */ +public class WeightedSetSummaryToTestCase extends AbstractSchemaTestCase { + + @Test + public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { + Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); + assertNotNull(schema); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java deleted file mode 100644 index 8ece5cd0fe4..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.io.IOUtils; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; - -import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals; -import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; - -public abstract class AbstractSchemaTestCase { - - protected static void assertConfigFile(String filename, String cfg) throws IOException { - IOUtils.writeFile(filename + ".actual", cfg, false); - if (! cfg.endsWith("\n")) { - IOUtils.writeFile(filename + ".actual", "\n", true); - } - assertSerializedConfigFileEquals(filename, cfg); - } - - protected static void assertConfigFiles(String expectedFile, - String cfgFile, - boolean orderMatters, - boolean updateOnAssert) throws IOException { - try { - assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile), orderMatters); - } catch (AssertionError e) { - if (updateOnAssert) { - BufferedWriter writer = IOUtils.createWriter(expectedFile, false); - writer.write(readAndCensorIndexes(cfgFile)); - writer.newLine(); - writer.flush(); - writer.close(); - System.err.println(e.getMessage() + " [not equal files: >>>"+expectedFile+"<<< and >>>"+cfgFile+"<<< in assertConfigFiles]"); - return; - } - throw new AssertionError(e.getMessage() + " [not equal files: >>>"+expectedFile+"<<< and >>>"+cfgFile+"<<< in assertConfigFiles]", e); - } - } - /** - * This is to avoid having to keep those pesky array index numbers in the config format up to date - * as new entries are added and removed. - */ - private static String readAndCensorIndexes(String file) throws IOException { - StringBuilder b = new StringBuilder(); - try (BufferedReader r = IOUtils.createReader(file)) { - int character; - boolean lastWasNewline = false; - boolean inBrackets = false; - while (-1 != (character = r.read())) { - // skip empty lines - if (character == '\n') { - if (lastWasNewline) continue; - lastWasNewline = true; - } - else { - lastWasNewline = false; - } - - // skip quoted strings - if (character == '"') { - b.appendCodePoint(character); - while (-1 != (character = r.read()) && character != '"') { - b.appendCodePoint(character); - } - } - - // skip bracket content - if (character == ']') - inBrackets = false; - if (! inBrackets) - b.appendCodePoint(character); - if (character == '[') - inBrackets = true; - } - } - return b.toString(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AnnotationReferenceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AnnotationReferenceTestCase.java deleted file mode 100644 index 25de24a4315..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AnnotationReferenceTestCase.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.DataType; -import com.yahoo.document.StructDataType; -import com.yahoo.document.Field; -import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.config.model.deploy.TestProperties; -import org.junit.Test; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author arnej - */ -public class AnnotationReferenceTestCase { - - static final String sd = - joinLines("search test {", - " document test { ", - " struct mystruct {", - " field x type int {}", - " }", - " field a type string {}", - " field b type mystruct {}", - " annotation marker {}", - " annotation person {", - " field name type string {}", - " field age type int {}", - " }", - " annotation complex {", - " field title type string {}", - " field tag type annotationreference {}", - " field owner type annotationreference {}", - " }", - " }", - "}"); - - @Test - public void noAnnotationReferenceInDocument() throws Exception { - var builder = new ApplicationBuilder(new TestProperties()); - builder.addSchema(sd); - builder.build(true); - var doc = builder.getSchema().getDocument(); - checkForAnnRef(doc); - var complex = doc.findAnnotation("complex"); - var dt = complex.getDataType(); - assertTrue(dt instanceof StructDataType); - var struct = (StructDataType)dt; - var field = struct.getField("owner"); - assertTrue(field.getDataType() instanceof AnnotationReferenceDataType); - } - - void checkForAnnRef(SDDocumentType doc) { - for (var child : doc.getTypes()) { - checkForAnnRef(child); - } - for (Field field : doc.fieldSet()) { - DataType fieldType = field.getDataType(); - assertFalse(fieldType instanceof AnnotationReferenceDataType); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java deleted file mode 100644 index 2ed1ab6892a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * tests importing of document containing array type fields - * - * @author bratseth - */ -public class ArraysTestCase extends AbstractSchemaTestCase { - - @Test - public void testArrayImporting() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arrays.sd"); - - SDField tags = (SDField) schema.getDocument().getField("tags"); - assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); - - SDField ratings = (SDField) schema.getDocument().getField("ratings"); - assertTrue(ratings.getDataType() instanceof ArrayDataType); - assertEquals(DataType.INT, ((ArrayDataType)ratings.getDataType()).getNestedType()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java deleted file mode 100644 index 9c70fcf79f7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.document.SDField; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * tests importing of document containing array type fields and weighted set type fields, new syntax. - * - * @author Einar M R Rosenvinge - */ -public class ArraysWeightedSetsTestCase extends AbstractSchemaTestCase { - @Test - public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); - - SDField tags = (SDField) schema.getDocument().getField("tags"); - assertTrue(tags.getDataType() instanceof ArrayDataType); - assertEquals(DataType.STRING, ((CollectionDataType)tags.getDataType()).getNestedType()); - - SDField ratings = (SDField) schema.getDocument().getField("ratings"); - assertTrue(ratings.getDataType() instanceof ArrayDataType); - assertEquals(DataType.INT, ((CollectionDataType)ratings.getDataType()).getNestedType()); - - SDField flags = (SDField) schema.getDocument().getField("flags"); - assertTrue(flags.getDataType() instanceof WeightedSetDataType); - assertEquals(DataType.STRING, ((CollectionDataType)flags.getDataType()).getNestedType()); - - SDField banners = (SDField) schema.getDocument().getField("banners"); - assertTrue(banners.getDataType() instanceof WeightedSetDataType); - assertEquals(DataType.INT, ((CollectionDataType)banners.getDataType()).getNestedType()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java deleted file mode 100644 index 92d72677086..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.StructDataType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.IndexingScript; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import org.junit.Test; - -import java.io.IOException; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Attribute settings - * - * @author bratseth - */ -public class AttributeSettingsTestCase extends AbstractSchemaTestCase { - - @Test - public void testAttributeSettings() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); - - SDField f1=(SDField) schema.getDocument().getField("f1"); - assertEquals(1, f1.getAttributes().size()); - Attribute a1 = f1.getAttributes().get(f1.getName()); - assertEquals(Attribute.Type.LONG, a1.getType()); - assertEquals(Attribute.CollectionType.SINGLE, a1.getCollectionType()); - assertTrue(a1.isHuge()); - assertFalse(a1.isFastSearch()); - assertFalse(a1.isFastAccess()); - assertFalse(a1.isRemoveIfZero()); - assertFalse(a1.isCreateIfNonExistent()); - - SDField f2=(SDField) schema.getDocument().getField("f2"); - assertEquals(1, f2.getAttributes().size()); - Attribute a2 = f2.getAttributes().get(f2.getName()); - assertEquals(Attribute.Type.LONG, a2.getType()); - assertEquals(Attribute.CollectionType.SINGLE, a2.getCollectionType()); - assertFalse(a2.isHuge()); - assertTrue(a2.isFastSearch()); - assertFalse(a2.isFastAccess()); - assertFalse(a2.isRemoveIfZero()); - assertFalse(a2.isCreateIfNonExistent()); - assertEquals("f2", f2.getAliasToName().get("f2alias")); - SDField f3=(SDField) schema.getDocument().getField("f3"); - assertEquals(1, f3.getAttributes().size()); - assertEquals("f3", f3.getAliasToName().get("f3alias")); - - Attribute a3 = f3.getAttributes().get(f3.getName()); - assertEquals(Attribute.Type.LONG, a3.getType()); - assertEquals(Attribute.CollectionType.SINGLE, a3.getCollectionType()); - assertFalse(a3.isHuge()); - assertFalse(a3.isFastSearch()); - assertFalse(a3.isFastAccess()); - assertFalse(a3.isRemoveIfZero()); - assertFalse(a3.isCreateIfNonExistent()); - - assertWeightedSet(schema, "f4", true, true); - assertWeightedSet(schema, "f5", true, true); - assertWeightedSet(schema, "f6", true, true); - assertWeightedSet(schema, "f7", true, false); - assertWeightedSet(schema, "f8", true, false); - assertWeightedSet(schema, "f9", false, true); - assertWeightedSet(schema, "f10", false, true); - - assertAttrSettings(schema, "f4", false, false, false); - assertAttrSettings(schema, "f5", true, true, true); - assertAttrSettings(schema, "f6", false, false, false); - assertAttrSettings(schema, "f7", false, false, false); - assertAttrSettings(schema, "f8", false, false, false); - assertAttrSettings(schema, "f9", false, false, false); - assertAttrSettings(schema, "f10", false, false, false); - } - - private void assertWeightedSet(Schema schema, String name, boolean createIfNonExistent, boolean removeIfZero) { - SDField f4 = (SDField) schema.getDocument().getField(name); - assertEquals(1, f4.getAttributes().size()); - Attribute a4 = f4.getAttributes().get(f4.getName()); - assertEquals(Attribute.Type.STRING, a4.getType()); - assertEquals(Attribute.CollectionType.WEIGHTEDSET, a4.getCollectionType()); - assertEquals(a4.isRemoveIfZero(), removeIfZero); - assertEquals(a4.isCreateIfNonExistent(), createIfNonExistent); - } - - private void assertAttrSettings(Schema schema, String name, boolean fastAccess, boolean fastSearch, boolean paged) { - SDField f4 = (SDField) schema.getDocument().getField(name); - assertEquals(1, f4.getAttributes().size()); - Attribute a4 = f4.getAttributes().get(f4.getName()); - assertEquals(Attribute.Type.STRING, a4.getType()); - assertEquals(Attribute.CollectionType.WEIGHTEDSET, a4.getCollectionType()); - assertEquals(a4.isFastSearch(), fastSearch); - assertEquals(a4.isFastAccess(), fastAccess); - assertEquals(a4.isPaged(), paged); - } - - @Test - public void requireThatFastAccessCanBeSet() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesettings.sd"); - SDField field = (SDField) schema.getDocument().getField("fast_access"); - assertEquals(1, field.getAttributes().size()); - Attribute attr = field.getAttributes().get(field.getName()); - assertTrue(attr.isFastAccess()); - } - - private Schema getSchema(String sd) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(sd); - builder.build(true); - return builder.getSchema(); - } - - private Attribute getAttributeF(String sd) throws ParseException { - Schema schema = getSchema(sd); - SDField field = (SDField) schema.getDocument().getField("f"); - return field.getAttributes().get(field.getName()); - } - - @Test - public void requireThatPagedIsDefaultOff() throws ParseException { - Attribute attr = getAttributeF( - "search test {\n" + - " document test { \n" + - " field f type tensor(x[2]) { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - "}\n"); - assertFalse(attr.isPaged()); - } - @Test - public void requireThatPagedCanBeSet() throws ParseException { - Attribute attr = getAttributeF( - "search test {\n" + - " document test { \n" + - " field f type tensor(x[2]) { \n" + - " indexing: attribute \n" + - " attribute: paged \n" + - " }\n" + - " }\n" + - "}\n"); - assertTrue(attr.isPaged()); - } - - @Test - public void requireThatMutableIsDefaultOff() throws ParseException { - Attribute attr = getAttributeF( - "search test {\n" + - " document test { \n" + - " field f type int { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - "}\n"); - assertFalse(attr.isMutable()); - } - - @Test - public void requireThatMutableCanNotbeSetInDocument() throws ParseException { - try { - getSchema("search test {\n" + - " document test {\n" + - " field f type int {\n" + - " indexing: attribute\n" + - " attribute: mutable\n" + - " }\n" + - " }\n" + - "}\n"); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("Field 'f' in 'test' can not be marked mutable as it is inside the document clause.", e.getMessage()); - } - } - - @Test - public void requireThatMutableExtraFieldCanBeSet() throws ParseException { - Attribute attr = getAttributeF( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " field f type long {\n" + - " indexing: 0 | to_long | attribute\n" + - " attribute: mutable\n" + - " }\n" + - "}\n"); - assertTrue(attr.isMutable()); - } - - private Schema getSearchWithMutables() throws ParseException { - return getSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " field m type long {\n" + - " indexing: attribute\n" + - " attribute: mutable\n" + - " }\n" + - " field f type long {\n" + - " indexing: 0 | to_long | attribute\n" + - " }\n" + - "}\n"); - } - - @Test - public void requireThatMutableConfigIsProperlyPropagated() throws ParseException { - AttributeFields attributes = new AttributeFields(getSearchWithMutables()); - AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, true); - AttributesConfig cfg = builder.build(); - assertEquals("a", cfg.attribute().get(0).name()); - assertFalse(cfg.attribute().get(0).ismutable()); - - assertEquals("f", cfg.attribute().get(1).name()); - assertFalse(cfg.attribute().get(1).ismutable()); - - assertEquals("m", cfg.attribute().get(2).name()); - assertTrue(cfg.attribute().get(2).ismutable()); - } - - @Test - public void requireMaxUnCommittedMemoryIsProperlyPropagated() throws ParseException { - AttributeFields attributes = new AttributeFields(getSearchWithMutables()); - AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, true); - AttributesConfig cfg = builder.build(); - assertEquals("a", cfg.attribute().get(0).name()); - assertEquals(13333, cfg.attribute().get(0).maxuncommittedmemory()); - - assertEquals("f", cfg.attribute().get(1).name()); - assertEquals(13333, cfg.attribute().get(1).maxuncommittedmemory()); - - assertEquals("m", cfg.attribute().get(2).name()); - assertEquals(13333, cfg.attribute().get(2).maxuncommittedmemory()); - } - - private void verifyEnableBitVectorDefault(Schema schema, boolean enableBitVectors) { - AttributeFields attributes = new AttributeFields(schema); - AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 13333, enableBitVectors); - AttributesConfig cfg = builder.build(); - assertEquals("a", cfg.attribute().get(0).name()); - assertEquals(enableBitVectors, cfg.attribute().get(0).enablebitvectors()); - - assertEquals("b", cfg.attribute().get(1).name()); - assertFalse(cfg.attribute().get(1).enablebitvectors()); - } - - @Test - public void requireEnableBitVectorsIsProperlyPropagated() throws ParseException { - Schema schema = getSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - " field b type int { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - "}\n"); - verifyEnableBitVectorDefault(schema, false); - verifyEnableBitVectorDefault(schema, true); - } - - @Test - public void requireThatMutableIsAllowedThroughIndexing() throws ParseException { - IndexingScript script = new IndexingScript(getSearchWithMutables()); - IlscriptsConfig.Builder builder = new IlscriptsConfig.Builder(); - script.getConfig(builder); - IlscriptsConfig cfg = builder.build(); - assertEquals(1, cfg.ilscript().size()); - IlscriptsConfig.Ilscript ils = cfg.ilscript(0); - assertEquals("test", ils.doctype()); - assertEquals(2, ils.docfield().size()); - assertEquals("a", ils.docfield(0)); - assertEquals("m", ils.docfield(1)); - - } - - @Test - public void attribute_convert_to_array_copies_internal_state() { - StructDataType refType = new StructDataType("my_struct"); - Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE, - Optional.of(TensorType.fromSpec("tensor(x{})")), Optional.of(refType)); - single.setRemoveIfZero(true); - single.setCreateIfNonExistent(true); - single.setPrefetch(Boolean.TRUE); - single.setEnableBitVectors(true); - single.setEnableOnlyBitVector(true); - single.setFastSearch(true); - single.setHuge(true); - single.setPaged(true); - single.setFastAccess(true); - single.setPosition(true); - single.setArity(5); - single.setLowerBound(7); - single.setUpperBound(11); - single.setDensePostingListThreshold(13.3); - single.getSorting().setAscending(); - single.getAliases().add("foo"); - - Attribute array = single.convertToArray(); - assertEquals("foo", array.getName()); - assertEquals(Attribute.Type.STRING, array.getType()); - assertEquals(Attribute.CollectionType.ARRAY, array.getCollectionType()); - assertEquals(Optional.of(TensorType.fromSpec("tensor(x{})")), array.tensorType()); - assertSame(single.referenceDocumentType(), array.referenceDocumentType()); - assertTrue(array.isRemoveIfZero()); - assertTrue(array.isCreateIfNonExistent()); - assertTrue(array.isPrefetch()); - assertTrue(array.isEnabledBitVectors()); - assertTrue(array.isEnabledOnlyBitVector()); - assertTrue(array.isFastSearch()); - assertTrue(array.isHuge()); - assertTrue(array.isPaged()); - assertTrue(array.isFastAccess()); - assertTrue(array.isPosition()); - assertEquals(5, array.arity()); - assertEquals(7, array.lowerBound()); - assertEquals(11, array.upperBound()); - assertEquals(13.3, array.densePostingListThreshold(), 0.00001); - assertSame(single.getSorting(), array.getSorting()); - assertSame(single.getAliases(), array.getAliases()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java deleted file mode 100644 index 5d91d7df3ca..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDField; - -/** - * Convenience class for tests that need to set attribute properties on fields. - */ -public class AttributeUtils { - - public static void addAttributeAspect(SDField field) { - field.parseIndexingScript("{ attribute }"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java deleted file mode 100644 index 3e92c1df9e7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; - -/** - * Tests comment handling - * - * @author bratseth - */ -public class CommentTestCase extends AbstractSchemaTestCase { - - @Test - public void testComments() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/comment.sd"); - SDField field = schema.getConcreteField("a"); - assertEquals("{ input a | tokenize normalize stem:\"BEST\" | summary a | index a; }", - field.getIndexingScript().toString()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java deleted file mode 100644 index ca7313a5c3f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; -import static org.junit.Assert.fail; - -import static org.junit.Assert.assertEquals; - -/** - * @author baldersheim - */ -public class DiversityTestCase { - @Test - public void testDiversity() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - " field b type int {\n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " cutoff-factor: 17.3\n" + - " cutoff-strategy: strict" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " max-filter-coverage: 0.065" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); - RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); - assertEquals("b", diversity.getAttribute()); - assertEquals(74, diversity.getMinGroups()); - assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); - assertEquals(Diversity.CutoffStrategy.strict, diversity.getCutoffStrategy()); - assertEquals(120, matchPhase.getMaxHits()); - assertEquals("a", matchPhase.getAttribute()); - assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); - } - - private static String getMessagePrefix() { - return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; - } - @Test - public void requireSingleNumericOrString() throws ParseException { - ApplicationBuilder builder = getSearchBuilder("field b type predicate { indexing: attribute }"); - - try { - builder.build(true); - fail("Should throw."); - } catch (IllegalArgumentException e) { - assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'predicate'", e.getMessage()); - } - } - - @Test - public void requireSingle() throws ParseException { - ApplicationBuilder builder = getSearchBuilder("field b type array { indexing: attribute }"); - - try { - builder.build(true); - fail("Should throw."); - } catch (IllegalArgumentException e) { - assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array'", e.getMessage()); - } - } - private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - diversity + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " }\n" + - " }\n" + - "}\n"); - return builder; - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java deleted file mode 100644 index 5bbb751585b..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.toList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bjorncs - */ -public class DocumentGraphValidatorTest { - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void simple_ref_dag_is_allowed() { - Schema advertiserSchema = createSearchWithName("advertiser"); - Schema campaignSchema = createSearchWithName("campaign"); - Schema adSchema = createSearchWithName("ad"); - createDocumentReference(adSchema, advertiserSchema, "advertiser_ref"); - createDocumentReference(adSchema, campaignSchema, "campaign_ref"); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - validator.validateDocumentGraph(documentListOf(advertiserSchema, campaignSchema, adSchema)); - } - - @Test - public void simple_inheritance_dag_is_allowed() { - Schema grandfather = createSearchWithName("grandfather"); - Schema father = createSearchWithName("father", grandfather); - Schema son = createSearchWithName("son", father); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - validator.validateDocumentGraph(documentListOf(son, father, grandfather)); - } - - @Test - public void complex_dag_is_allowed() { - Schema grandfather = createSearchWithName("grandfather"); - Schema father = createSearchWithName("father", grandfather); - Schema mother = createSearchWithName("mother", grandfather); - createDocumentReference(father, mother, "wife_ref"); - Schema son = createSearchWithName("son", father, mother); - Schema daughter = createSearchWithName("daughter", father, mother); - createDocumentReference(daughter, son, "brother_ref"); - - Schema randomGuy1 = createSearchWithName("randomguy1"); - Schema randomGuy2 = createSearchWithName("randomguy2"); - createDocumentReference(randomGuy1, mother, "secret_ref"); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - validator.validateDocumentGraph(documentListOf(son, father, grandfather, son, daughter, randomGuy1, randomGuy2)); - } - - @Test - public void ref_cycle_is_forbidden() { - Schema schema1 = createSearchWithName("doc1"); - Schema schema2 = createSearchWithName("doc2"); - Schema schema3 = createSearchWithName("doc3"); - createDocumentReference(schema1, schema2, "ref_2"); - createDocumentReference(schema2, schema3, "ref_3"); - createDocumentReference(schema3, schema1, "ref_1"); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); - exceptionRule.expectMessage("Document dependency cycle detected: doc1->doc2->doc3->doc1."); - validator.validateDocumentGraph(documentListOf(schema1, schema2, schema3)); - } - - @Test - public void inherit_cycle_is_forbidden() { - Schema schema1 = createSearchWithName("doc1"); - Schema schema2 = createSearchWithName("doc2", schema1); - Schema schema3 = createSearchWithName("doc3", schema2); - schema1.getDocument().inherit(schema3.getDocument()); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); - exceptionRule.expectMessage("Document dependency cycle detected: doc1->doc3->doc2->doc1."); - validator.validateDocumentGraph(documentListOf(schema1, schema2, schema3)); - } - - @Test - public void combined_inherit_and_ref_cycle_is_forbidden() { - Schema schema1 = createSearchWithName("doc1"); - Schema schema2 = createSearchWithName("doc2", schema1); - Schema schema3 = createSearchWithName("doc3", schema2); - createDocumentReference(schema1, schema3, "ref_1"); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); - exceptionRule.expectMessage("Document dependency cycle detected: doc1->doc3->doc2->doc1."); - validator.validateDocumentGraph(documentListOf(schema1, schema2, schema3)); - } - - @Test - public void self_reference_is_forbidden() { - Schema adSchema = createSearchWithName("ad"); - createDocumentReference(adSchema, adSchema, "ad_ref"); - - DocumentGraphValidator validator = new DocumentGraphValidator(); - exceptionRule.expect(DocumentGraphValidator.DocumentGraphException.class); - exceptionRule.expectMessage("Document dependency cycle detected: ad->ad."); - validator.validateDocumentGraph(documentListOf(adSchema)); - } - - /** - * Self inheritance is checked early because it is possible, and because it otherwise - * produces a stack overflow before getting to graph validation. - */ - @Test - public void self_inheritance_forbidden() { - try { - Schema adSchema = createSearchWithName("ad"); - SDDocumentType document = adSchema.getDocument(); - document.inherit(document); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Document type 'ad' cannot inherit itself", e.getMessage()); - } - } - - private static List documentListOf(Schema... schemas) { - return Arrays.stream(schemas).map(Schema::getDocument).collect(toList()); - } - - private static Schema createSearchWithName(String name, Schema... parents) { - Schema campaignSchema = new Schema(name, MockApplicationPackage.createEmpty()); - SDDocumentType document = new SDDocumentType(name); - campaignSchema.addDocument(document); - document.setDocumentReferences(new DocumentReferences(Collections.emptyMap())); - Arrays.stream(parents) - .map(Schema::getDocument) - .forEach(document::inherit); - return campaignSchema; - } - - @SuppressWarnings("deprecation") - private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDDocumentType fromDocument = from.getDocument(); - SDField refField = new TemporarySDField(fromDocument, refFieldName, NewDocumentReferenceDataType.forDocumentName(to.getName())); - fromDocument.addField(refField); - Map originalMap = fromDocument.getDocumentReferences().get().referenceMap(); - HashMap modifiedMap = new HashMap<>(originalMap); - modifiedMap.put(refFieldName, new DocumentReference(refField, to)); - fromDocument.setDocumentReferences(new DocumentReferences(modifiedMap)); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java deleted file mode 100644 index fe1b19be64b..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.Map; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -/** - * @author bjorncs - */ -public class DocumentReferenceResolverTest { - - private static final String BAR = "bar"; - private static final String FOO = "foo"; - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void reference_from_one_document_to_another_is_resolved() { - // Create bar document with no fields - Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); - SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); - barSchema.addDocument(barDocument); - - // Create foo document with document reference to bar and add another field - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); - SDField fooRefToBarField = new SDField - (fooDocument, "bar_ref", new NewDocumentReferenceDataType(barDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToBarField); - SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); - fooDocument.addField(fooRefToBarField); - fooDocument.addField(irrelevantField); - fooSchema.addDocument(fooDocument); - - DocumentReferenceResolver resolver = new DocumentReferenceResolver(asList(fooSchema, barSchema)); - resolver.resolveReferences(fooDocument); - assertTrue(fooDocument.getDocumentReferences().isPresent()); - - Map fooReferenceMap = fooDocument.getDocumentReferences().get().referenceMap(); - assertEquals(1, fooReferenceMap.size()); - assertSame(barSchema, fooReferenceMap.get("bar_ref").targetSearch()); - assertSame(fooRefToBarField, fooReferenceMap.get("bar_ref").referenceField()); - } - - @SuppressWarnings("deprecation") - @Test - public void throws_user_friendly_exception_if_referenced_document_does_not_exist() { - // Create foo document with document reference to non-existing document bar - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); - SDField fooRefToBarField = new SDField( - fooDocument, - "bar_ref", NewDocumentReferenceDataType.forDocumentName("bar")); - AttributeUtils.addAttributeAspect(fooRefToBarField); - fooDocument.addField(fooRefToBarField); - fooSchema.addDocument(fooDocument); - - DocumentReferenceResolver resolver = new DocumentReferenceResolver(singletonList(fooSchema)); - - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "Invalid document reference 'bar_ref': Could not find document type 'bar'"); - resolver.resolveReferences(fooDocument); - } - - @Test - public void throws_exception_if_reference_is_not_an_attribute() { - // Create bar document with no fields - Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); - SDDocumentType barDocument = new SDDocumentType("bar", barSchema); - barSchema.addDocument(barDocument); - - // Create foo document with document reference to bar - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); - SDField fooRefToBarField = new SDField - (fooDocument, "bar_ref", new NewDocumentReferenceDataType(barDocument.getDocumentType())); - fooDocument.addField(fooRefToBarField); - fooSchema.addDocument(fooDocument); - - DocumentReferenceResolver resolver = new DocumentReferenceResolver(asList(fooSchema, barSchema)); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "The field 'bar_ref' is an invalid document reference. The field must be an attribute."); - resolver.resolveReferences(fooDocument); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java deleted file mode 100644 index 3494ab780b2..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import org.junit.Ignore; -import org.junit.Test; - -import java.util.function.Function; -import java.util.regex.Pattern; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Tests rank feature names. - * - * @author bratseth - */ -public class FeatureNamesTestCase { - - @Test - public void testArgument() { - assertFalse(FeatureNames.argumentOf("foo(bar)").isPresent()); - assertFalse(FeatureNames.argumentOf("foo(bar.baz)").isPresent()); - assertEquals("bar", FeatureNames.argumentOf("query(bar)").get()); - assertEquals("bar.baz", FeatureNames.argumentOf("query(bar.baz)").get()); - assertEquals("bar", FeatureNames.argumentOf("attribute(bar)").get()); - assertEquals("bar.baz", FeatureNames.argumentOf("attribute(bar.baz)").get()); - assertEquals("bar", FeatureNames.argumentOf("constant(bar)").get()); - assertEquals("bar.baz", FeatureNames.argumentOf("constant(bar.baz)").get()); - } - - @Test - public void testConstantFeature() { - assertEquals("constant(foo)", - FeatureNames.asConstantFeature("foo").toString()); - } - - @Test - public void testAttributeFeature() { - assertEquals("attribute(foo)", - FeatureNames.asAttributeFeature("foo").toString()); - } - - @Test - public void testQueryFeature() { - assertEquals("query(\"foo.bar\")", - FeatureNames.asQueryFeature("foo.bar").toString()); - } - - @Test - public void testLegalFeatureNames() { - assertTrue(FeatureNames.notNeedQuotes("_")); - assertFalse(FeatureNames.notNeedQuotes("-")); - assertTrue(FeatureNames.notNeedQuotes("_-")); - assertTrue(FeatureNames.notNeedQuotes("0_-azAZxy98-_")); - assertFalse(FeatureNames.notNeedQuotes("0_-azAZxy98-_+")); - } - - @Test - @Ignore - /* - * Unignore to verify performance - * 2021/09/05 performance was a factor of 5.25 - * 'Identifier handcoded validity check took 4301ms - * Identifier regexp validity check took 22609ms' - */ - public void benchMarkPatternMatching() { - Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); - String[] strings = new String[1000]; - for (int i = 0; i < strings.length; i++) { - strings[i] = i + "-legal_string" + i; - } - - countValid(strings, 1000, "handcoded warmup", FeatureNames::notNeedQuotes); - countValid(strings, 1000, "regexp warmup", (s) -> identifierRegexp.matcher(s).matches()); - - countValid(strings, 100000, "handcoded", FeatureNames::notNeedQuotes); - countValid(strings, 100000, "regexp", (s) -> identifierRegexp.matcher(s).matches()); - } - - private void countValid(String [] strings, int numReps, String text, Function func) { - long start = System.nanoTime(); - int validCount = 0; - for (int i = 0; i < numReps; i++) { - for (String s : strings) { - if (func.apply(s)) validCount++; - } - } - long end = System.nanoTime(); - assertEquals(strings.length * numReps, validCount); - System.out.println("Identifier " + text + " validity check took " + (end - start)/1000000 + "ms"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java deleted file mode 100644 index fdd7fe95c45..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentTypeManagerConfigurer; -import com.yahoo.document.Field; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.derived.Deriver; -import org.junit.Test; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; - -/** - * @author Einar M R Rosenvinge - */ -public class FieldOfTypeDocumentTestCase extends AbstractSchemaTestCase { - - @Test - public void testDocument() throws IOException { - - List sds = new ArrayList<>(); - sds.add("src/test/examples/music.sd"); - sds.add("src/test/examples/fieldoftypedocument.sd"); - DocumentmanagerConfig.Builder value = Deriver.getDocumentManagerConfig(sds); - assertConfigFile("src/test/examples/fieldoftypedocument.cfg", - new DocumentmanagerConfig(value).toString() + "\n"); - - DocumentTypeManager manager = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(manager, "raw:" + new DocumentmanagerConfig(value).toString()); - - - DocumentType musicType = manager.getDocumentType("music"); - assertEquals(3, musicType.getFieldCount()); - - Field intField = musicType.getField("intfield"); - assertEquals(DataType.INT, intField.getDataType()); - Field stringField = musicType.getField("stringfield"); - assertEquals(DataType.STRING, stringField.getDataType()); - Field longField = musicType.getField("longfield"); - assertEquals(DataType.LONG, longField.getDataType()); - - - DocumentType bookType = manager.getDocumentType("book"); - assertEquals(1, bookType.getFieldCount()); - - Field musicField = bookType.getField("soundtrack"); - assertSame(musicType, musicField.getDataType()); - } - -} - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java deleted file mode 100644 index f486126ba4f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; -import org.junit.Test; - -import java.util.HashSet; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class ImportedFieldsEnumeratorTest { - - @Test - public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { - String PARENT = "parent"; - Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); - SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); - var parentField = new SDField(parentDocument, "their_field", DataType.INT); - AttributeUtils.addAttributeAspect(parentField); - parentDocument.addField(parentField); - parentSchema.addDocument(parentDocument); - - String FOO = "foo"; - Schema fooSchema = new Schema(FOO, MockApplicationPackage.createEmpty()); - /* - SDField fooRefToParent = new SDField( - "foo_ref", NewDocumentReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToParent); - */ - var fooImports = fooSchema.temporaryImportedFields().get(); - fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); - fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); - SDDocumentType fooDocument = new SDDocumentType(FOO, fooSchema); - fooSchema.addDocument(fooDocument); - - String BAR = "bar"; - Schema barSchema = new Schema(BAR, MockApplicationPackage.createEmpty()); - /* - SDField barRefToParent = new SDField( - "bar_ref", NewDocumentReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(barRefToParent); - */ - var barImports = barSchema.temporaryImportedFields().get(); - barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); - SDDocumentType barDocument = new SDDocumentType(BAR, barSchema); - barSchema.addDocument(barDocument); - - var enumerator = new ImportedFieldsEnumerator(List.of(parentSchema, fooSchema, barSchema)); - - enumerator.enumerateImportedFields(parentDocument); - assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent - - enumerator.enumerateImportedFields(fooDocument); - assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); - - enumerator.enumerateImportedFields(barDocument); - assertImportedFieldsAre(barDocument, List.of("my_cool_import")); - } - - private void assertImportedFieldsAre(SDDocumentType documentType, List expectedNames) { - assertNotNull(documentType.getTemporaryImportedFields()); - var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); - var expectedNameSet = new HashSet<>(expectedNames); - assertEquals(expectedNameSet, actualNames); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java deleted file mode 100644 index 23be59e38f5..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class IncorrectRankingExpressionFileRefTestCase extends AbstractSchemaTestCase { - - @Test - public void testIncorrectRef() throws IOException, ParseException { - try { - RankProfileRegistry registry = new RankProfileRegistry(); - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", - registry, - new QueryProfileRegistry()); - new DerivedConfiguration(schema, registry); // cause rank profile parsing - fail("parsing should have failed"); - } catch (IllegalArgumentException e) { - String message = Exceptions.toMessageString(e); - assertTrue(message.contains("Could not read ranking expression file")); - assertTrue(message.contains("wrongending.expr.expression")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java deleted file mode 100644 index d81d07bf6e2..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -/** - * Tests importing a search definition with conflicting summary types - * - * @author bratseth - */ -public class IncorrectSummaryTypesTestCase extends AbstractSchemaTestCase { - @Test - public void testImportingIncorrect() throws ParseException { - try { - ApplicationBuilder.createFromString( - "search incorrectsummarytypes {\n" + - " document incorrectsummarytypes {\n" + - " field somestring type string {\n" + - " indexing: summary\n" + - " }\n" + - " }\n" + - " document-summary incorrect {\n" + - " summary somestring type int {\n" + - " }\n" + - " }\n" + - "}\n"); - fail("processing should have failed"); - } catch (RuntimeException e) { - assertEquals("'summary somestring type string' in 'destinations(default )' is inconsistent with 'summary somestring type int' in 'destinations(incorrect )': All declarations of the same summary field must have the same type", e.getMessage()); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java deleted file mode 100644 index 198f8b6c79e..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Rank settings - * - * @author bratseth - */ -public class IndexSettingsTestCase extends AbstractSchemaTestCase { - - @Test - public void testStemmingSettings() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/indexsettings.sd"); - - SDField usingDefault=(SDField) schema.getDocument().getField("usingdefault"); - assertEquals(Stemming.SHORTEST,usingDefault.getStemming(schema)); - - SDField notStemmed=(SDField) schema.getDocument().getField("notstemmed"); - assertEquals(Stemming.NONE,notStemmed.getStemming(schema)); - - SDField allStemmed=(SDField) schema.getDocument().getField("allstemmed"); - assertEquals(Stemming.SHORTEST,allStemmed.getStemming(schema)); - - SDField multiStemmed=(SDField) schema.getDocument().getField("multiplestems"); - assertEquals(Stemming.MULTIPLE, multiStemmed.getStemming(schema)); - } - - @Test - public void requireThatInterlavedFeaturesAreSetOnExtraField() throws ParseException { - ApplicationBuilder builder = ApplicationBuilder.createFromString(joinLines( - "search test {", - " document test {", - " field content type string {", - " indexing: index | summary", - " index: enable-bm25", - " }", - " }", - " field extra type string {", - " indexing: input content | index | summary", - " index: enable-bm25", - " }", - "}" - )); - Schema schema = builder.getSchema(); - Index contentIndex = schema.getIndex("content"); - assertTrue(contentIndex.useInterleavedFeatures()); - Index extraIndex = schema.getIndex("extra"); - assertTrue(extraIndex.useInterleavedFeatures()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java deleted file mode 100644 index 8ba7dfdb14f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; - -/** - * Tests that indexing statements are parsed correctly. - * - * @author frodelu - */ -public class IndexingParsingTestCase extends AbstractSchemaTestCase { - - @Test - public void requireThatIndexingExpressionsCanBeParsed() throws Exception { - assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/indexing.sd")); - } - - @Test - public void requireThatParseExceptionPositionIsCorrect() throws Exception { - try { - ApplicationBuilder.buildFromFile("src/test/examples/indexing_invalid_expression.sd"); - } catch (ParseException e) { - if (!e.getMessage().contains("at line 5, column 57.")) { - throw e; - } - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java deleted file mode 100644 index 317b860d7d2..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * tests importing of document containing array type fields - * - * @author bratseth - */ -public class MultipleSummariesTestCase extends AbstractSchemaTestCase { - - @Test - public void testArrayImporting() throws IOException, ParseException { - var builder = new ApplicationBuilder(new TestProperties()); - builder.addSchemaFile("src/test/examples/multiplesummaries.sd"); - builder.build(true); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java deleted file mode 100644 index 1e766b7793d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Tests that "name" is not allowed as name for a field. - * - * And that duplicate names are not allowed. - * - * @author Lars Christian Jensen - */ -public class NameFieldCheckTestCase extends AbstractSchemaTestCase { - - @Test - public void testNameField() { - try { - ApplicationBuilder.createFromString( - "search simple {\n" + - " document name-check {\n" + - " field title type string {\n" + - " indexing: summary | index\n" + - " }\n" + - " # reserved name, should trigger error\n" + - " field sddocname type string {\n" + - " indexing: index\n" + - " }\n" + - " }\n" + - "}"); - fail("Should throw exception."); - } catch (Exception expected) { - // Success - } - } - - @Test - public void testDuplicateNamesInSearchDifferentType() { - try { - ApplicationBuilder.createFromString( - "search duplicatenamesinsearch {\n" + - " document {\n" + - " field grpphotoids64 type string { }\n" + - " }\n" + - " field grpphotoids64 type array {\n" + - " indexing: input grpphotoids64 | split \" \" | for_each {\n" + - " base64decode } | attribute\n" + - " }\n" + - "}"); - fail("Should throw exception."); - } catch (Exception e) { - assertEquals("For schema 'duplicatenamesinsearch', field 'grpphotoids64': " + - "Incompatible types. Expected Array for index field 'grpphotoids64', got string.", e.getMessage()); - } - } - - @Test - public void testDuplicateNamesInDoc() { - try { - ApplicationBuilder.createFromString( - "search duplicatenamesindoc {\n" + - " document {\n" + - " field foo type int {\n" + - " indexing: attribute\n" + - " }\n" + - " field fOo type string {\n" + - " indexing: index\n" + - " }\n" + - " }\n" + - "}"); - fail("Should throw exception."); - } catch (Exception e) { - assertTrue(e.getMessage().matches(".*Duplicate.*")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java deleted file mode 100644 index 6563fb8d3f1..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -/** - * Tests settings outside the document - * - * @author bratseth - */ -public class OutsideTestCase extends AbstractSchemaTestCase { - - @Test - public void testOutsideIndex() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/outsidedoc.sd"); - - Index defaultIndex= schema.getIndex("default"); - assertTrue(defaultIndex.isPrefix()); - assertEquals("default.default",defaultIndex.aliasIterator().next()); - } - - @Test - public void testOutsideSummary() throws IOException, ParseException { - ApplicationBuilder.buildFromFile("src/test/examples/outsidesummary.sd"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java deleted file mode 100644 index 2b49f3d67a6..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.ImmutableSDField; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.parser.ParseException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author Lester Solbakken - */ - -public class PredicateDataTypeTestCase { - - private String searchSd(String field) { - return "search p {\n document p {\n" + field + "}\n}\n"; - } - - private String predicateFieldSd(String index) { - return "field pf type predicate {\n" + index + "}\n"; - } - - private String arrayPredicateFieldSd(String index) { - return "field apf type array {\n" + index + "}\n"; - } - - private String stringFieldSd(String index) { - return "field sf type string {\n" + index + "}\n"; - } - - private String attributeFieldSd(String terms) { - return "indexing: attribute\n index {\n" + terms + "}\n"; - } - - private String arityParameter(int arity) { - return "arity: " + arity + "\n"; - } - - private String lowerBoundParameter(long bound) { - return "lower-bound: " + bound + "\n"; - } - - private String upperBoundParameter(long bound) { - return "upper-bound: " + bound + "\n"; - } - - @SuppressWarnings("deprecation") - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void requireThatBuilderSetsIndexParametersCorrectly() throws ParseException { - int arity = 2; - long lowerBound = -100; - long upperBound = 100; - String sd = searchSd( - predicateFieldSd( - attributeFieldSd( - arityParameter(arity) + - lowerBoundParameter(lowerBound) + - upperBoundParameter(upperBound)))); - - ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); - for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { - if (field.getDataType() == DataType.PREDICATE) { - for (Index index : field.getIndices().values()) { - assertTrue(index.getBooleanIndexDefiniton().hasArity()); - assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); - assertTrue(index.getBooleanIndexDefiniton().hasLowerBound()); - assertEquals(lowerBound, index.getBooleanIndexDefiniton().getLowerBound()); - assertTrue(index.getBooleanIndexDefiniton().hasUpperBound()); - assertEquals(upperBound, index.getBooleanIndexDefiniton().getUpperBound()); - } - } - } - } - - @Test - public void requireThatBuilderHandlesLongValues() throws ParseException { - int arity = 2; - long lowerBound = -100000000000000000L; - long upperBound = 1000000000000000000L; - String sd = searchSd( - predicateFieldSd( - attributeFieldSd( - arityParameter(arity) + - "lower-bound: -100000000000000000L\n" + // +'L' - upperBoundParameter(upperBound)))); - - ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); - for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { - if (field.getDataType() == DataType.PREDICATE) { - for (Index index : field.getIndices().values()) { - assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); - assertEquals(lowerBound, index.getBooleanIndexDefiniton().getLowerBound()); - assertEquals(upperBound, index.getBooleanIndexDefiniton().getUpperBound()); - } - } - } - } - - @Test - public void requireThatBuilderHandlesMissingParameters() throws ParseException { - String sd = searchSd( - predicateFieldSd( - attributeFieldSd( - arityParameter(2)))); - ApplicationBuilder sb = ApplicationBuilder.createFromString(sd); - for (ImmutableSDField field : sb.getSchema().allConcreteFields()) { - if (field.getDataType() == DataType.PREDICATE) { - for (Index index : field.getIndices().values()) { - assertTrue(index.getBooleanIndexDefiniton().hasArity()); - assertFalse(index.getBooleanIndexDefiniton().hasLowerBound()); - assertFalse(index.getBooleanIndexDefiniton().hasUpperBound()); - } - } - } - } - - @Test - public void requireThatBuilderFailsIfNoArityValue() throws ParseException { - String sd = searchSd(predicateFieldSd(attributeFieldSd(""))); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Missing arity value in predicate field."); - ApplicationBuilder.createFromString(sd); - fail(); - } - - @Test - public void requireThatBuilderFailsIfBothIndexAndAttribute() throws ParseException { - String sd = searchSd(predicateFieldSd("indexing: summary | index | attribute\nindex { arity: 2 }")); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - ApplicationBuilder.createFromString(sd); - } - - @Test - public void requireThatBuilderFailsIfIndex() throws ParseException { - String sd = searchSd(predicateFieldSd("indexing: summary | index \nindex { arity: 2 }")); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'p', field 'pf': Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); - ApplicationBuilder.createFromString(sd); - } - - - @Test - public void requireThatBuilderFailsIfIllegalArityValue() throws ParseException { - String sd = searchSd(predicateFieldSd(attributeFieldSd(arityParameter(0)))); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Invalid arity value in predicate field, must be greater than 1."); - ApplicationBuilder.createFromString(sd); - } - - @Test - public void requireThatBuilderFailsIfArityParameterExistButNotPredicateField() throws ParseException { - String sd = searchSd(stringFieldSd(attributeFieldSd(arityParameter(2)))); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Arity parameter is used only for predicate type fields."); - ApplicationBuilder.createFromString(sd); - } - - @Test - public void requireThatBuilderFailsIfBoundParametersExistButNotPredicateField() throws ParseException { - String sd = searchSd( - stringFieldSd( - attributeFieldSd( - lowerBoundParameter(100) + upperBoundParameter(1000)))); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Parameters lower-bound and upper-bound are used only for predicate type fields."); - ApplicationBuilder.createFromString(sd); - } - - @Test - public void requireThatArrayOfPredicateFails() throws ParseException { - String sd = searchSd( - arrayPredicateFieldSd( - attributeFieldSd( - arityParameter(1)))); - - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Collections of predicates are not allowed."); - ApplicationBuilder.createFromString(sd); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java deleted file mode 100644 index cbbb53d8ec8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.model.test.TestDriver; -import com.yahoo.config.model.test.TestRoot; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import org.junit.Test; - -import java.io.File; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -/** - * @author Ulf Lilleengen - */ -public class RankProfileRegistryTest { - - private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; - - @Test - public void testRankProfileInheritance() { - TestRoot root = new TestDriver().buildModel(FilesApplicationPackage.fromFile(new File(TESTDIR))); - RankProfilesConfig left = root.getConfig(RankProfilesConfig.class, "inherit/search/cluster.inherit/left"); - RankProfilesConfig right = root.getConfig(RankProfilesConfig.class, "inherit/search/cluster.inherit/right"); - assertEquals(3, left.rankprofile().size()); - assertEquals(2, right.rankprofile().size()); - } - - @Test(expected = IllegalArgumentException.class) - public void testRankProfileDuplicateNameIsIllegal() { - Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - RankProfile barRankProfile = new RankProfile("bar", schema, rankProfileRegistry); - rankProfileRegistry.add(barRankProfile); - rankProfileRegistry.add(barRankProfile); - } - - @Test - public void testRankProfileDuplicateNameLegalForOverridableRankProfiles() { - Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - - for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { - assertNull(rankProfileRegistry.get(schema, rankProfileName).getFunctions().get("foo")); - RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, schema, rankProfileRegistry); - rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); - rankProfileRegistry.add(rankProfileWithAddedFunction); - assertNotNull(rankProfileRegistry.get(schema, rankProfileName).getFunctions().get("foo")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java deleted file mode 100644 index 50f930dde86..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ /dev/null @@ -1,436 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.collections.Pair; -import com.yahoo.component.ComponentId; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.search.query.profile.QueryProfile; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.FieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; - -import static com.yahoo.config.model.test.TestUtil.joinLines; - -import org.junit.Test; - -import java.util.Iterator; -import java.util.List; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Tests rank profiles - * - * @author bratseth - */ -public class RankProfileTestCase extends AbstractSchemaTestCase { - - @Test - public void testRankProfileInheritance() { - Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - SDDocumentType document = new SDDocumentType("test"); - SDField a = document.addField("a", DataType.STRING); - a.setRankType(RankType.IDENTITY); - document.addField("b", DataType.STRING); - schema.addDocument(document); - RankProfile child = new RankProfile("child", schema, rankProfileRegistry); - child.inherit("default"); - rankProfileRegistry.add(child); - - Iterator i = child.rankSettingIterator(); - - RankProfile.RankSetting setting = i.next(); - assertEquals(RankType.IDENTITY, setting.getValue()); - assertEquals("a", setting.getFieldName()); - assertEquals(RankProfile.RankSetting.Type.RANKTYPE, setting.getType()); - - setting = i.next(); - assertEquals(RankType.DEFAULT, setting.getValue()); - } - - @Test - public void requireThatIllegalInheritanceIsChecked() throws ParseException { - try { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "search test {", - " document test { } ", - " rank-profile p1 inherits notexist {}", - "}")); - builder.build(true); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'p1' inherits 'notexist', but this is not found in schema 'test'", e.getMessage()); - } - } - - @Test - public void requireThatSelfInheritanceIsIllegal() throws ParseException { - try { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "schema test {", - " document test { } ", - " rank-profile self inherits self {}", - "}")); - builder.build(true); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("There is a cycle in the inheritance for rank-profile 'test.self' = [test.self, test.self]", e.getMessage()); - } - } - - @Test - public void requireThatSelfInheritanceIsLegalWhenOverloading() throws ParseException { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "schema base {", - " document base { } ", - " rank-profile self inherits default {}", - "}")); - builder.addSchema(joinLines( - "schema test {", - " document test inherits base { } ", - " rank-profile self inherits self {}", - "}")); - builder.build(true); - } - - @Test - public void requireThatSidewaysInheritanceIsImpossible() throws ParseException { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "schema child1 {", - " document child1 {", - " field field1 type int {", - " indexing: attribute", - " }", - " }", - " rank-profile child inherits parent {", - " function function2() {", - " expression: attribute(field1) + 5", - " }", - " first-phase {", - " expression: function2() * function1()", - " }", - " summary-features {", - " function1", - " function2", - " attribute(field1)", - " }", - " }", - "}\n")); - builder.addSchema(joinLines( - "schema child2 {", - " document child2 {", - " field field1 type int {", - " indexing: attribute", - " }", - " }", - " rank-profile parent {", - " first-phase {", - " expression: function1()", - " }", - " function function1() {", - " expression: attribute(field1) + 7", - " }", - " summary-features {", - " function1", - " attribute(field1)", - " }", - " }", - "}")); - try { - builder.build(true); - fail("Sideways inheritance should have been enforced"); - } catch (IllegalArgumentException e) { - assertEquals("rank-profile 'child' inherits 'parent', but this is not found in schema 'child1'", e.getMessage()); - } - } - - @Test - public void requireThatDefaultInheritingDefaultIsIgnored() throws ParseException { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "schema test {", - " document test { } ", - " rank-profile default inherits default {}", - "}")); - builder.build(true); - } - - @Test - public void requireThatCyclicInheritanceIsIllegal() throws ParseException { - try { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "search test {", - " document test { } ", - " rank-profile a inherits b {}", - " rank-profile b inherits c {}", - " rank-profile c inherits a {}", - "}")); - builder.build(true); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("There is a cycle in the inheritance for rank-profile 'test.c' = [test.c, test.a, test.b, test.c]", e.getMessage()); - } - } - - @Test - public void requireThatRankProfilesCanInheritNotYetSeenProfiles() throws ParseException - { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "search test {", - " document test { } ", - " rank-profile p1 inherits not_yet_defined {}", - " rank-profile not_yet_defined {}", - "}")); - builder.build(true); - assertNotNull(registry.get("test","p1")); - assertTrue(registry.get("test","p1").inherits("not_yet_defined")); - assertNotNull(registry.get("test","not_yet_defined")); - } - - private String createSD(Double termwiseLimit) { - return joinLines( - "search test {", - " document test { ", - " field a type string { ", - " indexing: index ", - " }", - " }", - " ", - " rank-profile parent {", - (termwiseLimit != null ? (" termwise-limit:" + termwiseLimit + "\n") : ""), - " num-threads-per-search:8", - " min-hits-per-thread:70", - " num-search-partitions:1200", - " }", - " rank-profile child inherits parent { }", - "}"); - } - - @Test - public void testTermwiseLimitWithDeployOverride() throws ParseException { - verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties(), createSD(null), null); - verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties(), createSD(0.78), 0.78); - verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties().setDefaultTermwiseLimit(0.09), createSD(null), 0.09); - verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties().setDefaultTermwiseLimit(0.09), createSD(0.37), 0.37); - } - - private void verifyTermwiseLimitAndSomeMoreIncludingInheritance(ModelContext.Properties deployProperties, String sd, Double termwiseLimit) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema(sd); - builder.build(true); - Schema schema = builder.getSchema(); - AttributeFields attributeFields = new AttributeFields(schema); - verifyRankProfile(rankProfileRegistry.get(schema, "parent"), attributeFields, deployProperties, termwiseLimit); - verifyRankProfile(rankProfileRegistry.get(schema, "child"), attributeFields, deployProperties, termwiseLimit); - } - - private void verifyRankProfile(RankProfile rankProfile, AttributeFields attributeFields, ModelContext.Properties deployProperties, - Double expectedTermwiseLimit) { - assertEquals(8, rankProfile.getNumThreadsPerSearch()); - assertEquals(70, rankProfile.getMinHitsPerThread()); - assertEquals(1200, rankProfile.getNumSearchPartitions()); - RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), - new ImportedMlModels(), attributeFields, deployProperties); - if (expectedTermwiseLimit != null) { - assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent()); - assertEquals(String.valueOf(expectedTermwiseLimit), findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").get()); - } else { - assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent()); - } - assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").isPresent()); - assertEquals("8", findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").get()); - assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.minhitsperthread").isPresent()); - assertEquals("70", findProperty(rawRankProfile.configProperties(), "vespa.matching.minhitsperthread").get()); - assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numsearchpartitions").isPresent()); - assertEquals("1200", findProperty(rawRankProfile.configProperties(), "vespa.matching.numsearchpartitions").get()); - } - - @Test - public void requireThatConfigIsDerivedForAttributeTypeSettings() throws ParseException { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10]) { indexing: attribute }", - " field b type tensor(y{}) { indexing: attribute }", - " field c type tensor(x[5]) { indexing: attribute }", - " }", - " rank-profile p1 {}", - " rank-profile p2 {}", - "}")); - builder.build(true); - Schema schema = builder.getSchema(); - - assertEquals(4, registry.all().size()); - assertAttributeTypeSettings(registry.get(schema, "default"), schema); - assertAttributeTypeSettings(registry.get(schema, "unranked"), schema); - assertAttributeTypeSettings(registry.get(schema, "p1"), schema); - assertAttributeTypeSettings(registry.get(schema, "p2"), schema); - } - - @Test - public void requireThatDenseDimensionsMustBeBound() throws ParseException { - try { - ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[]) { indexing: attribute }", - " }", - "}")); - builder.build(true); - } - catch (IllegalArgumentException e) { - assertEquals("Illegal type in field a type tensor(x[]): Dense tensor dimensions must have a size", - e.getMessage()); - } - } - - private static RawRankProfile createRawRankProfile(RankProfile profile, Schema schema) { - return new RawRankProfile(profile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(schema), new TestProperties()); - } - - private static void assertAttributeTypeSettings(RankProfile profile, Schema schema) { - RawRankProfile rawProfile = createRawRankProfile(profile, schema); - assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.a").get()); - assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.attribute.b").get()); - assertEquals("tensor(x[5])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); - } - - @Test - public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { - RankProfileRegistry registry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(registry, setupQueryProfileTypes()); - builder.addSchema(joinLines( - "search test {", - " document test { } ", - " rank-profile p1 {}", - " rank-profile p2 {}", - "}")); - builder.build(true); - Schema schema = builder.getSchema(); - - assertEquals(4, registry.all().size()); - assertQueryFeatureTypeSettings(registry.get(schema, "default"), schema); - assertQueryFeatureTypeSettings(registry.get(schema, "unranked"), schema); - assertQueryFeatureTypeSettings(registry.get(schema, "p1"), schema); - assertQueryFeatureTypeSettings(registry.get(schema, "p2"), schema); - } - - private static QueryProfileRegistry setupQueryProfileTypes() { - QueryProfileRegistry registry = new QueryProfileRegistry(); - QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); - QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); - type.addField(new FieldDescription("ranking.features.query(tensor1)", - FieldType.fromString("tensor(x[10])", typeRegistry)), typeRegistry); - type.addField(new FieldDescription("ranking.features.query(tensor2)", - FieldType.fromString("tensor(y{})", typeRegistry)), typeRegistry); - type.addField(new FieldDescription("ranking.features.invalid(tensor3)", - FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); - type.addField(new FieldDescription("ranking.features.query(numeric)", - FieldType.fromString("integer", typeRegistry)), typeRegistry); - typeRegistry.register(type); - var profile = new QueryProfile(new ComponentId("testprofile")); - profile.setType(type); - registry.register(profile); - return registry; - } - - private static void assertQueryFeatureTypeSettings(RankProfile profile, Schema schema) { - RawRankProfile rawProfile =createRawRankProfile(profile, schema); - assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor1").get()); - assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor2").get()); - assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.query.tensor3").isPresent()); - assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.query.numeric").isPresent()); - } - - private static Optional findProperty(List> properties, String key) { - for (Pair property : properties) - if (property.getFirst().equals(key)) - return Optional.of(property.getSecond()); - return Optional.empty(); - } - - @Test - public void approximate_nearest_neighbor_threshold_settings_are_configurable() throws ParseException { - verifyApproximateNearestNeighborThresholdSettings(0.7, null); - verifyApproximateNearestNeighborThresholdSettings(null, 0.3); - verifyApproximateNearestNeighborThresholdSettings(0.7, 0.3); - } - - private void verifyApproximateNearestNeighborThresholdSettings(Double postFilterThreshold, Double approximateThreshold) throws ParseException { - var rankProfileRegistry = new RankProfileRegistry(); - var props = new TestProperties(); - var queryProfileRegistry = new QueryProfileRegistry(); - var builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry, props); - builder.addSchema(createSDWithRankProfileThresholds(postFilterThreshold, approximateThreshold)); - builder.build(true); - - var schema = builder.getSchema(); - var rankProfile = rankProfileRegistry.get(schema, "my_profile"); - var rawRankProfile = new RawRankProfile(rankProfile, new LargeRankExpressions(new MockFileRegistry()), queryProfileRegistry, - new ImportedMlModels(), new AttributeFields(schema), props); - - if (postFilterThreshold != null) { - assertEquals((double)postFilterThreshold, rankProfile.getPostFilterThreshold().getAsDouble(), 0.000001); - assertEquals(String.valueOf(postFilterThreshold), findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.upper_limit").get()); - } else { - assertTrue(rankProfile.getPostFilterThreshold().isEmpty()); - assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.upper_limit").isPresent()); - } - - if (approximateThreshold != null) { - assertEquals((double)approximateThreshold, rankProfile.getApproximateThreshold().getAsDouble(), 0.000001); - assertEquals(String.valueOf(approximateThreshold), findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.lower_limit").get()); - } else { - assertTrue(rankProfile.getApproximateThreshold().isEmpty()); - assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.global_filter.lower_limit").isPresent()); - } - } - - private String createSDWithRankProfileThresholds(Double postFilterThreshold, Double approximateThreshold) { - return joinLines( - "search test {", - " document test {}", - " rank-profile my_profile {", - (postFilterThreshold != null ? (" post-filter-threshold: " + postFilterThreshold) : ""), - (approximateThreshold != null ? (" approximate-threshold: " + approximateThreshold) : ""), - " }", - "}"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java deleted file mode 100644 index 6de890549c1..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import org.junit.Test; - -import java.util.List; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class RankPropertiesTestCase extends AbstractSchemaTestCase { - - @Test - public void testRankPropertyInheritance() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema(joinLines( - "search test {", - " document test {", - " field a type string { ", - " indexing: index ", - " }", - " }", - " rank-profile parent {", - " first-phase {", - " expression: a", - " }", - " rank-properties {", - " query(a): 1500 ", - " }", - " }", - " rank-profile child inherits parent {", - " first-phase {", - " expression: a", - " }", - " rank-properties {", - " query(a): 2000 ", - " }", - " }", - "}")); - builder.build(true); - Schema schema = builder.getSchema(); - AttributeFields attributeFields = new AttributeFields(schema); - - { - // Check declared model - RankProfile parent = rankProfileRegistry.get(schema, "parent"); - assertEquals("query(a) = 1500", parent.getRankProperties().get(0).toString()); - - // Check derived model - RawRankProfile rawParent = new RawRankProfile(parent, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), attributeFields, new TestProperties()); - assertEquals("(query(a), 1500)", rawParent.configProperties().get(0).toString()); - } - - { - // Check declared model - RankProfile parent = rankProfileRegistry.get(schema, "child"); - assertEquals("query(a) = 2000", parent.getRankProperties().get(0).toString()); - - // Check derived model - RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.get(schema, "child"), - new LargeRankExpressions(new MockFileRegistry()), - new QueryProfileRegistry(), - new ImportedMlModels(), - attributeFields, - new TestProperties()); - assertEquals("(query(a), 2000)", rawChild.configProperties().get(0).toString()); - } - } - @Test - public void testRankProfileMutate() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema(joinLines( - "search test {", - " document test {", - " field a type int { ", - " indexing: attribute ", - " }", - " }", - " field synthetic_attribute_a type int {", - " indexing: attribute", - " attribute: mutable", - " }", - " field synthetic_attribute_b type double {", - " indexing: attribute", - " attribute: mutable", - " }", - " field synthetic_attribute_c type long {", - " indexing: attribute", - " attribute: mutable", - " }", - " rank-profile a {", - " mutate {", - " on-match {", - " synthetic_attribute_a += 7", - " }", - " on-first-phase {", - " synthetic_attribute_b +=1", - " }", - " on-second-phase {", - " synthetic_attribute_b = 1.01", - " }", - " on-summary {", - " synthetic_attribute_c -= 1", - " }", - " }", - " first-phase {", - " expression: a", - " }", - " }", - " rank-profile b {", - " first-phase {", - " expression: a", - " }", - " second-phase {", - " expression: a", - " }", - " }", - "}")); - builder.build(true); - Schema schema = builder.getSchema(); - RankProfile a = rankProfileRegistry.get(schema, "a"); - List operations = a.getMutateOperations(); - assertEquals(4, operations.size()); - assertEquals(RankProfile.MutateOperation.Phase.on_match, operations.get(0).phase); - assertEquals("synthetic_attribute_a", operations.get(0).attribute); - assertEquals("+=7", operations.get(0).operation); - assertEquals(RankProfile.MutateOperation.Phase.on_first_phase, operations.get(1).phase); - assertEquals("synthetic_attribute_b", operations.get(1).attribute); - assertEquals("+=1", operations.get(1).operation); - assertEquals(RankProfile.MutateOperation.Phase.on_second_phase, operations.get(2).phase); - assertEquals("synthetic_attribute_b", operations.get(2).attribute); - assertEquals("=1.01", operations.get(2).operation); - assertEquals(RankProfile.MutateOperation.Phase.on_summary, operations.get(3).phase); - assertEquals("synthetic_attribute_c", operations.get(3).attribute); - assertEquals("-=1", operations.get(3).operation); - - AttributeFields attributeFields = new AttributeFields(schema); - RawRankProfile raw = new RawRankProfile(a, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), attributeFields, new TestProperties()); - assertEquals(9, raw.configProperties().size()); - assertEquals("(vespa.mutate.on_match.attribute, synthetic_attribute_a)", raw.configProperties().get(0).toString()); - assertEquals("(vespa.mutate.on_match.operation, +=7)", raw.configProperties().get(1).toString()); - assertEquals("(vespa.mutate.on_first_phase.attribute, synthetic_attribute_b)", raw.configProperties().get(2).toString()); - assertEquals("(vespa.mutate.on_first_phase.operation, +=1)", raw.configProperties().get(3).toString()); - assertEquals("(vespa.mutate.on_second_phase.attribute, synthetic_attribute_b)", raw.configProperties().get(4).toString()); - assertEquals("(vespa.mutate.on_second_phase.operation, =1.01)", raw.configProperties().get(5).toString()); - assertEquals("(vespa.mutate.on_summary.attribute, synthetic_attribute_c)", raw.configProperties().get(6).toString()); - assertEquals("(vespa.mutate.on_summary.operation, -=1)", raw.configProperties().get(7).toString()); - assertEquals("(vespa.rank.firstphase, a)", raw.configProperties().get(8).toString()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java deleted file mode 100644 index 3aab1a7bbe6..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.Iterator; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.fail; - -/** - * @author gjoranv - */ -public class RankingConstantTest { - - @SuppressWarnings("deprecation") - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Test - public void tensor_constant_properties_are_set() throws Exception { - final String TENSOR_NAME = "my_global_tensor"; - final String TENSOR_FILE = "path/my-tensor-file.json"; - final String TENSOR_TYPE = "tensor(x{})"; - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(constant(my_global_tensor))", - " }", - " }", - " constant " + TENSOR_NAME + " {", - " file: " + TENSOR_FILE, - " type: " + TENSOR_TYPE, - " }", - "}" - )); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - - Iterator constantIterator = schema.constants().values().iterator(); - RankProfile.Constant constant = constantIterator.next(); - assertEquals(TENSOR_NAME, constant.name().simpleArgument().get()); - assertEquals(TENSOR_FILE, constant.valuePath().get()); - assertEquals(TENSOR_TYPE, constant.type().toString()); - assertEquals(DistributableResource.PathType.FILE, constant.pathType().get()); - - assertFalse(constantIterator.hasNext()); - } - - @Test - public void tensor_constant_must_have_a_type() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("must have a type"); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " file: bar.baz", - " }", - "}" - )); - } - - @Test - public void tensor_constant_must_have_a_file() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("must have a file"); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x[])", - " }", - "}" - )); - } - - @Test - public void constant_file_does_not_need_path_or_ending() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x{})", - " file: simplename", - " }", - "}" - )); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - RankProfile.Constant constant = schema.constants().values().iterator().next(); - assertEquals("simplename", constant.valuePath().get()); - } - - @Test - public void constant_uri_is_allowed() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x{})", - " uri: http://somewhere.far.away/in/another-galaxy", - " }", - "}" - )); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - RankProfile.Constant constant = schema.constants().values().iterator().next(); - assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); - assertEquals("http://somewhere.far.away/in/another-galaxy", constant.valuePath().get()); - } - - @Test - public void constant_https_uri_is_allowed() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x{})", - " uri: https://somewhere.far.away:4443/in/another-galaxy", - " }", - "}" - )); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - RankProfile.Constant constant = schema.constants().values().iterator().next(); - assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); - assertEquals("https://somewhere.far.away:4443/in/another-galaxy", constant.valuePath().get()); - } - - @Test - public void constant_uri_with_port_is_allowed() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x{})", - " uri: http://somewhere.far.away:4080/in/another-galaxy", - " }", - "}" - )); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - RankProfile.Constant constant = schema.constants().values().iterator().next(); - assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); - assertEquals("http://somewhere.far.away:4080/in/another-galaxy", constant.valuePath().get()); - } - - @Test - public void constant_uri_no_dual_slashes_is_allowed() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x{})", - " uri: http:somewhere.far.away/in/another-galaxy", - " }", - "}" - )); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - RankProfile.Constant constant = schema.constants().values().iterator().next(); - assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); - assertEquals("http:somewhere.far.away/in/another-galaxy", constant.valuePath().get()); - } - - @Test - public void constant_uri_only_supports_http_and_https() { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); - String expectedMessage = "Encountered \" \"ftp\"\" at line 5, column 10.\n\n" + - "Was expecting:\n\n" + - " ..."; - try { - schemaBuilder.addSchema(joinLines( - "schema test {", - " document test { }", - " constant foo {", - " type: tensor(x{})", - " uri: ftp:somewhere.far.away/in/another-galaxy", - " }", - "}" - )); - } catch (ParseException e) { - if (! e.getMessage().startsWith(expectedMessage)) - fail("Expected exception with message starting with:\n'" + expectedMessage + "\nBut got:\n'" + e.getMessage()); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java deleted file mode 100644 index e2f06bd8bc6..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.collections.Pair; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.yolean.Exceptions; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.*; - -/** - * @author bratseth - */ -public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { - - @Test - public void testConstants() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "schema test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " constants {\n" + - " p1 double: 7 \n" + - " constant(p2) double: 0 \n" + - " }\n" + - " first-phase {\n" + - " expression: p2 * (1.3 + p1 )\n" + - " }\n" + - " }\n" + - " rank-profile child1 inherits parent {\n" + - " first-phase {\n" + - " expression: a + b + c \n" + - " }\n" + - " second-phase {\n" + - " expression: a + p1 + c \n" + - " }\n" + - " constants {\n" + - " a: 1.0 \n" + - " constant(b): 2 \n" + - " c: 3.5 \n" + - " }\n" + - " }\n" + - " rank-profile child2 inherits parent {\n" + - " constants {\n" + - " p2: 2.0 \n" + - " }\n" + - " function foo() {\n" + - " expression: p2*p1\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile parent = rankProfileRegistry.get(s, "parent").compile(queryProfileRegistry, new ImportedMlModels()); - assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); - - RankProfile child1 = rankProfileRegistry.get(s, "child1").compile(queryProfileRegistry, new ImportedMlModels()); - assertEquals("6.5", child1.getFirstPhaseRanking().getRoot().toString()); - assertEquals("11.5", child1.getSecondPhaseRanking().getRoot().toString()); - - RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedMlModels()); - assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); - List> rankProperties = new RawRankProfile(child2, - new LargeRankExpressions(new MockFileRegistry()), - queryProfileRegistry, - new ImportedMlModels(), - new AttributeFields(s), - new TestProperties()).configProperties(); - assertEquals("(rankingExpression(foo).rankingScript, 14.0)", rankProperties.get(0).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript, 16.6)", rankProperties.get(2).toString()); - } - - @Test - public void testNameCollision() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "schema test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " constants {\n" + - " c: 7 \n" + - " }\n" + - " function c() {\n" + - " expression: p2*p1\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - try { - rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - fail("Should have caused an exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", - Exceptions.toMessageString(e)); - } - } - - @Test - public void testNegativeLiteralArgument() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function POP_SLOW_SCORE() {\n" + - " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile profile = rankProfileRegistry.get(s, "test"); - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); - } - - @Test - public void testNegativeConstantArgument() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "schema test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " constants {\n" + - " myValue: -9.21034037\n" + - " }\n" + - " function POP_SLOW_SCORE() {\n" + - " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile profile = rankProfileRegistry.get(s, "test"); - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedMlModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); - } - - @Test - public void testConstantDivisorInFunction() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function rank_default(){\n" + - " expression: k1 + (k2 + k3) / 100000000.0\n\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile profile = rankProfileRegistry.get(s, "test"); - assertEquals("k1 + (k2 + k3) / 1.0E8", - profile.compile(new QueryProfileRegistry(), new ImportedMlModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); - } - - @Test - public void test3() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field rating_yelp type int {" + - " indexing: attribute" + - " }" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function rank_default(){\n" + - " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile profile = rankProfileRegistry.get(s, "test"); - assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedMlModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java deleted file mode 100644 index fd35c19906d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.collections.Pair; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Optional; -import java.util.logging.Level; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author bratseth - */ -public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { - - @Test - public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type double { \n" + - " indexing: attribute \n" + - " }\n" + - " field b type double { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " constants {\n" + - " p1: 7 \n" + - " p2: 0 \n" + - " }\n" + - " first-phase {\n" + - " expression: p1 * add\n" + - " }\n" + - " function inline add() {\n" + - " expression: 3 + attribute(a) + attribute(b) * mul3\n" + - " }\n" + - " function inline mul3() {\n" + - " expression: attribute(a) * 3 + singleif\n" + - " }\n" + - " function inline singleif() {\n" + - " expression: if (p1 < attribute(a), 1, 2) == 0\n" + - " }\n" + - " }\n" + - " rank-profile child inherits parent {\n" + - " function inline add() {\n" + - " expression: 9 + attribute(a)\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - - RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("7.0 * (3 + attribute(a) + attribute(b) * (attribute(a) * 3 + if (7.0 < attribute(a), 1, 2) == 0))", - parent.getFirstPhaseRanking().getRoot().toString()); - RankProfile child = rankProfileRegistry.get(s, "child").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("7.0 * (9 + attribute(a))", - child.getFirstPhaseRanking().getRoot().toString()); - } - - @Test - public void testConstants() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " constants {\n" + - " p1: 7 \n" + - " p2: 0 \n" + - " }\n" + - " first-phase {\n" + - " expression: p1 + foo\n" + - " }\n" + - " second-phase {\n" + - " expression: p2 * foo\n" + - " }\n" + - " function inline foo() {\n" + - " expression: 3 + p1 + p2\n" + - " }\n" + - " }\n" + - " rank-profile child inherits parent {\n" + - " first-phase {\n" + - " expression: p1 + foo + baz + bar + arg(4.0)\n" + - " }\n" + - " constants {\n" + - " p2: 2.0 \n" + - " }\n" + - " function bar() {\n" + - " expression: p2*p1\n" + - " }\n" + - " function inline baz() {\n" + - " expression: p2+p1+boz\n" + - " }\n" + - " function inline boz() {\n" + - " expression: 3.0\n" + - " }\n" + - " function inline arg(a1) {\n" + - " expression: a1*2\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - - RankProfile parent = rankProfileRegistry.get(s, "parent").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("17.0", parent.getFirstPhaseRanking().getRoot().toString()); - assertEquals("0.0", parent.getSecondPhaseRanking().getRoot().toString()); - assertEquals("10.0", getRankingExpression("foo", parent, s)); - assertEquals("17.0", getRankingExpression("firstphase", parent, s)); - assertEquals("0.0", getRankingExpression("secondphase", parent, s)); - - RankProfile child = rankProfileRegistry.get(s, "child").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("31.0 + bar + arg(4.0)", child.getFirstPhaseRanking().getRoot().toString()); - assertEquals("24.0", child.getSecondPhaseRanking().getRoot().toString()); - assertEquals("12.0", getRankingExpression("foo", child, s)); - assertEquals("12.0", getRankingExpression("baz", child, s)); - assertEquals("3.0", getRankingExpression("boz", child, s)); - assertEquals("14.0", getRankingExpression("bar", child, s)); - assertEquals("a1 * 2", getRankingExpression("arg", child, s)); - assertEquals("31.0 + rankingExpression(bar) + rankingExpression(arg@)", getRankingExpression("firstphase", child, s)); - assertEquals("24.0", getRankingExpression("secondphase", child, s)); - } - - @Test - public void testNonTopLevelInlining() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type double { \n" + - " indexing: attribute \n" + - " }\n" + - " field b type double { \n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: A + C + D\n" + - " }\n" + - " function inline D() {\n" + - " expression: B + 1\n" + - " }\n" + - " function C() {\n" + - " expression: A + B\n" + - " }\n" + - " function inline B() {\n" + - " expression: attribute(b)\n" + - " }\n" + - " function inline A() {\n" + - " expression: attribute(a)\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - - RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("attribute(a) + C + (attribute(b) + 1)", test.getFirstPhaseRanking().getRoot().toString()); - assertEquals("attribute(a) + attribute(b)", getRankingExpression("C", test, s)); - assertEquals("attribute(b) + 1", getRankingExpression("D", test, s)); - } - - @Test - public void testFunctionInliningWithReplacement() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - MockDeployLogger deployLogger = new MockDeployLogger(); - ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - deployLogger, - new TestProperties(), - rankProfileRegistry, - new QueryProfileRegistry()); - builder.addSchema( - "search test {\n" + - " document test { }\n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo\n" + - " }\n" + - " function foo(x) {\n" + - " expression: x + x\n" + - " }\n" + - " function inline foo() {\n" + // replaces previous "foo" during parsing - " expression: foo(2)\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); - assertTrue("Does not contain expected warning", - deployLogger.contains("Function 'foo' is defined twice in rank profile 'test'")); - } - - /** - * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of - * each function for each binding, using hashes to name the bound variants of the function. - * This method censors those hashes for string comparison. - */ - private String censorBindingHash(String s) { - StringBuilder b = new StringBuilder(); - boolean areInHash = false; - for (int i = 0; i < s.length() ; i++) { - char current = s.charAt(i); - - if ( ! Character.isLetterOrDigit(current)) // end of hash - areInHash = false; - - if ( ! areInHash) - b.append(current); - - if (current == '@') // start of hash - areInHash = true; - } - return b.toString(); - } - - private String getRankingExpression(String name, RankProfile rankProfile, Schema schema) { - Optional rankExpression = - new RawRankProfile(rankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(schema), new TestProperties()) - .configProperties() - .stream() - .filter(r -> r.getFirst().equals("rankingExpression(" + name + ").rankingScript")) - .map(Pair::getSecond) - .findFirst(); - assertTrue(rankExpression.isPresent()); - return censorBindingHash(rankExpression.get()); - } - - private static class MockDeployLogger implements DeployLogger { - private final ArrayList msgs = new ArrayList<>(); - - @Override - public void log(Level level, String message) { - msgs.add(message); - } - - public boolean contains(String expected) { - return msgs.stream().anyMatch(msg -> msg.equals(expected)); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java deleted file mode 100644 index ff8a9df4076..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class RankingExpressionLoopDetectionTestCase { - - @Test - public void testSelfLoop() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo\n" + - " }\n" + - " function foo() {\n" + - " expression: foo\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - try { - builder.build(true); - fail("Excepted exception"); - } - catch (IllegalArgumentException e) { - assertEquals("In schema 'test', rank profile 'test': The function 'foo' is invalid: foo is invalid: Invocation loop: foo -> foo", - Exceptions.toMessageString(e)); - } - } - - @Test - public void testNestedLoop() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo\n" + - " }\n" + - " function foo() {\n" + - " expression: arg(5)\n" + - " }\n" + - " function arg(a1) {\n" + - " expression: foo + a1*2\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - try { - builder.build(true); - fail("Excepted exception"); - } - catch (IllegalArgumentException e) { - assertEquals("In schema 'test', rank profile 'test': The function 'foo' is invalid: arg(5) is invalid: foo is invalid: arg(5) is invalid: Invocation loop: arg(5) -> foo -> arg(5)", - Exceptions.toMessageString(e)); - } - } - - @Test - public void testSelfArgumentLoop() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo\n" + - " }\n" + - " function foo() {\n" + - " expression: arg(foo)\n" + - " }\n" + - " function arg(a1) {\n" + - " expression: a1*2\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - try { - builder.build(true); - fail("Excepted exception"); - } - catch (IllegalArgumentException e) { - assertEquals("In schema 'test', rank profile 'test': The function 'foo' is invalid: arg(foo) is invalid: a1 is invalid: foo is invalid: arg(foo) is invalid: Invocation loop: arg(foo) -> foo -> arg(foo)", - Exceptions.toMessageString(e)); - } - } - - @Test - public void testNoLoopWithSameLocalArgument() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo(3)\n" + - " }\n" + - " function foo(a1) {\n" + - " expression: bar(3)\n" + - " }\n" + - " function bar(a1) {\n" + - " expression: a1*2\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - } - - @Test - public void testNoLoopWithMultipleInvocations() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo(3)\n" + - " }\n" + - " function foo(a1) {\n" + - " expression: bar(3) + bar(a1)\n" + - " }\n" + - " function bar(a1) {\n" + - " expression: a1*2\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - } - - @Test - public void testNoLoopWithBoundIdentifiers() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " }\n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo(bar(2))\n" + - " }\n" + - " function foo(x) {\n" + - " expression: x * x\n" + - " }\n" + - " function bar(x) {\n" + - " expression: x + x\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - } - - @Test - public void testNoLoopWithTheSameNestedIdentifierWhichIsUnbound() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " }\n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo()\n" + - " }\n" + - " function foo() {\n" + - " expression: bar(x)\n" + - " }\n" + - " function bar(x) {\n" + - " expression: x + x\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - } - - @Test - public void testNoLoopWithTheSameAlternatingNestedIdentifierWhichIsUnbound() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " }\n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo()\n" + - " }\n" + - " function foo() {\n" + - " expression: bar(x)\n" + - " }\n" + - " function bar(y) {\n" + - " expression: baz(y)\n" + - " }\n" + - " function baz(x) {\n" + - " expression: x + x\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java deleted file mode 100644 index 8354eb59416..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.collections.Pair; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfile; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author lesters - */ -public class RankingExpressionShadowingTestCase extends AbstractSchemaTestCase { - - @Test - public void testBasicFunctionShadowing() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function sin(x) {\n" + - " expression: x * x\n" + - " }\n" + - " first-phase {\n" + - " expression: sin(2)\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); - assertEquals("(rankingExpression(sin@).rankingScript, 2 * 2)", - censorBindingHash(testRankProperties.get(0).toString())); - assertEquals("(rankingExpression(sin).rankingScript, x * x)", - testRankProperties.get(1).toString()); - assertEquals("(vespa.rank.firstphase, rankingExpression(sin@))", - censorBindingHash(testRankProperties.get(2).toString())); - } - - - @Test - public void testMultiLevelFunctionShadowing() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function tan(x) {\n" + - " expression: x * x\n" + - " }\n" + - " function cos(x) {\n" + - " expression: tan(x)\n" + - " }\n" + - " function sin(x) {\n" + - " expression: cos(x)\n" + - " }\n" + - " first-phase {\n" + - " expression: sin(2)\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); - assertEquals("(rankingExpression(tan@).rankingScript, 2 * 2)", - censorBindingHash(testRankProperties.get(0).toString())); - assertEquals("(rankingExpression(cos@).rankingScript, rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(sin@).rankingScript, rankingExpression(cos@))", - censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(tan).rankingScript, x * x)", - testRankProperties.get(3).toString()); - assertEquals("(rankingExpression(tan@).rankingScript, x * x)", - censorBindingHash(testRankProperties.get(4).toString())); - assertEquals("(rankingExpression(cos).rankingScript, rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(5).toString())); - assertEquals("(rankingExpression(cos@).rankingScript, rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(6).toString())); - assertEquals("(rankingExpression(sin).rankingScript, rankingExpression(cos@))", - censorBindingHash(testRankProperties.get(7).toString())); - assertEquals("(vespa.rank.firstphase, rankingExpression(sin@))", - censorBindingHash(testRankProperties.get(8).toString())); - } - - @Test - public void testFunctionShadowingArguments() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function sin(x) {\n" + - " expression: x * x\n" + - " }\n" + - " first-phase {\n" + - " expression: cos(sin(2*2)) + sin(cos(1+4))\n" + - " }\n" + - " }\n" + - "\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - List> testRankProperties = createRawRankProfile(test, new QueryProfileRegistry(), s).configProperties(); - assertEquals("(rankingExpression(sin@).rankingScript, 4.0 * 4.0)", - censorBindingHash(testRankProperties.get(0).toString())); - assertEquals("(rankingExpression(sin@).rankingScript, cos(5.0) * cos(5.0))", - censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(sin).rankingScript, x * x)", - testRankProperties.get(2).toString()); - assertEquals("(vespa.rank.firstphase, rankingExpression(firstphase))", - censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(firstphase).rankingScript, cos(rankingExpression(sin@)) + rankingExpression(sin@))", - censorBindingHash(testRankProperties.get(4).toString())); - } - - @Test - public void testNeuralNetworkSetup() throws ParseException { - // Note: the type assigned to query profile and constant tensors here is not the correct type - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(input[1])"); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test {\n" + - " function relu(x) {\n" + // relu is a built in function, redefined here - " expression: max(1.0, x)\n" + - " }\n" + - " function hidden_layer() {\n" + - " expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" + - " }\n" + - " function final_layer() {\n" + - " expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" + - " }\n" + - " second-phase {\n" + - " expression: sum(final_layer)\n" + - " }\n" + - " }\n" + - " constant W_hidden {\n" + - " type: tensor(hidden[1])\n" + - " file: ignored.json\n" + - " }\n" + - " constant b_input {\n" + - " type: tensor(hidden[1])\n" + - " file: ignored.json\n" + - " }\n" + - " constant W_final {\n" + - " type: tensor(final[1])\n" + - " file: ignored.json\n" + - " }\n" + - " constant b_final {\n" + - " type: tensor(final[1])\n" + - " file: ignored.json\n" + - " }\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); - List> testRankProperties = createRawRankProfile(test, queryProfiles, s).configProperties(); - assertEquals("(rankingExpression(autogenerated_ranking_feature@).rankingScript, reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input))", - censorBindingHash(testRankProperties.get(0).toString())); - assertEquals("(rankingExpression(relu@).rankingScript, max(1.0,rankingExpression(autogenerated_ranking_feature@)))", - censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(hidden_layer).rankingScript, rankingExpression(relu@))", - censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(final_layer).rankingScript, sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", - testRankProperties.get(4).toString()); - assertEquals("(rankingExpression(relu).rankingScript, max(1.0,x))", - testRankProperties.get(6).toString()); - assertEquals("(vespa.rank.secondphase, rankingExpression(secondphase))", - testRankProperties.get(7).toString()); - assertEquals("(rankingExpression(secondphase).rankingScript, reduce(rankingExpression(final_layer), sum))", - testRankProperties.get(8).toString()); - } - - private static RawRankProfile createRawRankProfile(RankProfile profile, QueryProfileRegistry queryProfiles, Schema schema) { - return new RawRankProfile(profile, - new LargeRankExpressions(new MockFileRegistry()), - queryProfiles, - new ImportedMlModels(), - new AttributeFields(schema), - new TestProperties()); - } - - private QueryProfileRegistry queryProfileWith(String field, String type) { - QueryProfileType queryProfileType = new QueryProfileType("root"); - queryProfileType.addField(new FieldDescription(field, type)); - QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); - queryProfileRegistry.getTypeRegistry().register(queryProfileType); - QueryProfile profile = new QueryProfile("default"); - profile.setType(queryProfileType); - queryProfileRegistry.register(profile); - return queryProfileRegistry; - } - - private String censorBindingHash(String s) { - StringBuilder b = new StringBuilder(); - boolean areInHash = false; - for (int i = 0; i < s.length() ; i++) { - char current = s.charAt(i); - if ( ! Character.isLetterOrDigit(current)) // end of hash - areInHash = false; - if ( ! areInHash) - b.append(current); - if (current == '@') // start of hash - areInHash = true; - } - return b.toString(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java deleted file mode 100644 index ee3b4ab25e0..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -/** - * @author bratseth - */ -public class RankingExpressionValidationTestCase extends AbstractSchemaTestCase { - - @Test - public void testInvalidExpressionProducesException() throws ParseException { - assertFailsExpression("&/%(/%&"); - assertFailsExpression("if(a==b,b)"); - } - - private void assertFailsExpression(String expression) throws ParseException { - try { - RankProfileRegistry registry = new RankProfileRegistry(); - Schema schema = importWithExpression(expression, registry); - new DerivedConfiguration(schema, registry); // cause rank profile parsing - fail("No exception on incorrect ranking expression " + expression); - } catch (IllegalArgumentException e) { - // Success - assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); - } - } - - private Schema importWithExpression(String expression, RankProfileRegistry registry) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(registry); - builder.addSchema("search test {" + - " document test { " + - " field a type string { " + - " indexing: index " + - " }" + - " }" + - " rank-profile default {" + - " first-phase {" + - " expression: " + expression + - " }" + - " }" + - "}"); - builder.build(true); - return builder.getSchema(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java deleted file mode 100644 index a8aaee83938..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertNotNull; - -/** - * @author bratseth - */ -public class ReservedWordsAsFieldNamesTestCase extends AbstractSchemaTestCase { - - @Test - public void testIt() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/reserved_words_as_field_names.sd"); - assertNotNull(schema.getDocument().getField("inline")); - assertNotNull(schema.getDocument().getField("constants")); - assertNotNull(schema.getDocument().getField("reference")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java deleted file mode 100755 index 652a06a6025..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.document.DataTypeName; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDDocumentType; -import com.yahoo.searchdefinition.document.TemporarySDField; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author Einar M R Rosenvinge - */ -public class SDDocumentTypeOrdererTestCase { - - @Test - public void testOrder() { - List types = new ArrayList<>(); - - SDDocumentType a = new SDDocumentType("a"); - SDDocumentType b = new SDDocumentType("b"); - SDDocumentType c = new SDDocumentType("c"); - SDDocumentType d = new SDDocumentType("d"); - SDDocumentType e = new SDDocumentType("e"); - SDDocumentType f = new SDDocumentType("f"); - SDDocumentType g = new SDDocumentType("g"); - b.inherit(new TemporarySDDocumentType(new DataTypeName("a"))); - c.inherit(new TemporarySDDocumentType(new DataTypeName("b"))); - d.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); - g.inherit(new TemporarySDDocumentType(new DataTypeName("e"))); - g.inherit(new TemporarySDDocumentType(new DataTypeName("c"))); - - SDField aFieldTypeB = new TemporarySDField(a, "atypeb", DataType.STRING); - a.addField(aFieldTypeB); - - SDField bFieldTypeC = new TemporarySDField(b, "btypec", DataType.STRING); - b.addField(bFieldTypeC); - - SDField cFieldTypeG = new TemporarySDField(c, "ctypeg", DataType.STRING); - c.addField(cFieldTypeG); - - SDField gFieldTypeF = new TemporarySDField(g, "gtypef", DataType.STRING); - g.addField(gFieldTypeF); - - SDField fFieldTypeC = new TemporarySDField(f, "ftypec", DataType.STRING); - f.addField(fFieldTypeC); - - SDField dFieldTypeE = new TemporarySDField(d, "dtypee", DataType.STRING); - d.addField(dFieldTypeE); - - types.add(a); - types.add(b); - types.add(c); - types.add(d); - types.add(e); - types.add(f); - types.add(g); - - SDDocumentTypeOrderer app = new SDDocumentTypeOrderer(types, new BaseDeployLogger()); - app.process(); - assertEquals(7, app.processingOrder.size()); - assertEquals(a, app.processingOrder.get(0)); - assertEquals(b, app.processingOrder.get(1)); - assertEquals(c, app.processingOrder.get(2)); - assertEquals(e, app.processingOrder.get(3)); - assertEquals(d, app.processingOrder.get(4)); - assertEquals(f, app.processingOrder.get(5)); - assertEquals(g, app.processingOrder.get(6)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java deleted file mode 100644 index 9c895b452d5..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.document.DataType; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.MakeAliases; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; -import java.util.Iterator; - -import static com.google.common.collect.testing.Helpers.assertEmpty; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Tests importing of search definitions - * - * @author bratseth - */ -public class SchemaImporterTestCase extends AbstractSchemaTestCase { - - @Test - @SuppressWarnings("deprecation") - public void testSimpleImporting() throws IOException, ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); - sb.addSchemaFile("src/test/examples/simple.sd"); - sb.build(true); - Schema schema = sb.getSchema(); - assertEquals("simple", schema.getName()); - assertTrue(schema.hasDocument()); - - SDDocumentType document = schema.getDocument(); - assertEquals("simple", document.getName()); - assertEquals(23, document.getFieldCount()); - - SDField field; - Attribute attribute; - - new MakeAliases(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); - - // First field - field = (SDField) document.getField("title"); - assertEquals(DataType.STRING, field.getDataType()); - assertEquals("{ input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", field.getIndexingScript().toString()); - assertFalse(schema.getIndex("default").isPrefix()); - assertTrue(schema.getIndex("title").isPrefix()); - Iterator titleAliases = schema.getIndex("title").aliasIterator(); - assertEquals("aliaz", titleAliases.next()); - assertEquals("analias.totitle", titleAliases.next()); - assertEquals("analias.todefault", - schema.getIndex("default").aliasIterator().next()); - assertEquals(RankType.IDENTITY, field.getRankType()); - assertEquals(0, field.getAttributes().size()); - assertNull(field.getStemming()); - assertTrue(field.getNormalizing().doRemoveAccents()); - assertTrue(field.isHeader()); - - // Second field - field = (SDField) document.getField("description"); - assertEquals(RankType.ABOUT, field.getRankType()); - assertEquals(SummaryTransform.NONE, - field.getSummaryField("description").getTransform()); - assertEquals(SummaryTransform.DYNAMICTEASER, - field.getSummaryField("dyndesc").getTransform()); - assertNull(field.getStemming()); - assertTrue(field.getNormalizing().doRemoveAccents()); - assertEquals("hallo", schema.getIndex("description").aliasIterator().next()); - - // Third field - field = (SDField) document.getField("chatter"); - assertEquals(RankType.ABOUT, field.getRankType()); - assertNull(field.getStemming()); - assertTrue(field.getNormalizing().doRemoveAccents()); - - // Fourth field - field = (SDField) document.getField("category"); - assertEquals(0, field.getAttributes().size()); - assertEquals(Stemming.NONE, field.getStemming()); - assertFalse(field.getNormalizing().doRemoveAccents()); - - // Fifth field - field = (SDField) document.getField("popularity"); - assertEquals("{ input popularity | attribute popularity; }", - field.getIndexingScript().toString()); - - // Sixth field - field = (SDField) document.getField("measurement"); - assertEquals(DataType.INT, field.getDataType()); - assertEquals(RankType.EMPTY, field.getRankType()); - assertEquals(1, field.getAttributes().size()); - - // Seventh field - field = schema.getConcreteField("categories"); - assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", - field.getIndexingScript().toString()); - assertTrue(field.isHeader()); - - // Eight field - field= schema.getConcreteField("categoriesagain"); - assertEquals("{ input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", - field.getIndexingScript().toString()); - assertTrue(field.isHeader()); - - // Ninth field - field= schema.getConcreteField("exactemento"); - assertEquals("{ input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", - field.getIndexingScript().toString()); - - // Tenth field - field = schema.getConcreteField("category_arr"); - assertEquals(1, field.getAttributes().size()); - attribute = field.getAttributes().get("category_arr"); - assertNotNull(attribute); - assertEquals("category_arr", attribute.getName()); - assertEquals(Attribute.Type.STRING, attribute.getType()); - assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); - assertTrue(field.isHeader()); - - // Eleventh field - field = schema.getConcreteField("measurement_arr"); - assertEquals(1, field.getAttributes().size()); - attribute = field.getAttributes().get("measurement_arr"); - assertEquals("measurement_arr", attribute.getName()); - assertEquals(Attribute.Type.INTEGER, attribute.getType()); - assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); - - // Rank Profiles - RankProfile profile = rankProfileRegistry.get(schema, "default"); - assertNotNull(profile); - assertEmpty(profile.inheritedNames()); - assertNull(profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE)); - assertEquals(RankType.EMPTY, - profile.getRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); - profile = rankProfileRegistry.get(schema, "experimental"); - assertNotNull(profile); - assertEquals("default", profile.inheritedNames().get(0)); - assertEquals(RankType.IDENTITY, - profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); - - profile = rankProfileRegistry.get(schema, "other"); - assertNotNull(profile); - assertEquals("experimental", profile.inheritedNames().get(0)); - - // The extra-document field - SDField exact = schema.getConcreteField("exact"); - assertNotNull("Extra field was parsed", exact); - assertEquals("exact", exact.getName()); - assertEquals(Stemming.NONE, exact.getStemming()); - assertFalse(exact.getNormalizing().doRemoveAccents()); - assertEquals("{ input title . \" \" . input category | tokenize | summary exact | index exact; }", - exact.getIndexingScript().toString()); - assertEquals(RankType.IDENTITY, exact.getRankType()); - } - - @Test - public void testDocumentImporting() throws IOException, ParseException { - try { - // Having two documents in one sd-file is illegal. - ApplicationBuilder.buildFromFile("src/test/examples/documents.sd"); - fail(); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testIdImporting() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/strange.sd"); - SDField idecidemyide = (SDField)schema.getDocument().getField("idecidemyide"); - assertEquals(5, idecidemyide.getId()); - SDField sodoi = (SDField) schema.getDocument().getField("sodoi"); - assertEquals(7, sodoi.getId()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java deleted file mode 100644 index 6d619b76c18..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import java.io.IOException; - -import com.yahoo.searchdefinition.parser.ParseException; - -import org.junit.Test; - -import static org.junit.Assert.*; - -/** - * Tests that search definitions are parsed correctly and that correct line number is reported in - * error message. - * - * @author hmusum - */ -public class SchemaParsingTestCase extends AbstractSchemaTestCase { - - @Test - public void requireThatIndexingExpressionsCanBeParsed() throws Exception { - assertNotNull(ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); - } - - @Test - public void requireThatParseExceptionPositionIsCorrect() throws Exception { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_construct.sd"); - } catch (ParseException e) { - if ( ! e.getMessage().contains("at line 5, column 36.")) { - throw e; - } - } - } - - @Test - public void requireThatParserHandlesLexicalError() throws Exception { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_lexical_error.sd"); - } catch (ParseException e) { - if (!e.getMessage().contains("at line 7, column 27.")) { - throw e; - } - } - } - - @Test - public void requireErrorWhenJunkAfterSearchBlock() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_junk_at_end.sd"); - fail("Illegal junk at end of SD passed"); - } catch (ParseException e) { - if (!e.getMessage().contains("at line 10, column 1")) { - throw e; - } - } - } - - @Test - public void requireErrorWhenMissingClosingSearchBracket() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_no_closing_bracket.sd"); - fail("SD without closing bracket passed"); - } catch (ParseException e) { - if (!e.getMessage().contains("Encountered \"\" at line 8, column 1")) { - throw e; - } - } - } - - @Test - public void illegalSearchDefinitionName() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalid-name.sd"); - fail("Name with dash passed"); - } catch (ParseException e) { - if ( ! e.getMessage().contains("invalid-name")) { - throw e; - } - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java deleted file mode 100644 index cb57746d82f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.ImportedFieldsResolver; -import com.yahoo.searchdefinition.processing.OnnxModelTypeResolver; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.model.test.utils.DeployLoggerStub; -import org.junit.Test; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Schema tests that don't depend on files. - * - * @author bratseth - */ -public class SchemaTestCase { - - @Test - public void testValidationOfInheritedSchema() throws ParseException { - try { - String schema = joinLines( - "schema test inherits nonesuch {" + - " document test inherits nonesuch {" + - " }" + - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", - logger.entries.get(0).message); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("schema 'test' inherits 'nonesuch', but this schema does not exist", e.getMessage()); - } - } - - @Test - public void testValidationOfSchemaAndDocumentInheritanceConsistency() throws ParseException { - try { - String parent = joinLines( - "schema parent {" + - " document parent {" + - " field pf1 type string {" + - " indexing: summary" + - " }" + - " }" + - "}"); - String child = joinLines( - "schema child inherits parent {" + - " document child {" + - " field cf1 type string {" + - " indexing: summary" + - " }" + - " }" + - "}"); - ApplicationBuilder.createFromStrings(new DeployLoggerStub(), parent, child); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("schema 'child' inherits 'parent', " + - "but its document type does not inherit the parent's document type" - , e.getMessage()); - } - } - - @Test - public void testSchemaInheritance() throws ParseException { - String parentLines = joinLines( - "schema parent {" + - " document parent {" + - " field pf1 type string {" + - " indexing: summary" + - " }" + - " }" + - " fieldset parent_set {" + - " fields: pf1" + - " }" + - " stemming: none" + - " index parent_index {" + - " stemming: best" + - " }" + - " field parent_field type string {" + - " indexing: input pf1 | lowercase | index | attribute | summary" + - " }" + - " rank-profile parent_profile {" + - " }" + - " constant parent_constant {" + - " file: constants/my_constant_tensor_file.json" + - " type: tensor(x{},y{})" + - " }" + - " onnx-model parent_model {" + - " file: models/my_model.onnx" + - " }" + - " document-summary parent_summary {" + - " summary pf1 type string {}" + - " }" + - " import field parentschema_ref.name as parent_imported {}" + - " raw-as-base64-in-summary" + - "}"); - String child1Lines = joinLines( - "schema child1 inherits parent {" + - " document child1 inherits parent {" + - " field c1f1 type string {" + - " indexing: summary" + - " }" + - " }" + - " fieldset child1_set {" + - " fields: c1f1, pf1" + - " }" + - " stemming: shortest" + - " index child1_index {" + - " stemming: shortest" + - " }" + - " field child1_field type string {" + - " indexing: input pf1 | lowercase | index | attribute | summary" + - " }" + - " rank-profile child1_profile inherits parent_profile {" + - " constants {" + - " child1_constant tensor(x{},y{}): file:constants/my_constant_tensor_file.json" + - " }" + - " }" + - " onnx-model child1_model {" + - " file: models/my_model.onnx" + - " }" + - " document-summary child1_summary inherits parent_summary {" + - " summary c1f1 type string {}" + - " }" + - " import field parentschema_ref.name as child1_imported {}" + - "}"); - String child2Lines = joinLines( - "schema child2 inherits parent {" + - " document child2 inherits parent {" + - " field c2f1 type string {" + - " indexing: summary" + - " }" + - " }" + - " fieldset child2_set {" + - " fields: c2f1, pf1" + - " }" + - " stemming: shortest" + - " index child2_index {" + - " stemming: shortest" + - " }" + - " field child2_field type string {" + - " indexing: input pf1 | lowercase | index | attribute | summary" + - " }" + - " rank-profile child2_profile inherits parent_profile {" + - " }" + - " constant child2_constant {" + - " file: constants/my_constant_tensor_file.json" + - " type: tensor(x{},y{})" + - " }" + - " onnx-model child2_model {" + - " file: models/my_model.onnx" + - " }" + - " document-summary child2_summary inherits parent_summary {" + - " summary c2f1 type string {}" + - " }" + - " import field parentschema_ref.name as child2_imported {}" + - "}"); - - ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); - builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist - builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere - builder.addSchema(parentLines); - builder.addSchema(child1Lines); - builder.addSchema(child2Lines); - builder.build(true); - var application = builder.application(); - - var child1 = application.schemas().get("child1"); - assertEquals("pf1", child1.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); - assertEquals("[c1f1, pf1]", child1.fieldSets().userFieldSets().get("child1_set").getFieldNames().toString()); - assertEquals(Stemming.SHORTEST, child1.getStemming()); - assertEquals(Stemming.BEST, child1.getIndex("parent_index").getStemming()); - assertEquals(Stemming.SHORTEST, child1.getIndex("child1_index").getStemming()); - assertNotNull(child1.getField("parent_field")); - assertNotNull(child1.getField("child1_field")); - assertNotNull(child1.getExtraField("parent_field")); - assertNotNull(child1.getExtraField("child1_field")); - assertNotNull(builder.getRankProfileRegistry().get(child1, "parent_profile")); - assertNotNull(builder.getRankProfileRegistry().get(child1, "child1_profile")); - var child1profile = builder.getRankProfileRegistry().get(child1, "child1_profile"); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").inheritedNames().get(0)); - assertNotNull(child1.constants().get(FeatureNames.asConstantFeature("parent_constant"))); - assertNotNull(child1profile.constants().get(FeatureNames.asConstantFeature("child1_constant"))); - assertTrue(child1.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); - assertTrue(child1profile.constants().containsKey(FeatureNames.asConstantFeature("child1_constant"))); - assertTrue(child1profile.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); - assertNotNull(child1.onnxModels().get("parent_model")); - assertNotNull(child1.onnxModels().get("child1_model")); - assertTrue(child1.onnxModels().containsKey("parent_model")); - assertTrue(child1.onnxModels().containsKey("child1_model")); - assertNotNull(child1.getSummary("parent_summary")); - assertNotNull(child1.getSummary("child1_summary")); - assertEquals("parent_summary", child1.getSummary("child1_summary").inherited().get().getName()); - assertTrue(child1.getSummaries().containsKey("parent_summary")); - assertTrue(child1.getSummaries().containsKey("child1_summary")); - assertNotNull(child1.getSummaryField("pf1")); - assertNotNull(child1.getSummaryField("c1f1")); - assertNotNull(child1.getExplicitSummaryField("pf1")); - assertNotNull(child1.getExplicitSummaryField("c1f1")); - assertNotNull(child1.getUniqueNamedSummaryFields().get("pf1")); - assertNotNull(child1.getUniqueNamedSummaryFields().get("c1f1")); - assertNotNull(child1.temporaryImportedFields().get().fields().get("parent_imported")); - assertNotNull(child1.temporaryImportedFields().get().fields().get("child1_imported")); - - var child2 = application.schemas().get("child2"); - assertEquals("pf1", child2.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); - assertEquals("[c2f1, pf1]", child2.fieldSets().userFieldSets().get("child2_set").getFieldNames().toString()); - assertEquals(Stemming.SHORTEST, child2.getStemming()); - assertEquals(Stemming.BEST, child2.getIndex("parent_index").getStemming()); - assertEquals(Stemming.SHORTEST, child2.getIndex("child2_index").getStemming()); - assertNotNull(child2.getField("parent_field")); - assertNotNull(child2.getField("child2_field")); - assertNotNull(child2.getExtraField("parent_field")); - assertNotNull(child2.getExtraField("child2_field")); - assertNotNull(builder.getRankProfileRegistry().get(child2, "parent_profile")); - assertNotNull(builder.getRankProfileRegistry().get(child2, "child2_profile")); - assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").inheritedNames().get(0)); - assertNotNull(child2.constants().get(FeatureNames.asConstantFeature("parent_constant"))); - assertNotNull(child2.constants().get(FeatureNames.asConstantFeature("child2_constant"))); - assertTrue(child2.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); - assertTrue(child2.constants().containsKey(FeatureNames.asConstantFeature("child2_constant"))); - assertNotNull(child2.onnxModels().get("parent_model")); - assertNotNull(child2.onnxModels().get("child2_model")); - assertTrue(child2.onnxModels().containsKey("parent_model")); - assertTrue(child2.onnxModels().containsKey("child2_model")); - assertNotNull(child2.getSummary("parent_summary")); - assertNotNull(child2.getSummary("child2_summary")); - assertEquals("parent_summary", child2.getSummary("child2_summary").inherited().get().getName()); - assertTrue(child2.getSummaries().containsKey("parent_summary")); - assertTrue(child2.getSummaries().containsKey("child2_summary")); - assertNotNull(child2.getSummaryField("pf1")); - assertNotNull(child2.getSummaryField("c2f1")); - assertNotNull(child2.getExplicitSummaryField("pf1")); - assertNotNull(child2.getExplicitSummaryField("c2f1")); - assertNotNull(child2.getUniqueNamedSummaryFields().get("pf1")); - assertNotNull(child2.getUniqueNamedSummaryFields().get("c2f1")); - assertNotNull(child2.temporaryImportedFields().get().fields().get("parent_imported")); - assertNotNull(child2.temporaryImportedFields().get().fields().get("child2_imported")); - DocumentSummary child2DefaultSummary = child2.getSummary("default"); - assertEquals(6, child2DefaultSummary.getSummaryFields().size()); - assertTrue(child2DefaultSummary.getSummaryFields().containsKey("child2_field")); - assertTrue(child2DefaultSummary.getSummaryFields().containsKey("parent_field")); - assertTrue(child2DefaultSummary.getSummaryFields().containsKey("pf1")); - assertTrue(child2DefaultSummary.getSummaryFields().containsKey("c2f1")); - DocumentSummary child2AttributeprefetchSummary = child2.getSummary("attributeprefetch"); - assertEquals(4, child2AttributeprefetchSummary.getSummaryFields().size()); - assertTrue(child2AttributeprefetchSummary.getSummaryFields().containsKey("child2_field")); - assertTrue(child2AttributeprefetchSummary.getSummaryFields().containsKey("parent_field")); - } - - @Test - public void testSchemaInheritanceEmptyChildren() throws ParseException { - String parentLines = joinLines( - "schema parent {" + - " document parent {" + - " field pf1 type string {" + - " indexing: summary" + - " }" + - " }" + - " fieldset parent_set {" + - " fields: pf1" + - " }" + - " stemming: none" + - " index parent_index {" + - " stemming: best" + - " }" + - " field parent_field type string {" + - " indexing: input pf1 | lowercase | index | attribute | summary" + - " }" + - " rank-profile parent_profile {" + - " }" + - " constant parent_constant {" + - " file: constants/my_constant_tensor_file.json" + - " type: tensor(x{},y{})" + - " }" + - " onnx-model parent_model {" + - " file: models/my_model.onnx" + - " }" + - " document-summary parent_summary {" + - " summary pf1 type string {}" + - " }" + - " import field parentschema_ref.name as parent_imported {}" + - " raw-as-base64-in-summary" + - "}"); - String childLines = joinLines( - "schema child inherits parent {" + - " document child inherits parent {" + - " field cf1 type string {" + - " indexing: summary" + - " }" + - " }" + - "}"); - String grandchildLines = joinLines( - "schema grandchild inherits child {" + - " document grandchild inherits child {" + - " field gf1 type string {" + - " indexing: summary" + - " }" + - " }" + - "}"); - - ApplicationBuilder builder = new ApplicationBuilder(new DeployLoggerStub()); - builder.processorsToSkip().add(OnnxModelTypeResolver.class); // Avoid discovering the Onnx model referenced does not exist - builder.processorsToSkip().add(ImportedFieldsResolver.class); // Avoid discovering the document reference leads nowhere - builder.addSchema(parentLines); - builder.addSchema(childLines); - builder.addSchema(grandchildLines); - builder.build(true); - var application = builder.application(); - - assertInheritedFromParent(application.schemas().get("child"), builder.getRankProfileRegistry()); - assertInheritedFromParent(application.schemas().get("grandchild"), builder.getRankProfileRegistry()); - } - - @Test - public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed1() throws ParseException { - try { - String profile = joinLines( - "schema test {" + - " document test {" + - " field title type string {" + - " indexing: summary" + - " }" + - " }" + - " rank-profile r1 {" + - " first-phase {" + - " expression: fieldMatch(title)" + - " }" + - " }" + - " rank-profile r2 {" + - " first-phase {" + - " expression: fieldMatch(title)" + - " }" + - " }" + - " rank-profile r3 inherits r1, r2 {" + - " }" + - "}"); - ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Only one of the profiles inherited by rank profile 'r3' can contain first-phase expression, but it is present in multiple", - e.getMessage()); - } - } - - @Test - public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsAllowedWhenDefinedInChild() throws ParseException { - String profile = joinLines( - "schema test {" + - " document test {" + - " field title type string {" + - " indexing: summary" + - " }" + - " field myFilter type string {" + - " indexing: attribute\n" + - " rank: filter" + - " }" + - " }" + - " rank-profile r1 {" + - " first-phase {" + - " expression: fieldMatch(title)" + - " }" + - " }" + - " rank-profile r2 {" + - " first-phase {" + - " expression: fieldMatch(title)" + - " }" + - " }" + - " rank-profile r3 inherits r1, r2 {" + - " first-phase {" + // Redefined here so this does not cause failure - " expression: nativeRank" + - " }" + - " }" + - "}"); - var builder = ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); - var r3 = builder.getRankProfileRegistry().resolve(builder.application().schemas().get("test").getDocument(), "r3"); - assertEquals(1, r3.allFilterFields().size()); - } - - @Test - public void testInheritingMultipleRankProfilesWithOverlappingConstructsIsDisallowed2() throws ParseException { - try { - String profile = joinLines( - "schema test {" + - " document test {" + - " field title type string {" + - " indexing: summary" + - " }" + - " }" + - " rank-profile r1 {" + - " function f1() {" + - " expression: fieldMatch(title)" + - " }" + - " }" + - " rank-profile r2 {" + - " function f1() {" + - " expression: fieldMatch(title)" + - " }" + - " }" + - " rank-profile r3 inherits r1, r2 {" + - " }" + - "}"); - ApplicationBuilder.createFromStrings(new DeployLoggerStub(), profile); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("rank profile 'r3' inherits rank profile 'r2' which contains function 'f1', but this function is already defined in another profile this inherits", - e.getMessage()); - } - } - - private void assertInheritedFromParent(Schema schema, RankProfileRegistry rankProfileRegistry) { - assertEquals("pf1", schema.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); - assertEquals(Stemming.NONE, schema.getStemming()); - assertEquals(Stemming.BEST, schema.getIndex("parent_index").getStemming()); - assertNotNull(schema.getField("parent_field")); - assertNotNull(schema.getExtraField("parent_field")); - assertNotNull(rankProfileRegistry.get(schema, "parent_profile")); - assertNotNull(schema.constants().get(FeatureNames.asConstantFeature("parent_constant"))); - assertTrue(schema.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); - assertNotNull(schema.onnxModels().get("parent_model")); - assertTrue(schema.onnxModels().containsKey("parent_model")); - assertNotNull(schema.getSummary("parent_summary")); - assertTrue(schema.getSummaries().containsKey("parent_summary")); - assertNotNull(schema.getSummaryField("pf1")); - assertNotNull(schema.getExplicitSummaryField("pf1")); - assertNotNull(schema.getUniqueNamedSummaryFields().get("pf1")); - assertNotNull(schema.temporaryImportedFields().get().fields().get("parent_imported")); - assertTrue(schema.isRawAsBase64()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java deleted file mode 100644 index aec258f2df0..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Stemming settings test - * - * @author bratseth - */ -public class StemmingSettingTestCase extends AbstractSchemaTestCase { - - @Test - public void testStemmingSettings() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingsetting.sd"); - - SDField artist = (SDField) schema.getDocument().getField("artist"); - assertEquals(Stemming.SHORTEST, artist.getStemming(schema)); - - SDField title = (SDField) schema.getDocument().getField("title"); - assertEquals(Stemming.NONE, title.getStemming(schema)); - - SDField song = (SDField) schema.getDocument().getField("song"); - assertEquals(Stemming.MULTIPLE, song.getStemming(schema)); - - SDField track = (SDField) schema.getDocument().getField("track"); - assertEquals(Stemming.SHORTEST, track.getStemming(schema)); - - SDField backward = (SDField) schema.getDocument().getField("backward"); - assertEquals(Stemming.SHORTEST, backward.getStemming(schema)); - - Index defaultIndex = schema.getIndex("default"); - assertEquals(Stemming.SHORTEST, defaultIndex.getStemming()); - } - - @Test - public void requireThatStemmingIsDefaultBest() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); - assertNull(schema.getConcreteField("my_str").getStemming()); - assertEquals(Stemming.BEST, schema.getConcreteField("my_str").getStemming(schema)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java deleted file mode 100755 index 078eccc166f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ /dev/null @@ -1,59 +0,0 @@ - -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.document.config.DocumenttypesConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.derived.Deriver; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Ignore; -import org.junit.Test; -import java.io.IOException; -import static org.junit.Assert.fail; - -/** - * Tests importing of document containing array type fields - * - * @author bratseth - */ -public class StructTestCase extends AbstractSchemaTestCase { - - @Test - public void testStruct() throws IOException { - assertConfigFile("src/test/examples/structresult.cfg", - new DocumentmanagerConfig(Deriver.getDocumentManagerConfig("src/test/examples/struct.sd")) + "\n"); - } - - @Test - public void testBadStruct() throws IOException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/badstruct.sd"); - fail("Should throw exception."); - } catch (IllegalArgumentException|ParseException expected) { - System.err.println("As expected, with message: "+expected.getMessage()); - // success - } - } - - @Test - @Ignore - public void testStructAndDocumentWithSameNames() { - try { - DocumenttypesConfig.Builder dt = Deriver.getDocumentTypesConfig("src/test/examples/structanddocumentwithsamenames.sd"); - // while the above line may work, the config generated will fail. - // See also NameCollisionTestCase. - } catch (Exception e) { - fail("Should not have thrown exception " + e); - } - } - - /** - * Declaring a struct before a document should work - */ - @Test - public void testStructOutsideDocumentLegal() throws IOException, ParseException { - new ApplicationBuilder().addSchemaFile("src/test/examples/structoutsideofdocument.sd"); - } - -} - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java deleted file mode 100644 index 086e3485962..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.model.test.utils.DeployLoggerStub; -import com.yahoo.vespa.objects.FieldBase; -import org.junit.Test; -import static com.yahoo.config.model.test.TestUtil.joinLines; - -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Tests summary validation - * - * @author bratseth - */ -public class SummaryTestCase { - - @Test - public void testMemorySummary() throws ParseException { - String sd = joinLines( - "schema memorysummary {", - " document memorysummary {", - " field inmemory type string {", - " indexing: attribute | summary", - " }", - " field ondisk type string {", - " indexing: index # no summary, so ignored", - " }", - " }", - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromString(sd, logger); - assertTrue(logger.entries.isEmpty()); - } - - @Test - public void testDiskSummary() throws ParseException { - String sd = joinLines( - "schema disksummary {", - " document-summary foobar {", - " summary foo1 type string { source: inmemory }", - " summary foo2 type string { source: ondisk }", - " }", - " document disksummary {", - " field inmemory type string {", - " indexing: attribute | summary", - " }", - " field ondisk type string {", - " indexing: index | summary", - " }", - " }", - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromString(sd, logger); - assertEquals(1, logger.entries.size()); - assertEquals(Level.WARNING, logger.entries.get(0).level); - assertEquals("summary field 'foo2' in document summary 'foobar' references source field 'ondisk', " + - "which is not an attribute: Using this summary will cause disk accesses. " + - "Set 'from-disk' on this summary class to silence this warning.", - logger.entries.get(0).message); - } - - @Test - public void testDiskSummaryExplicit() throws ParseException { - String sd = joinLines( - "schema disksummary {", - " document disksummary {", - " field inmemory type string {", - " indexing: attribute | summary", - " }", - " field ondisk type string {", - " indexing: index | summary", - " }", - " }", - " document-summary foobar {", - " summary foo1 type string { source: inmemory }", - " summary foo2 type string { source: ondisk }", - " from-disk", - " }", - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromString(sd, logger); - assertTrue(logger.entries.isEmpty()); - } - - @Test - public void testStructMemorySummary() throws ParseException { - String sd = joinLines( - "schema structmemorysummary {", - " document structmemorysummary {", - " struct elem {", - " field name type string {}", - " field weight type int {}", - " }", - " field elem_array type array {", - " indexing: summary", - " struct-field name {", - " indexing: attribute", - " }", - " struct-field weight {", - " indexing: attribute", - " }", - " }", - " }", - " document-summary filtered {", - " summary elem_array_filtered type array {", - " source: elem_array", - " matched-elements-only", - " }", - " }", - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromString(sd, logger); - assertTrue(logger.entries.isEmpty()); - } - - @Test - public void testInheritance() throws Exception { - String sd = joinLines( - "schema music {", - " document music {", - " field title type string {", - " indexing: summary | attribute | index", - " }", - " field artist type string {", - " indexing: summary | attribute | index", - " }", - " field album type string {", - " indexing: summary | attribute | index", - " }", - " }", - " document-summary title {", - " summary title type string {", - " source: title", - " }", - " }", - " document-summary title_artist inherits title {", - " summary artist type string {", - " source: artist", - " }", - " }", - " document-summary everything inherits title_artist {", - " summary album type string {", - " source: album", - " }", - " }", - "}"); - var logger = new DeployLoggerStub(); - var search = ApplicationBuilder.createFromString(sd, logger).getSchema(); - assertEquals(List.of(), logger.entries); - - var titleField = "title"; - var artistField = "artist"; - var albumField = "album"; - var titleSummary = search.getSummary(titleField); - var titleArtistSummary = search.getSummary(titleField + "_" + artistField); - var everythingSummary = search.getSummary("everything"); - - var implicitFields = List.of("rankfeatures", "summaryfeatures"); - var tests = List.of( - new TestValue(titleSummary, null, List.of(List.of(titleField), implicitFields)), - new TestValue(titleArtistSummary, titleSummary, List.of(List.of(titleField), implicitFields, List.of(artistField))), - new TestValue(everythingSummary, titleArtistSummary, List.of(List.of(titleField), implicitFields, List.of(artistField, albumField))) - ); - tests.forEach(testValue -> { - var actualFields = testValue.summary.getSummaryFields().values().stream() - .map(FieldBase::getName) - .collect(Collectors.toList()); - assertEquals(testValue.summary.getName() + (testValue.parent == null ? " does not inherit anything" : " inherits " + testValue.parent.getName()), - Optional.ofNullable(testValue.parent), - testValue.summary.inherited()); - assertEquals("Summary " + testValue.summary.getName() + " has expected fields", testValue.fields, actualFields); - }); - } - - @Test - public void testRedeclaringInheritedFieldFails() throws Exception { - String sd = joinLines( - "schema music {", - " document music {", - " field title type string {", - " indexing: summary | attribute | index", - " }", - " field title_short type string {", - " indexing: summary | attribute | index", - " }", - " }", - " document-summary title {", - " summary title type string {", - " source: title", - " }", - " }", - " document-summary title2 inherits title {", - " summary title type string {", - " source: title_short", - " }", - " }", - "}"); - var logger = new DeployLoggerStub(); - try { - ApplicationBuilder.createFromString(sd, logger); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'music', summary class 'title2', summary field 'title': Can not use " + - "source 'title_short' for this summary field, an equally named field in summary class 'title' " + - "uses a different source: 'title'.", e.getMessage()); - } - } - - @Test - public void testValidationOfInheritedSummary() throws ParseException { - try { - String schema = joinLines( - "schema test {" + - " document test {" + - " }" + - " document-summary test_summary inherits nonesuch {" + - " }" + - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("document summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - logger.entries.get(0).message); - // fail("Expected failure"); - } - catch (IllegalArgumentException e) { - // assertEquals("document summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - // e.getMessage()); - } - } - - @Test - public void testInheritingParentSummary() throws ParseException { - String parent = joinLines( - "schema parent {" + - " document parent {" + - " field pf1 type string {" + - " indexing: summary" + - " }" + - " }" + - " document-summary parent_summary {" + - " summary pf1 type string {}" + - " }" + - "}"); - String child = joinLines( - "schema child inherits parent {" + - " document child inherits parent {" + - " field cf1 type string {" + - " indexing: summary" + - " }" + - " }" + - " document-summary child_summary inherits parent_summary {" + - " summary cf1 type string {}" + - " }" + - "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, parent, child); - logger.entries.forEach(e -> System.out.println(e)); - //assertTrue(logger.entries.isEmpty()); - } - - private static class TestValue { - - private final DocumentSummary summary; - private final DocumentSummary parent; - private final List fields; - - public TestValue(DocumentSummary summary, DocumentSummary parent, List> fields) { - this.summary = summary; - this.parent = parent; - this.fields = fields.stream().flatMap(Collection::stream).collect(Collectors.toList());; - } - - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java deleted file mode 100644 index f2d78796553..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class UrlFieldValidationTestCase { - - @Test - public void requireThatInheritedRiseFieldsStillCanBeInConflictButDontThrowException() throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema("search test {" + - " document test { " + - " field a type uri { indexing: attribute | summary }" + - " }" + - "}"); - try { - builder.build(true); - fail("Should have caused an exception"); - // success - } catch (IllegalArgumentException e) { - assertEquals("Error in field 'a' in schema 'test': uri type fields cannot be attributes", - Exceptions.toMessageString(e)); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java deleted file mode 100644 index 615f6be5b4f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.concurrent.InThreadExecutorService; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.document.config.DocumenttypesConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; - -import java.io.File; -import java.io.IOException; - -/** - * Superclass of tests needing file comparisons - * - * @author bratseth - */ -public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { - - private static final String tempDir = "temp/"; - private static final String schemaRoot = "src/test/derived/"; - - private DerivedConfiguration derive(String dirName, - String schemaName, - TestProperties properties, - DeployLogger logger) throws IOException, ParseException { - File toDir = new File(tempDir + dirName); - toDir.mkdirs(); - deleteContent(toDir); - - ApplicationBuilder builder = ApplicationBuilder.createFromDirectory(schemaRoot + dirName + "/", - new MockFileRegistry(), - logger, - properties); - return derive(dirName, schemaName, properties, builder, logger); - } - - private DerivedConfiguration derive(String dirName, - String schemaName, - TestProperties properties, - ApplicationBuilder builder, - DeployLogger logger) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(schemaName), - new DeployState.Builder().properties(properties) - .deployLogger(logger) - .rankProfileRegistry(builder.getRankProfileRegistry()) - .queryProfiles(builder.getQueryProfileRegistry()) - .build()); - return export(dirName, builder, config); - } - - DerivedConfiguration derive(String dirName, ApplicationBuilder builder, Schema schema) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(schema, - builder.getRankProfileRegistry(), - builder.getQueryProfileRegistry()); - return export(dirName, builder, config); - } - - private DerivedConfiguration export(String name, ApplicationBuilder builder, DerivedConfiguration config) throws IOException { - String path = exportConfig(name, config); - DerivedConfiguration.exportDocuments(new DocumentManager() - .produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); - DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); - DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); - config.exportConstants(path); - return config; - } - - private String exportConfig(String name, DerivedConfiguration config) throws IOException { - String path = tempDir + name; - config.export(path); - return path; - } - - /** - * Derives a config from name/name.sd below the test dir and verifies that every .cfg file in name/ has a - * corresponding file with the same content in temp/name. Versions can and should be omitted from the .cfg file - * names. This will fail if the search definition dir has multiple search definitions. - * - * @param dirName the name of the directory containing the searchdef file to verify - * @throws ParseException if the .sd file could not be parsed - * @throws IOException if file access failed - */ - protected DerivedConfiguration assertCorrectDeriving(String dirName) throws IOException, ParseException { - return assertCorrectDeriving(dirName, new TestableDeployLogger()); - } - protected DerivedConfiguration assertCorrectDeriving(String dirName, DeployLogger logger) throws IOException, ParseException { - return assertCorrectDeriving(dirName, null, logger); - } - - protected DerivedConfiguration assertCorrectDeriving(String dirName, - String searchDefinitionName, - DeployLogger logger) throws IOException, ParseException { - return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger); - } - - protected DerivedConfiguration assertCorrectDeriving(String dirName, - TestProperties properties) throws IOException, ParseException { - return assertCorrectDeriving(dirName, null, properties, new TestableDeployLogger()); - } - - protected DerivedConfiguration assertCorrectDeriving(String dirName, - String schemaName, - TestProperties properties, - DeployLogger logger) throws IOException, ParseException { - DerivedConfiguration derived = derive(dirName, schemaName, properties, logger); - assertCorrectConfigFiles(dirName); - return derived; - } - - /** - * Asserts config is correctly derived given a builder. - * This will fail if the builder contains multiple search definitions. - */ - protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, String dirName, DeployLogger logger) throws IOException { - builder.build(true); - DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger); - assertCorrectConfigFiles(dirName); - return derived; - } - - protected DerivedConfiguration assertCorrectDeriving(ApplicationBuilder builder, Schema schema, String name) throws IOException { - DerivedConfiguration derived = derive(name, builder, schema); - assertCorrectConfigFiles(name); - return derived; - } - - /** - * Assert that search is derived into the files in the directory given by name. - * - * @param name the local name of the directory containing the files to check - * @throws IOException if file access failed - */ - void assertCorrectConfigFiles(String name) throws IOException { - File[] files = new File(schemaRoot, name).listFiles(); - if (files == null) return; - for (File file : files) { - if ( ! file.getName().endsWith(".cfg")) continue; - boolean orderMatters = file.getName().equals("ilscripts.cfg"); - assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName(), orderMatters); - } - } - - static void assertEqualFiles(String correctFileName, String checkFileName, boolean orderMatters) throws IOException { - // Set updateOnAssert to true if you want update the files with correct answer. - assertConfigFiles(correctFileName, checkFileName, orderMatters, false); - } - - void deleteContent(File dir) { - File[] files = dir.listFiles(); - if (files == null) return; - - for (File file : files) { - file.delete(); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java deleted file mode 100755 index bfbf539074c..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author Einar M R Rosenvinge - */ -public class AnnotationsTestCase extends AbstractExportingTestCase { - - @Test - public void requireThatStructRegistersIfOnlyUsedByAnnotation() throws IOException, ParseException { - assertCorrectDeriving("annotationsstruct"); - } - - @Test - public void requireThatStructRegistersIfOnlyUsedAsArrayByAnnotation() throws IOException, ParseException { - assertCorrectDeriving("annotationsstructarray"); - } - - @Test - public void testSimpleAnnotationDeriving() throws IOException, ParseException { - assertCorrectDeriving("annotationssimple"); - } - - @Test - public void testAnnotationDerivingWithImplicitStruct() throws IOException, ParseException { - assertCorrectDeriving("annotationsimplicitstruct"); - } - - @Test - public void testAnnotationDerivingInheritance() throws IOException, ParseException { - assertCorrectDeriving("annotationsinheritance"); - } - - @Test - public void testAnnotationDerivingInheritance2() throws IOException, ParseException { - assertCorrectDeriving("annotationsinheritance2"); - } - - @Test - public void testSimpleReference() throws IOException, ParseException { - assertCorrectDeriving("annotationsreference"); - } - - @Test - public void testAdvancedReference() throws IOException, ParseException { - assertCorrectDeriving("annotationsreference2"); - } - - @Test - public void testAnnotationsPolymorphy() throws IOException, ParseException { - assertCorrectDeriving("annotationspolymorphy"); - } - - /** - * An annotation declared before document {} should work. - */ - @Test - public void testAnnotationOutsideOfDocumentNew() throws IOException, ParseException { - assertCorrectDeriving("annotationsoutsideofdocument"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java deleted file mode 100644 index c2bd86ba5f7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests array type deriving. Indexing statements over array - * types is not yet supported, so this tests document type - * configuration deriving only. Expand later. - * - * @author bratseth - */ -public class ArraysTestCase extends AbstractExportingTestCase { - - @Test - public void testDocumentDeriving() throws IOException, ParseException { - assertCorrectDeriving("arrays"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java deleted file mode 100644 index a272ab14dad..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; -import java.util.Iterator; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Tests attribute deriving - * - * @author bratseth - */ -public class AttributeListTestCase extends AbstractSchemaTestCase { - - @Test - public void testDeriving() throws IOException, ParseException { - // Test attribute importing - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); - - // Test attribute deriving - AttributeFields attributeFields = new AttributeFields(schema); - Iterator attributes = attributeFields.attributeIterator(); - Attribute attribute; - attribute = (Attribute)attributes.next(); - assertEquals("popularity", attribute.getName()); - assertEquals(Attribute.Type.INTEGER, attribute.getType()); - assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); - - attribute = (Attribute)attributes.next(); - assertEquals("measurement", attribute.getName()); - assertEquals(Attribute.Type.INTEGER, attribute.getType()); - assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); - - attribute = (Attribute)attributes.next(); - assertEquals("smallattribute", attribute.getName()); - assertEquals(Attribute.Type.BYTE, attribute.getType()); - assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); - - attribute = (Attribute)attributes.next(); - assertEquals("access", attribute.getName()); - assertEquals(Attribute.Type.BYTE, attribute.getType()); - assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); - - attribute = (Attribute)attributes.next(); - assertEquals("category_arr", attribute.getName()); - assertEquals(Attribute.Type.STRING, attribute.getType()); - assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); - - attribute = (Attribute)attributes.next(); - assertEquals("measurement_arr", attribute.getName()); - assertEquals(Attribute.Type.INTEGER, attribute.getType()); - assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); - - attribute = (Attribute)attributes.next(); - assertEquals("popsiness", attribute.getName()); - assertEquals(Attribute.Type.INTEGER, attribute.getType()); - assertEquals(Attribute.CollectionType.SINGLE, attribute.getCollectionType()); - - assertFalse(attributes.hasNext()); - } - - @Test - public void fields_in_array_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); - Iterator attributes = new AttributeFields(schema).attributeIterator(); - - assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); - assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); - assertFalse(attributes.hasNext()); - } - - @Test - public void fields_in_map_of_struct_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); - Iterator attributes = new AttributeFields(schema).attributeIterator(); - - assertAttribute("str_elem_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); - assertAttribute("str_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, false, attributes.next()); - assertAttribute("str_elem_map.value.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); - assertAttribute("int_elem_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); - assertAttribute("int_elem_map.value.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); - assertFalse(attributes.hasNext()); - } - - private static void assertAttribute(String name, Attribute.Type type, Attribute.CollectionType collection, boolean isFastSearch, Attribute attr) { - assertEquals(name, attr.getName()); - assertEquals(type, attr.getType()); - assertEquals(collection, attr.getCollectionType()); - assertEquals(isFastSearch, attr.isFastSearch()); - } - - @Test - public void only_zcurve_attribute_is_derived_from_array_of_position_field() throws ParseException { - Schema schema = ApplicationBuilder.createFromString( - joinLines("search test {", - " document test {", - " field pos_array type array {", - " indexing: attribute", - " }", - " }", - "}")).getSchema(); - Iterator attributes = new AttributeFields(schema).attributeIterator(); - - assertAttribute("pos_array_zcurve", Attribute.Type.LONG, Attribute.CollectionType.ARRAY, true, attributes.next()); - assertFalse(attributes.hasNext()); - } - - @Test - public void fields_in_map_of_primitive_are_derived_into_array_attributes() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_attribute/test.sd"); - Iterator attributes = new AttributeFields(schema).attributeIterator(); - - assertAttribute("str_map.key", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next()); - assertAttribute("str_map.value", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, false, attributes.next()); - assertAttribute("int_map.key", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next()); - assertFalse(attributes.hasNext()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java deleted file mode 100644 index 2b345bdb158..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests attribute settings - * - * @author bratseth - */ -public class AttributesTestCase extends AbstractExportingTestCase { - - @Test - public void testDocumentDeriving() throws IOException, ParseException { - assertCorrectDeriving("attributes"); - } - - @Test - public void testArrayOfStructAttribute() throws IOException, ParseException { - assertCorrectDeriving("array_of_struct_attribute"); - } - - @Test - public void testMapOfStructAttribute() throws IOException, ParseException { - assertCorrectDeriving("map_of_struct_attribute"); - } - - @Test - public void testMapOfPrimitiveAttribute() throws IOException, ParseException { - assertCorrectDeriving("map_attribute"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java deleted file mode 100644 index ee9a9eac02c..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; - -/** - * Correct casing for derived attributes - * - * @author vegardh - */ -public class CasingTestCase extends AbstractSchemaTestCase { - - @Test - public void testCasing() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/casing.sd"); - assertEquals(schema.getIndex("color").getName(), "color"); - assertEquals(schema.getIndex("Foo").getName(), "Foo"); - assertEquals(schema.getIndex("Price").getName(), "Price"); - assertEquals(schema.getAttribute("artist").getName(), "artist"); - assertEquals(schema.getAttribute("Drummer").getName(), "Drummer"); - assertEquals(schema.getAttribute("guitarist").getName(), "guitarist"); - assertEquals(schema.getAttribute("title").getName(), "title"); - assertEquals(schema.getAttribute("Trumpetist").getName(), "Trumpetist"); - assertEquals(schema.getAttribute("Saxophonist").getName(), "Saxophonist"); - assertEquals(schema.getAttribute("TenorSaxophonist").getName(), "TenorSaxophonist"); - assertEquals(schema.getAttribute("Flutist").getName(), "Flutist"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSchemaTestCase.java deleted file mode 100644 index bb339688fcb..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSchemaTestCase.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests deriving a configuration with multiple summaries - * - * @author bratseth - */ -public class CombinedAttributeAndIndexSchemaTestCase extends AbstractExportingTestCase { - - @Test - public void testMultipleSummaries() throws IOException, ParseException { - assertCorrectDeriving("combinedattributeandindexsearch"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java deleted file mode 100644 index b1fb1962fc8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * Tests deriving using the Deriver facade - * - * @author bratseth - */ -public class DeriverTestCase extends AbstractSchemaTestCase { - - @Test - public void testDeriveDocManager() { - DocumentTypeManager dtm = new DocumentTypeManager(new DocumentmanagerConfig( - Deriver.getDocumentManagerConfig(List.of( - "src/test/derived/deriver/child.sd", - "src/test/derived/deriver/parent.sd", - "src/test/derived/deriver/grandparent.sd")))); - assertEquals(dtm.getDocumentType("child").getField("a").getDataType(), DataType.STRING); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DuplicateStructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DuplicateStructTestCase.java deleted file mode 100644 index ff68e47a1e9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DuplicateStructTestCase.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import org.junit.Test; - -/** - * @author arnej - */ -public class DuplicateStructTestCase extends AbstractExportingTestCase { - - @Test - public void exact_duplicate_struct_works() throws Exception { - assertCorrectDeriving("duplicate_struct", "foobar", - new TestProperties(), - new TestableDeployLogger()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java deleted file mode 100644 index a3123550efa..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import org.junit.Test; - -/** - * Tests deriving rank for files from search definitions - * - * @author bratseth - */ -public class EmptyRankProfileTestCase extends AbstractSchemaTestCase { - - @Test - public void testDeriving() { - Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - SDDocumentType doc = new SDDocumentType("test"); - schema.addDocument(doc); - doc.addField(new SDField(doc, "a", DataType.STRING)); - SDField field = new SDField(doc, "b", DataType.STRING); - field.setLiteralBoost(500); - doc.addField(field); - doc.addField(new SDField(doc, "c", DataType.STRING)); - - schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); - new DerivedConfiguration(schema, rankProfileRegistry); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java deleted file mode 100644 index 45e65bb90f9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author bratseth - */ -public class ExactMatchTestCase extends AbstractExportingTestCase { - @Test - public void testExactString() throws IOException, ParseException { - assertCorrectDeriving("exactmatch"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java deleted file mode 100644 index ac97863b7a7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; - -/** - * Tests exporting - * - * @author bratseth - */ -public class ExportingTestCase extends AbstractExportingTestCase { - - @Test - public void testIndexInfoLowerCase() throws IOException, ParseException { - assertCorrectDeriving("indexinfo_lowercase"); - } - - @Test - public void testPositionArray() throws IOException, ParseException { - assertCorrectDeriving("position_array", - new TestProperties().setUseV8GeoPositions(true)); - } - - @Test - public void testPositionAttribute() throws IOException, ParseException { - assertCorrectDeriving("position_attribute", - new TestProperties().setUseV8GeoPositions(true)); - } - - @Test - public void testPositionExtra() throws IOException, ParseException { - assertCorrectDeriving("position_extra", - new TestProperties().setUseV8GeoPositions(true)); - } - - @Test - public void testPositionNoSummary() throws IOException, ParseException { - assertCorrectDeriving("position_nosummary", - new TestProperties().setUseV8GeoPositions(true)); - } - - @Test - public void testPositionSummary() throws IOException, ParseException { - assertCorrectDeriving("position_summary", - new TestProperties().setUseV8GeoPositions(true)); - } - - @Test - public void testUriArray() throws IOException, ParseException { - assertCorrectDeriving("uri_array"); - } - - @Test - public void testUriWSet() throws IOException, ParseException { - assertCorrectDeriving("uri_wset"); - } - - @Test - public void testMusic() throws IOException, ParseException { - assertCorrectDeriving("music"); - } - - @Test - public void testComplexPhysicalExporting() throws IOException, ParseException { - assertCorrectDeriving("complex"); - } - - @Test - public void testAttributePrefetch() throws IOException, ParseException { - assertCorrectDeriving("attributeprefetch"); - } - - @Test - public void testAdvancedIL() throws IOException, ParseException { - assertCorrectDeriving("advanced"); - } - - @Test - public void testEmptyDefaultIndex() throws IOException, ParseException { - assertCorrectDeriving("emptydefault"); - } - - @Test - public void testIndexSwitches() throws IOException, ParseException { - assertCorrectDeriving("indexswitches"); - } - - @Test - public void testRankTypes() throws IOException, ParseException { - assertCorrectDeriving("ranktypes"); - } - - @Test - public void testAttributeRank() throws IOException, ParseException { - assertCorrectDeriving("attributerank"); - } - - @Test - public void testNewRank() throws IOException, ParseException { - assertCorrectDeriving("newrank"); - } - - @Test - public void testRankingExpression() throws IOException, ParseException { - assertCorrectDeriving("rankingexpression"); - } - - @Test - public void testAvoidRenamingRankingExpression() throws IOException, ParseException { - assertCorrectDeriving("renamedfeatures", "foo", - new TestProperties().setAvoidRenamingSummaryFeatures(true), - new TestableDeployLogger()); - } - - @Test - public void testMlr() throws IOException, ParseException { - assertCorrectDeriving("mlr"); - } - - @Test - public void testMusic3() throws IOException, ParseException { - assertCorrectDeriving("music3"); - } - - @Test - public void testIndexSchema() throws IOException, ParseException { - assertCorrectDeriving("indexschema"); - } - - @Test - public void testIndexinfoFieldsets() throws IOException, ParseException { - assertCorrectDeriving("indexinfo_fieldsets"); - } - - @Test - public void testStreamingJuniper() throws IOException, ParseException { - assertCorrectDeriving("streamingjuniper"); - } - - @Test - public void testPredicateAttribute() throws IOException, ParseException { - assertCorrectDeriving("predicate_attribute"); - } - - @Test - public void testTensor() throws IOException, ParseException { - assertCorrectDeriving("tensor"); - } - - @Test - public void testTensor2() throws IOException, ParseException { - String dir = "src/test/derived/tensor2/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "first.sd"); - builder.addSchemaFile(dir + "second.sd"); - builder.build(true); - derive("tensor2", builder, builder.getSchema("second")); - assertCorrectConfigFiles("tensor2"); - } - - @Test - public void testHnswIndex() throws IOException, ParseException { - assertCorrectDeriving("hnsw_index"); - } - - @Test - public void testRankProfileInheritance() throws IOException, ParseException { - assertCorrectDeriving("rankprofileinheritance", "child", new TestableDeployLogger()); - } - - @Test - public void testLanguage() throws IOException, ParseException { - TestableDeployLogger logger = new TestableDeployLogger(); - assertCorrectDeriving("language", logger); - assertEquals(0, logger.warnings.size()); - } - - @Test - public void testRankProfileModularity() throws IOException, ParseException { - assertCorrectDeriving("rankprofilemodularity"); - } - - @Test - public void testStructAndFieldSet() throws IOException, ParseException { - assertCorrectDeriving("structandfieldset"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExpressionsAsArgsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExpressionsAsArgsTestCase.java deleted file mode 100644 index 733786e2c71..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExpressionsAsArgsTestCase.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests correct deriving of expressions as arguments to functions. - * - * @author lesters - */ -public class ExpressionsAsArgsTestCase extends AbstractExportingTestCase { - - @Test - public void testDocumentDeriving() throws IOException, ParseException { - assertCorrectDeriving("function_arguments"); - assertCorrectDeriving("function_arguments_with_expressions"); - } - -} - - - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java deleted file mode 100644 index 3877b351ac8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -public class FieldsetTestCase extends AbstractExportingTestCase { - - @Test - public void testRankProfiles() throws IOException, ParseException { - assertCorrectDeriving("fieldset"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java deleted file mode 100644 index 207792ffe06..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.collections.Pair; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author bratseth - */ -public class GeminiTestCase extends AbstractExportingTestCase { - - @Test - public void testRanking2() throws IOException, ParseException { - DerivedConfiguration c = assertCorrectDeriving("gemini2"); - RawRankProfile p = c.getRankProfileList().getRankProfiles().get("test"); - Map ranking = removePartKeySuffixes(asMap(p.configProperties())); - assertEquals("attribute(right)", resolve(lookup("toplevel", ranking), ranking)); - } - - private Map asMap(List> properties) { - Map map = new HashMap<>(); - for (Pair property : properties) - map.put(property.getFirst(), property.getSecond()); - return map; - } - - private Map removePartKeySuffixes(Map p) { - Map pWithoutSuffixes = new HashMap<>(); - for (Map.Entry entry : p.entrySet()) - pWithoutSuffixes.put(removePartSuffix(entry.getKey()), entry.getValue()); - return pWithoutSuffixes; - } - - private String removePartSuffix(String s) { - int partIndex = s.indexOf(".part"); - if (partIndex <= 0) return s; - return s.substring(0, partIndex); - } - - /** - * Recursively resolves references to other ranking expressions - rankingExpression(name) - - * and replaces the reference by the expression - */ - private String resolve(String expression, Map ranking) { - int referenceStartIndex; - while ((referenceStartIndex = expression.indexOf("rankingExpression(")) >= 0) { - int referenceEndIndex = expression.indexOf(")", referenceStartIndex); - expression = expression.substring(0, referenceStartIndex) + - resolve(lookup(expression.substring(referenceStartIndex + "rankingExpression(".length(), referenceEndIndex), ranking), ranking) + - expression.substring(referenceEndIndex + 1); - } - return expression; - } - - private String lookup(String expressionName, Map ranking) { - String value = ranking.get("rankingExpression(" + expressionName + ").rankingScript"); - if (value == null) { - return expressionName; - } - return value; - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java deleted file mode 100644 index ba485b7b96b..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.util.Set; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -/** - * Tests that documents ids are treated as they should - * - * @author bratseth - */ -public class IdTestCase extends AbstractExportingTestCase { - - @Test - public void testExplicitUpperCaseIdField() { - Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); - SDDocumentType document = new SDDocumentType("test"); - schema.addDocument(document); - SDField uri = new SDField(document, "URI", DataType.URI); - uri.parseIndexingScript("{ summary | index }"); - document.addField(uri); - - new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), - true, false, Set.of()); - - assertNull(document.getField("uri")); - assertNull(document.getField("Uri")); - assertNotNull(document.getField("URI")); - } - - @Test - public void testCompleteDeriving() throws Exception { - assertCorrectDeriving("id"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java deleted file mode 100644 index 5eeae347c88..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author geirst - */ -public class ImportedFieldsTestCase extends AbstractExportingTestCase { - - @Test - public void configs_for_imported_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("importedfields", "child", new TestableDeployLogger()); - } - - @Test - public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_struct_fields", "child", - new TestProperties(), - new TestableDeployLogger()); - } - - @Test - public void configs_for_imported_position_field_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_position_field", "child", new TestableDeployLogger()); - } - - @Test - public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_position_field_summary", "child", new TestableDeployLogger()); - } - - @Test - public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException { - assertCorrectDeriving("imported_fields_inherited_reference", "child_c", new TestableDeployLogger()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java deleted file mode 100644 index 17f85840af8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.document.StructDataType; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author Simon Thoresen Hult - */ -public class IndexSchemaTestCase { - - @Test - public void requireThatPrimitiveIsNotFlattened() { - assertFlat(new Field("foo", DataType.BYTE), new Field("foo", DataType.BYTE)); - assertFlat(new Field("foo", DataType.DOUBLE), new Field("foo", DataType.DOUBLE)); - assertFlat(new Field("foo", DataType.FLOAT), new Field("foo", DataType.FLOAT)); - assertFlat(new Field("foo", DataType.INT), new Field("foo", DataType.INT)); - assertFlat(new Field("foo", DataType.LONG), new Field("foo", DataType.LONG)); - assertFlat(new Field("foo", DataType.RAW), new Field("foo", DataType.RAW)); - assertFlat(new Field("foo", DataType.STRING), new Field("foo", DataType.STRING)); - assertFlat(new Field("foo", DataType.URI), new Field("foo", DataType.URI)); - assertFlat(new Field("foo", DataType.PREDICATE), new Field("foo", DataType.PREDICATE)); - } - - @Test - public void requireThatArrayOfPrimitiveIsNotFlattened() { - assertFlat(new Field("foo", DataType.getArray(DataType.BYTE)), - new Field("foo", DataType.getArray(DataType.BYTE))); - assertFlat(new Field("foo", DataType.getArray(DataType.DOUBLE)), - new Field("foo", DataType.getArray(DataType.DOUBLE))); - assertFlat(new Field("foo", DataType.getArray(DataType.FLOAT)), - new Field("foo", DataType.getArray(DataType.FLOAT))); - assertFlat(new Field("foo", DataType.getArray(DataType.INT)), - new Field("foo", DataType.getArray(DataType.INT))); - assertFlat(new Field("foo", DataType.getArray(DataType.LONG)), - new Field("foo", DataType.getArray(DataType.LONG))); - assertFlat(new Field("foo", DataType.getArray(DataType.RAW)), - new Field("foo", DataType.getArray(DataType.RAW))); - assertFlat(new Field("foo", DataType.getArray(DataType.STRING)), - new Field("foo", DataType.getArray(DataType.STRING))); - assertFlat(new Field("foo", DataType.getArray(DataType.URI)), - new Field("foo", DataType.getArray(DataType.URI))); - assertFlat(new Field("foo", DataType.getArray(DataType.PREDICATE)), - new Field("foo", DataType.getArray(DataType.PREDICATE))); - } - - @Test - public void requireThatStructIsFlattened() { - StructDataType type = new StructDataType("my_struct"); - type.addField(new Field("my_byte", DataType.BYTE)); - type.addField(new Field("my_double", DataType.DOUBLE)); - type.addField(new Field("my_float", DataType.FLOAT)); - type.addField(new Field("my_int", DataType.INT)); - type.addField(new Field("my_long", DataType.LONG)); - type.addField(new Field("my_raw", DataType.RAW)); - type.addField(new Field("my_string", DataType.STRING)); - type.addField(new Field("my_uri", DataType.URI)); - - assertFlat(new Field("foo", type), - new Field("foo.my_byte", DataType.BYTE), - new Field("foo.my_double", DataType.DOUBLE), - new Field("foo.my_float", DataType.FLOAT), - new Field("foo.my_int", DataType.INT), - new Field("foo.my_long", DataType.LONG), - new Field("foo.my_raw", DataType.RAW), - new Field("foo.my_string", DataType.STRING), - new Field("foo.my_uri", DataType.URI)); - } - - @Test - public void requireThatArrayOfStructIsFlattened() { - StructDataType type = new StructDataType("my_struct"); - type.addField(new Field("my_byte", DataType.BYTE)); - type.addField(new Field("my_double", DataType.DOUBLE)); - type.addField(new Field("my_float", DataType.FLOAT)); - type.addField(new Field("my_int", DataType.INT)); - type.addField(new Field("my_long", DataType.LONG)); - type.addField(new Field("my_raw", DataType.RAW)); - type.addField(new Field("my_string", DataType.STRING)); - type.addField(new Field("my_uri", DataType.URI)); - - assertFlat(new Field("foo", DataType.getArray(type)), - new Field("foo.my_byte", DataType.getArray(DataType.BYTE)), - new Field("foo.my_double", DataType.getArray(DataType.DOUBLE)), - new Field("foo.my_float", DataType.getArray(DataType.FLOAT)), - new Field("foo.my_int", DataType.getArray(DataType.INT)), - new Field("foo.my_long", DataType.getArray(DataType.LONG)), - new Field("foo.my_raw", DataType.getArray(DataType.RAW)), - new Field("foo.my_string", DataType.getArray(DataType.STRING)), - new Field("foo.my_uri", DataType.getArray(DataType.URI))); - } - - @Test - public void requireThatArrayOfArrayOfStructIsFlattened() { - StructDataType type = new StructDataType("my_struct"); - type.addField(new Field("my_byte", DataType.BYTE)); - type.addField(new Field("my_double", DataType.DOUBLE)); - type.addField(new Field("my_float", DataType.FLOAT)); - type.addField(new Field("my_int", DataType.INT)); - type.addField(new Field("my_long", DataType.LONG)); - type.addField(new Field("my_raw", DataType.RAW)); - type.addField(new Field("my_string", DataType.STRING)); - type.addField(new Field("my_uri", DataType.URI)); - - assertFlat(new Field("foo", DataType.getArray(DataType.getArray(type))), - new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), - new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), - new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), - new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), - new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), - new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), - new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), - new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); - } - - @Test - public void requireThatStructWithArrayFieldIsFlattened() { - StructDataType type = new StructDataType("my_struct"); - type.addField(new Field("my_byte", DataType.getArray(DataType.BYTE))); - type.addField(new Field("my_double", DataType.getArray(DataType.DOUBLE))); - type.addField(new Field("my_float", DataType.getArray(DataType.FLOAT))); - type.addField(new Field("my_int", DataType.getArray(DataType.INT))); - type.addField(new Field("my_long", DataType.getArray(DataType.LONG))); - type.addField(new Field("my_raw", DataType.getArray(DataType.RAW))); - type.addField(new Field("my_string", DataType.getArray(DataType.STRING))); - type.addField(new Field("my_uri", DataType.getArray(DataType.URI))); - - assertFlat(new Field("foo", type), - new Field("foo.my_byte", DataType.getArray(DataType.BYTE)), - new Field("foo.my_double", DataType.getArray(DataType.DOUBLE)), - new Field("foo.my_float", DataType.getArray(DataType.FLOAT)), - new Field("foo.my_int", DataType.getArray(DataType.INT)), - new Field("foo.my_long", DataType.getArray(DataType.LONG)), - new Field("foo.my_raw", DataType.getArray(DataType.RAW)), - new Field("foo.my_string", DataType.getArray(DataType.STRING)), - new Field("foo.my_uri", DataType.getArray(DataType.URI))); - } - - @Test - public void requireThatStructWithArrayOfArrayFieldIsFlattened() { - StructDataType type = new StructDataType("my_struct"); - type.addField(new Field("my_byte", DataType.getArray(DataType.getArray(DataType.BYTE)))); - type.addField(new Field("my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE)))); - type.addField(new Field("my_float", DataType.getArray(DataType.getArray(DataType.FLOAT)))); - type.addField(new Field("my_int", DataType.getArray(DataType.getArray(DataType.INT)))); - type.addField(new Field("my_long", DataType.getArray(DataType.getArray(DataType.LONG)))); - type.addField(new Field("my_raw", DataType.getArray(DataType.getArray(DataType.RAW)))); - type.addField(new Field("my_string", DataType.getArray(DataType.getArray(DataType.STRING)))); - type.addField(new Field("my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); - - assertFlat(new Field("foo", type), - new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), - new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), - new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), - new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), - new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), - new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), - new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), - new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); - } - - @Test - public void requireThatArrayOfStructWithArrayFieldIsFlattened() { - StructDataType type = new StructDataType("my_struct"); - type.addField(new Field("my_byte", DataType.getArray(DataType.BYTE))); - type.addField(new Field("my_double", DataType.getArray(DataType.DOUBLE))); - type.addField(new Field("my_float", DataType.getArray(DataType.FLOAT))); - type.addField(new Field("my_int", DataType.getArray(DataType.INT))); - type.addField(new Field("my_long", DataType.getArray(DataType.LONG))); - type.addField(new Field("my_raw", DataType.getArray(DataType.RAW))); - type.addField(new Field("my_string", DataType.getArray(DataType.STRING))); - type.addField(new Field("my_uri", DataType.getArray(DataType.URI))); - - assertFlat(new Field("foo", DataType.getArray(type)), - new Field("foo.my_byte", DataType.getArray(DataType.getArray(DataType.BYTE))), - new Field("foo.my_double", DataType.getArray(DataType.getArray(DataType.DOUBLE))), - new Field("foo.my_float", DataType.getArray(DataType.getArray(DataType.FLOAT))), - new Field("foo.my_int", DataType.getArray(DataType.getArray(DataType.INT))), - new Field("foo.my_long", DataType.getArray(DataType.getArray(DataType.LONG))), - new Field("foo.my_raw", DataType.getArray(DataType.getArray(DataType.RAW))), - new Field("foo.my_string", DataType.getArray(DataType.getArray(DataType.STRING))), - new Field("foo.my_uri", DataType.getArray(DataType.getArray(DataType.URI)))); - } - - private static void assertFlat(Field fieldToFlatten, Field... expectedFields) { - List actual = new LinkedList<>(IndexSchema.flattenField(fieldToFlatten)); - List expected = new LinkedList<>(Arrays.asList(expectedFields)); - Collections.sort(actual); - Collections.sort(expected); - for (Field field : actual) { - if (!expected.remove(field)) { - fail("Unexpected field: " + field); - } - } - assertTrue("Missing fields: " + expected, expected.isEmpty()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java deleted file mode 100644 index 9f691be2956..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.junit.rules.TemporaryFolder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -/** - * Tests inheritance - * - * @author bratseth - */ -public class InheritanceTestCase extends AbstractExportingTestCase { - - @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); - - @Test - public void requireThatIndexedStructFieldCanBeInherited() throws IOException, ParseException { - String dir = "src/test/derived/inheritstruct/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "parent.sd"); - builder.addSchemaFile(dir + "child.sd"); - builder.build(true); - derive("inheritstruct", builder, builder.getSchema("child")); - assertCorrectConfigFiles("inheritstruct"); - } - - @Test - public void requireThatInheritFromNullIsCaught() throws IOException, ParseException { - try { - assertCorrectDeriving("inheritfromnull"); - } catch (IllegalArgumentException e) { - assertEquals("document inheritfromnull inherits from unavailable document foo", e.getMessage()); - } - } - - @Test - public void requireThatStructTypesAreInheritedThroughDiamond() throws IOException, ParseException { - String dir = "src/test/derived/inheritdiamond/"; - { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "grandparent.sd"); - builder.addSchemaFile(dir + "mother.sd"); - builder.addSchemaFile(dir + "father.sd"); - builder.addSchemaFile(dir + "child.sd"); - builder.build(true); - derive("inheritdiamond", builder, builder.getSchema("child")); - assertCorrectConfigFiles("inheritdiamond"); - } - List files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); - File outDir = tmpDir.newFolder("out"); - for (int startIdx = 0; startIdx < files.size(); ++startIdx) { - var builder = new ApplicationBuilder(new TestProperties()); - for (int fileIdx = startIdx; fileIdx < startIdx + files.size(); ++fileIdx) { - String fileName = files.get(fileIdx % files.size()); - builder.addSchemaFile(dir + fileName); - } - builder.build(true); - DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); - DerivedConfiguration.exportDocuments(new DocumentManager(). - produce(builder.getModel(), b), outDir.getPath()); - DocumentmanagerConfig dc = b.build(); - assertEquals(5, dc.doctype().size()); - - assertNull(structType("child.body", dc)); - var childHeader = structType("child.header", dc); - assertEquals(childHeader.field(0).name(), "foo"); - assertEquals(childHeader.field(1).name(), "bar"); - assertEquals(childHeader.field(2).name(), "baz"); - assertEquals(childHeader.field(3).name(), "cox"); - - var root = documentType("document", dc); - var child = documentType("child", dc); - var mother = documentType("mother", dc); - var father = documentType("father", dc); - var grandparent = documentType("grandparent", dc); - - assertEquals(child.inherits(0).idx(), root.idx()); - assertEquals(child.inherits(1).idx(), mother.idx()); - assertEquals(child.inherits(2).idx(), father.idx()); - assertEquals(mother.inherits(0).idx(), root.idx()); - assertEquals(mother.inherits(1).idx(), grandparent.idx()); - } - } - - private DocumentmanagerConfig.Doctype.Structtype structType(String name, DocumentmanagerConfig dc) { - for (var dt : dc.doctype()) { - for (var st : dt.structtype()) { - if (name.equals(st.name())) return st; - } - } - return null; - } - - private DocumentmanagerConfig.Doctype documentType(String name, DocumentmanagerConfig dc) { - for (var dot : dc.doctype()) { - if (name.equals(dot.name())) return dot; - } - return null; - } - - @Test - public void requireThatStructTypesAreInheritedFromParent() throws IOException, ParseException { - String dir = "src/test/derived/inheritfromparent/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "parent.sd"); - builder.addSchemaFile(dir + "child.sd"); - builder.build(true); - derive("inheritfromparent", builder, builder.getSchema("child")); - assertCorrectConfigFiles("inheritfromparent"); - } - - @Test - public void requireThatStructTypesAreInheritedFromGrandParent() throws IOException, ParseException { - String dir = "src/test/derived/inheritfromgrandparent/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "grandparent.sd"); - builder.addSchemaFile(dir + "parent.sd"); - builder.addSchemaFile(dir + "child.sd"); - builder.build(true); - derive("inheritfromgrandparent", builder, builder.getSchema("child")); - assertCorrectConfigFiles("inheritfromgrandparent"); - } - - @Test - public void testInheritance() throws IOException, ParseException { - String dir = "src/test/derived/inheritance/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "grandparent.sd"); - builder.addSchemaFile(dir + "father.sd"); - builder.addSchemaFile(dir + "mother.sd"); - builder.addSchemaFile(dir + "child.sd"); - builder.build(true); - derive("inheritance", builder, builder.getSchema("child")); - assertCorrectConfigFiles("inheritance"); - } - - @Test - public void testIndexSettingInheritance() { - SDDocumentType parent = new SDDocumentType("parent"); - Schema parentSchema = new Schema("parent", MockApplicationPackage.createEmpty()); - parentSchema.addDocument(parent); - SDField prefixed = parent.addField("prefixed", DataType.STRING); - prefixed.parseIndexingScript("{ index }"); - prefixed.addIndex(new Index("prefixed", true)); - - SDDocumentType child = new SDDocumentType("child"); - child.inherit(parent); - Schema childSchema = new Schema("child", MockApplicationPackage.createEmpty()); - childSchema.addDocument(child); - - prefixed = (SDField)child.getField("prefixed"); - assertNotNull(prefixed); - assertEquals(new Index("prefixed", true), childSchema.getIndex("prefixed")); - } - - @Test - public void testInheritStructDiamondNew() throws IOException, ParseException { - String dir = "src/test/derived/declstruct/"; - List files = Arrays.asList("common.sd", "foo.sd", "bar.sd", "foobar.sd"); - var builder = new ApplicationBuilder(new TestProperties()); - for (String fileName : files) { - builder.addSchemaFile(dir + fileName); - } - builder.build(true); - derive("declstruct", builder, builder.getSchema("foobar")); - assertCorrectConfigFiles("declstruct"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java deleted file mode 100644 index 0980f766579..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author bratseth - */ -public class IntegerAttributeToStringIndexTestCase extends AbstractExportingTestCase { - @Test - public void testIt() throws IOException, ParseException { - assertCorrectDeriving("integerattributetostringindex"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java deleted file mode 100644 index 70545c15ca0..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Set; - -import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; -import static org.junit.Assert.assertTrue; - -/** - * @author bratseth - */ -public class LiteralBoostTestCase extends AbstractExportingTestCase { - - /** - * Tests adding of literal boost constructs - */ - @Test - public void testLiteralBoost() { - Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - SDDocumentType document = new SDDocumentType("literalboost"); - schema.addDocument(document); - SDField field1 = document.addField("a", DataType.STRING); - field1.parseIndexingScript("{ index }"); - field1.setLiteralBoost(20); - RankProfile other = new RankProfile("other", schema, rankProfileRegistry); - rankProfileRegistry.add(other); - other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - - new Processing().process(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), - true, false, Set.of()); - DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); - - // Check attribute fields - derived.getAttributeFields(); // TODO: assert content - - // Check il script addition - assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", - "clear_state | guard { input a | tokenize | index a_literal; }"), - schema); - - // Check index info addition - IndexInfo indexInfo = derived.getIndexInfo(); - assertTrue(indexInfo.hasCommand("a", "literal-boost")); - } - - /** - * Tests adding a literal boost in a non-default rank profile only - */ - @Test - public void testNonDefaultRankLiteralBoost() { - Schema schema = new Schema("literalboost", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - SDDocumentType document = new SDDocumentType("literalboost"); - schema.addDocument(document); - SDField field1 = document.addField("a", DataType.STRING); - field1.parseIndexingScript("{ index }"); - RankProfile other = new RankProfile("other", schema, rankProfileRegistry); - rankProfileRegistry.add(other); - other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - - schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); - DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); - - // Check il script addition - assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", - "clear_state | guard { input a | tokenize | index a_literal; }"), - schema); - - // Check index info addition - IndexInfo indexInfo = derived.getIndexInfo(); - assertTrue(indexInfo.hasCommand("a","literal-boost")); - } - - /** Tests literal boosts in two fields going to the same index */ - @Test - public void testTwoLiteralBoostFields() { - Schema schema = new Schema("msb", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - SDDocumentType document = new SDDocumentType("msb"); - schema.addDocument(document); - SDField field1 = document.addField("title", DataType.STRING); - field1.parseIndexingScript("{ summary | index }"); - field1.setLiteralBoost(20); - SDField field2 = document.addField("body", DataType.STRING); - field2.parseIndexingScript("{ summary | index }"); - field2.setLiteralBoost(20); - - schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); - new DerivedConfiguration(schema, rankProfileRegistry); - assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", - "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", - "clear_state | guard { input title | tokenize | index title_literal; }", - "clear_state | guard { input body | tokenize | index body_literal; }"), - schema); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java deleted file mode 100644 index 5427c6a19e7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author bratseth - */ -public class LowercaseTestCase extends AbstractExportingTestCase { - - @Test - public void testDeriving() throws IOException, ParseException { - assertCorrectDeriving("lowercase"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java deleted file mode 100644 index 53bf55fc73f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; -import java.io.IOException; - -/** - * Tests streaming configuration deriving - * - * @author bratseth - */ -public class MailTestCase extends AbstractExportingTestCase { - - @Test - public void testMail() throws IOException, ParseException { - String dir = "src/test/derived/mail/"; - ApplicationBuilder sb = new ApplicationBuilder(); - sb.addSchemaFile(dir + "mail.sd"); - assertCorrectDeriving(sb, dir, new TestableDeployLogger()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java deleted file mode 100755 index 259a7ab8804..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author arnej - */ -public class MatchSettingsResolvingTestCase extends AbstractExportingTestCase { - - @Test - public void testSimpleDefaults() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_simple_def", new TestProperties()); - } - - @Test - public void testSimpleWithStructSettings() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_simple_wss", - new TestProperties()); - } - - @Test - public void testSimpleWithFieldSettings() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_simple_wfs", new TestProperties()); - } - - @Test - public void testSimpleStructAndFieldSettings() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_simple_wss_wfs", new TestProperties()); - } - - @Test - public void testMapDefaults() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_map_def", new TestProperties()); - } - - @Test - public void testMapWithStructSettings() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_map_wss", new TestProperties()); - } - - @Test - public void testMapWithFieldSettings() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_map_wfs", new TestProperties()); - } - - @Test - public void testMapAfter() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_map_after", new TestProperties()); - } - - - @Test - public void testMapInStruct() throws IOException, ParseException { - assertCorrectDeriving("matchsettings_map_in_struct", new TestProperties()); - } - - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java deleted file mode 100644 index 4afbb4c5436..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.ApplicationBuilder; -import org.junit.Test; - -/** - * Tests deriving a configuration with structs in multiple .sd files - * - * @author arnej - */ -public class MultiStructTestCase extends AbstractExportingTestCase { - - @Test - public void testDocTypeConfigs() throws Exception { - var logger = new TestableDeployLogger(); - var props = new TestProperties(); - ApplicationBuilder builder = ApplicationBuilder.createFromDirectory - ("src/test/derived/multi_struct/", new MockFileRegistry(), logger, props); - derive("multi_struct", builder, builder.getSchema("shop")); - assertCorrectConfigFiles("multi_struct"); - } - -} - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java deleted file mode 100644 index 245049fd4bc..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests deriving a configuration with multiple summaries - * - * @author bratseth - */ -public class MultipleSummariesTestCase extends AbstractExportingTestCase { - - @Test - public void testMultipleSummariesNew() throws IOException, ParseException { - assertCorrectDeriving("multiplesummaries", new TestProperties()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java deleted file mode 100644 index c8df63e1028..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.searchdefinition.ApplicationBuilder; - -import org.junit.Test; -import static org.junit.Assert.assertThrows; - -/** - * Verifies that a struct in a document type is preferred over another document type - * of the same name. - * - * @author bratseth - */ -public class NameCollisionTestCase extends AbstractExportingTestCase { - - @Test - public void testNameCollision() throws Exception { - assertCorrectDeriving("namecollision", "collisionstruct", - new TestProperties(), - new TestableDeployLogger()); - DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java deleted file mode 100644 index 8c6bc40ece0..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.RankType; -import org.junit.Test; - -import java.util.Iterator; - -import static org.junit.Assert.*; - -/** - * Testing stuff related to native rank type definitions - * - * @author geirst - */ -public class NativeRankTypeDefinitionsTestCase extends AbstractSchemaTestCase { - @Test - public void testTables() { - assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable"); - assertEquals(NativeTable.Type.OCCURRENCE_COUNT.getName(), "occurrenceCountTable"); - assertEquals(NativeTable.Type.PROXIMITY.getName(), "proximityTable"); - assertEquals(NativeTable.Type.REVERSE_PROXIMITY.getName(), "reverseProximityTable"); - assertEquals(NativeTable.Type.WEIGHT.getName(), "weightTable"); - } - @Test - public void testDefinitions() { - NativeRankTypeDefinitionSet defs = new NativeRankTypeDefinitionSet("default"); - - NativeRankTypeDefinition rank; - Iterator tables; - - assertEquals(4, defs.types().size()); - - { - rank = defs.getRankTypeDefinition(RankType.EMPTY); - assertNotNull(rank); - assertEquals(RankType.EMPTY, rank.getType()); - tables = rank.rankSettingIterator(); - assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)"), tables.next()); - assertFalse(tables.hasNext()); - } - - { - rank = defs.getRankTypeDefinition(RankType.ABOUT); - assertNotNull(rank); - assertEquals(RankType.ABOUT, rank.getType()); - tables = rank.rankSettingIterator(); - assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"), tables.next()); - assertFalse(tables.hasNext()); - } - - { - rank = defs.getRankTypeDefinition(RankType.IDENTITY); - assertNotNull(rank); - assertEquals(RankType.IDENTITY, rank.getType()); - tables = rank.rankSettingIterator(); - assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"), tables.next()); - assertFalse(tables.hasNext()); - } - - { - rank = defs.getRankTypeDefinition(RankType.TAGS); - assertNotNull(rank); - assertEquals(RankType.TAGS, rank.getType()); - tables = rank.rankSettingIterator(); - assertEquals(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"), tables.next()); - assertEquals(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)"), tables.next()); - assertFalse(tables.hasNext()); - } - - { - assertEquals(RankType.ABOUT, defs.getRankTypeDefinition(RankType.DEFAULT).getType()); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java deleted file mode 100644 index 16ec896ca66..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class NearestNeighborTestCase extends AbstractExportingTestCase { - - @Test - public void testNearestNeighbor() throws IOException, ParseException { - try { - ComponentId.resetGlobalCountersForTests(); - DerivedConfiguration c = assertCorrectDeriving("nearestneighbor"); - - CompiledQueryProfileRegistry queryProfiles = - QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); - Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5 - queryProfiles.getComponent("default")); - fail("This should fail when q_vec is parsed as a tensor"); - } catch (IllegalArgumentException e) { - // success - assertEquals("Could not set 'ranking.features.query(q_vec)' to '[1,2,3,4,5,6]'", e.getMessage()); - } catch (RuntimeException e) { - e.printStackTrace(); - throw e; - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java deleted file mode 100644 index c7b0d131eb6..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.search.Query; -import com.yahoo.search.query.profile.QueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; - -import com.yahoo.component.ComponentId; - -import static org.junit.Assert.assertEquals; - -public class NeuralNetTestCase extends AbstractExportingTestCase { - - @Test - public void testNeuralNet() throws IOException, ParseException { - ComponentId.resetGlobalCountersForTests(); - DerivedConfiguration c = assertCorrectDeriving("neuralnet"); - // Verify that query profiles end up correct when passed through the same intermediate forms as a full system - CompiledQueryProfileRegistry queryProfiles = - QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); - assertNeuralNetQuery(c, queryProfiles.getComponent("default")); - } - - @Test - public void testNeuralNet_noQueryProfiles() throws IOException, ParseException { - ComponentId.resetGlobalCountersForTests(); - DerivedConfiguration c = assertCorrectDeriving("neuralnet_noqueryprofile"); - } - - private void assertNeuralNetQuery(DerivedConfiguration c, CompiledQueryProfile defaultprofile) { - Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]", defaultprofile); - assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", - q.properties().get("ranking.features.query(b_1)").toString()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NuwaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NuwaTestCase.java deleted file mode 100644 index 75b8d3bf584..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NuwaTestCase.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; - -/** - * A real world mlr ranking model, useful to benchmark memory and cpu usage. - * - * @author bratseth - */ -public class NuwaTestCase extends AbstractExportingTestCase { - - @Test - @Ignore - public void testNuwa() throws IOException, ParseException { - System.gc(); - long freeBytesBefore = Runtime.getRuntime().freeMemory(); - long totalBytesBefore = Runtime.getRuntime().totalMemory(); - - DerivedConfiguration configuration = assertCorrectDeriving("nuwa"); - - System.gc(); - long freeBytesAfter = Runtime.getRuntime().freeMemory(); - long totalBytesAfter = Runtime.getRuntime().totalMemory(); - long additionalAllocated = totalBytesAfter - totalBytesBefore; - System.out.println("Consumed " + ((freeBytesBefore - (freeBytesAfter - additionalAllocated) ) / 1000000) + " Mb"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java deleted file mode 100755 index 1bb8083e7b2..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author Einar M R Rosenvinge - */ -public class OrderIlscriptsTestCase extends AbstractExportingTestCase { - - @Test - public void testOrderIlscripts() throws IOException, ParseException { - assertCorrectDeriving("orderilscripts"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java deleted file mode 100644 index 0b571d11917..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests deriving of various field types - * - * @author bratseth - */ -public class PrefixExactAttributeTestCase extends AbstractExportingTestCase { - - @Test - public void testTypes() throws IOException, ParseException { - assertCorrectDeriving("prefixexactattribute"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java deleted file mode 100644 index 3a900e0840a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests a search definition with various rank profiles having different settings - * - * @author bratseth - */ -public class RankProfilesTestCase extends AbstractExportingTestCase { - @Test - public void testRankProfiles() throws IOException, ParseException { - assertCorrectDeriving("rankprofiles", null, new TestProperties(), new TestableDeployLogger()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java deleted file mode 100644 index 0f29d2dda40..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author bratseth - */ -public class RankPropertiesTestCase extends AbstractExportingTestCase { - - @Test - public void testRankProperties() throws IOException, ParseException { - assertCorrectDeriving("rankproperties"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java deleted file mode 100644 index 086a69a8b20..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author geirst - */ -public class ReferenceFieldsTestCase extends AbstractExportingTestCase { - - @Test - public void configs_related_to_reference_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("reference_fields", "ad", new TestableDeployLogger()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java deleted file mode 100644 index dbbe1497ffe..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.ApplicationBuilder; -import org.junit.Test; - -/** - * Tests deriving a configuration with references from multiple .sd files - * - * @author arnej - */ -public class ReferenceFromSeveralTestCase extends AbstractExportingTestCase { - - @Test - public void testDocManConfigs() throws Exception { - var logger = new TestableDeployLogger(); - var props = new TestProperties(); - ApplicationBuilder builder = ApplicationBuilder.createFromDirectory - ("src/test/derived/reference_from_several/", new MockFileRegistry(), logger, props); - derive("reference_from_several", builder, builder.getSchema("foo")); - assertCorrectConfigFiles("reference_from_several"); - } - -} - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java deleted file mode 100644 index a458036a03f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; - -/** - * @author bratseth - */ -public class SchemaInheritanceTestCase extends AbstractExportingTestCase { - - @Test - public void testIt() throws IOException, ParseException { - try { - ApplicationBuilder builder = ApplicationBuilder.createFromDirectory("src/test/derived/schemainheritance/", - new MockFileRegistry(), - new TestableDeployLogger(), - new TestProperties()); - derive("schemainheritance", builder, builder.getSchema("child")); - assertCorrectConfigFiles("schemainheritance"); - } - finally { - IOUtils.recursiveDeleteDir(new File("src/test/derived/schemainheritance/models.generated/")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java deleted file mode 100644 index 11e21c7915d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; -import org.junit.Test; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static java.util.Collections.emptyMap; -import static java.util.stream.Collectors.toList; -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - * @author bjorncs - */ -public class SchemaOrdererTestCase extends AbstractSchemaTestCase { - - private static Map createSchemas() { - Map schemas = new HashMap<>(); - - Schema grandParent = createSchema("grandParent", schemas); - - Schema mother = createSchema("mother", schemas); - inherit(mother, grandParent); - - Schema father = createSchema("father", schemas); - inherit(father, grandParent); - createDocumentReference(father, mother, "wife_ref"); - - Schema daugther = createSchema("daughter", schemas); - inherit(daugther, father); - inherit(daugther, mother); - - Schema son = createSchema("son", schemas); - inherit(son, father); - inherit(son, mother); - - Schema product = createSchema("product", schemas); - - Schema pc = createSchema("pc", schemas); - inherit(pc, product); - Schema pcAccessory = createSchema("accessory-pc", schemas); - inherit(pcAccessory, product); - createDocumentReference(pcAccessory, pc, "pc_ref"); - - createSchema("alone", schemas); - - return schemas; - } - - private static Schema createSchema(String name, Map schemas) { - Schema schema = new Schema(name, MockApplicationPackage.createEmpty()); - SDDocumentType document = new SDDocumentType(name); - document.setDocumentReferences(new DocumentReferences(emptyMap())); - schema.addDocument(document); - schemas.put(schema.getName(), schema); - return schema; - } - - private static void inherit(Schema inheritee, Schema inherited) { - inheritee.getDocument().inherit(inherited.getDocument()); - } - - private static void assertOrder(List expectedSearchOrder, List inputNames) { - Map schemas = createSchemas(); - List inputSchemas = inputNames.stream() - .map(schemas::get) - .map(Objects::requireNonNull) - .collect(toList()); - List actualSearchOrder = new SearchOrderer() - .order(inputSchemas) - .stream() - .map(Schema::getName) - .collect(toList()); - assertEquals(expectedSearchOrder, actualSearchOrder); - } - - @SuppressWarnings("deprecation") - private static void createDocumentReference(Schema from, Schema to, String refFieldName) { - SDDocumentType fromDocument = from.getDocument(); - SDField refField = new TemporarySDField(fromDocument, refFieldName, NewDocumentReferenceDataType.forDocumentName(to.getName())); - fromDocument.addField(refField); - Map originalMap = fromDocument.getDocumentReferences().get().referenceMap(); - HashMap modifiedMap = new HashMap<>(originalMap); - modifiedMap.put(refFieldName, new DocumentReference(refField, to)); - fromDocument.setDocumentReferences(new DocumentReferences(modifiedMap)); - } - - - @Test - public void testPerfectOrderingIsKept() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); - } - - @Test - public void testOneLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); - } - - @Test - public void testMultiLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); - } - - @Test - public void testAloneIsKeptInPlaceWithMultiLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); - } - - @Test - public void testPartialMultiLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); - } - - @Test - public void testMultilevelReorderingAccrossHierarchies() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); - } - - @Test - public void referees_are_ordered_before_referrer() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"), - Arrays.asList("accessory-pc", "daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); - } - - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java deleted file mode 100644 index d89e5f2c957..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; - -/** - * Tests really simple inheriting - */ -public class SimpleInheritTestCase extends AbstractExportingTestCase { - - @Test - public void testEmptyChild() throws IOException, ParseException { - String name = "emptychild"; - final String expectedResultsDirName = "src/test/derived/" + name + "/"; - - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(expectedResultsDirName + "parent.sd"); - builder.addSchemaFile(expectedResultsDirName + "child.sd"); - builder.build(true); - - Schema schema = builder.getSchema("child"); - - String toDirName = "temp/" + name; - File toDir = new File(toDirName); - toDir.mkdirs(); - deleteContent(toDir); - - DerivedConfiguration config = new DerivedConfiguration(schema, builder.getRankProfileRegistry()); - config.export(toDirName); - - checkDir(toDirName, expectedResultsDirName); - } - - private void checkDir(String toDirName, String expectedResultsDirName) throws IOException { - File[] files = new File(expectedResultsDirName).listFiles(); - for (File file : files) { - if ( ! file.getName().endsWith(".cfg")) continue; - assertEqualFiles(file.getPath(), toDirName + "/" + file.getName(), false); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java deleted file mode 100644 index e6c7efd7052..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class SliceTestCase extends AbstractExportingTestCase { - - @Test - public void testSlice() throws IOException, ParseException { - ComponentId.resetGlobalCountersForTests(); - DerivedConfiguration c = assertCorrectDeriving("slice"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java deleted file mode 100644 index 2516b1b0d9a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests sort settings - * - * @author baldersheim - */ -public class SortingTestCase extends AbstractExportingTestCase { - - @Test - public void testDocumentDerivingNewParser() throws IOException, ParseException { - assertCorrectDeriving("sorting", new TestProperties()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java deleted file mode 100755 index f9bd3033e14..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests streaming search configuration deriving for structs - * - * @author bratseth - */ -public class StreamingStructTestCase extends AbstractExportingTestCase { - - @Test - public void testStreamingStruct() throws IOException, ParseException { - assertCorrectDeriving("streamingstruct"); - } - - @Test - public void testStreamingStructExplicitDefaultSummaryClass() throws IOException, ParseException { - assertCorrectDeriving("streamingstructdefault"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java deleted file mode 100755 index 3338f6cf87d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author Einar M R Rosenvinge - */ -public class StructAnyOrderTestCase extends AbstractExportingTestCase { - @Test - public void testStructAnyOrder() throws IOException, ParseException { - assertCorrectDeriving("structanyorder"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java deleted file mode 100644 index 1f6c70c9383..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - - -import com.yahoo.searchdefinition.ApplicationBuilder; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.IOException; - - -import org.junit.rules.TemporaryFolder; - -/** - * Tests struct inheritance - * - * @author arnej - */ -public class StructInheritanceTestCase extends AbstractExportingTestCase { - - @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void requireThatStructCanInherit() throws IOException, ParseException { - String dir = "src/test/derived/structinheritance/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "simple.sd"); - builder.build(false); - derive("structinheritance", builder, builder.getSchema("simple")); - assertCorrectConfigFiles("structinheritance"); - } - - @Test - public void requireThatRedeclareIsNotAllowed() throws IOException, ParseException { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("cannot inherit from base and redeclare field name"); - String dir = "src/test/derived/structinheritance/"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(dir + "bad.sd"); - builder.build(true); - derive("structinheritance", builder, builder.getSchema("bad")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java deleted file mode 100644 index 6abe0e763cb..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.searchdefinition.*; -import com.yahoo.vespa.config.search.SummarymapConfig; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Set; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -/** - * Tests summary map extraction - * - * @author bratseth - */ -public class SummaryMapTestCase extends AbstractSchemaTestCase { - @Test - public void testDeriving() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); - SummaryMap summaryMap = new SummaryMap(schema); - - Iterator transforms = summaryMap.resultTransforms().values().iterator(); - FieldResultTransform transform = transforms.next(); - assertEquals("dyndesc", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - - transform = transforms.next(); - assertEquals("dynlong", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - - transform = transforms.next(); - assertEquals("dyndesc2", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - - transform = transforms.next(); - assertEquals("measurement", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - - transform = transforms.next(); - assertEquals("rankfeatures", transform.getFieldName()); - assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); - - transform = transforms.next(); - assertEquals("summaryfeatures", transform.getFieldName()); - assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); - - transform = transforms.next(); - assertEquals("popsiness", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - - transform = transforms.next(); - assertEquals("popularity", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - - transform = transforms.next(); - assertEquals("access", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - - assertFalse(transforms.hasNext()); - } - @Test - public void testPositionDeriving() { - Schema schema = new Schema("store", MockApplicationPackage.createEmpty()); - SDDocumentType document = new SDDocumentType("store"); - schema.addDocument(document); - String fieldName = "location"; - SDField field = document.addField(fieldName, PositionDataType.INSTANCE); - field.parseIndexingScript("{ attribute | summary }"); - new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), - true, false, Set.of()); - SummaryMap summaryMap = new SummaryMap(schema); - - Iterator transforms = summaryMap.resultTransforms().values().iterator(); - - FieldResultTransform transform = transforms.next(); - - assertEquals(fieldName, transform.getFieldName()); - assertEquals(SummaryTransform.GEOPOS, transform.getTransform()); - - transform = transforms.next(); - assertEquals("rankfeatures", transform.getFieldName()); - assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); - - transform = transforms.next(); - assertEquals("summaryfeatures", transform.getFieldName()); - assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); - - transform = transforms.next(); - assertEquals("location_zcurve", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); - - assertFalse(transforms.hasNext()); - - SummarymapConfig.Builder scb = new SummarymapConfig.Builder(); - summaryMap.getConfig(scb); - SummarymapConfig c = scb.build(); - - assertEquals(-1, c.defaultoutputclass()); - assertEquals(c.override().size(), 4); - - assertEquals(c.override(0).field(), fieldName); - assertEquals(c.override(0).command(), "geopos"); - assertEquals(c.override(0).arguments(), PositionDataType.getZCurveFieldName(fieldName)); - - assertEquals(c.override(1).field(), "rankfeatures"); - assertEquals(c.override(1).command(), "rankfeatures"); - assertEquals(c.override(1).arguments(), ""); - - assertEquals(c.override(2).field(), "summaryfeatures"); - assertEquals(c.override(2).command(), "summaryfeatures"); - assertEquals(c.override(2).arguments(), ""); - - assertEquals(c.override(3).field(), "location_zcurve"); - assertEquals(c.override(3).command(), "attribute"); - assertEquals(c.override(3).arguments(), "location_zcurve"); - } - - @Test - public void testFailOnSummaryFieldSourceCollision() { - try { - ApplicationBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); - } catch (Exception e) { - assertTrue(e.getMessage().matches(".*equally named field.*")); - } - } - - @Test - public void source_field_is_passed_as_argument_in_matched_elements_filter_transforms() throws ParseException { - assertOverride(joinLines("field my_field type map {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field key { indexing: attribute }", - "}"), "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()); - - assertOverride(joinLines("field my_field type map {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - "}"), "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName()); - } - - @Test - public void commands_that_are_dynamic_and_require_the_query() { - assertTrue(SummaryMap.isDynamicCommand("dynamicteaser")); - assertTrue(SummaryMap.isDynamicCommand(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName())); - assertTrue(SummaryMap.isDynamicCommand(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER.getName())); - assertFalse(SummaryMap.isDynamicCommand(SummaryTransform.ATTRIBUTE.getName())); - } - - private void assertOverride(String fieldContent, String expFieldName, String expCommand) throws ParseException { - var summaryMap = new SummaryMap(buildSearch(fieldContent)); - var cfgBuilder = new SummarymapConfig.Builder(); - summaryMap.getConfig(cfgBuilder); - var cfg = new SummarymapConfig(cfgBuilder); - var override = cfg.override(0); - assertEquals(expFieldName, override.field()); - assertEquals(expCommand, override.command()); - assertEquals(expFieldName, override.arguments()); - } - - private Schema buildSearch(String field) throws ParseException { - var builder = new ApplicationBuilder(new RankProfileRegistry()); - builder.addSchema(joinLines("search test {", - " document test {", - field, - " }", - "}")); - builder.build(true); - return builder.getSchema(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java deleted file mode 100644 index fd5caeb6b6d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.config.search.SummaryConfig; -import org.junit.Test; - -import java.io.IOException; -import java.util.Iterator; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * Tests summary extraction - * - * @author bratseth - */ -public class SummaryTestCase extends AbstractSchemaTestCase { - - @Test - public void deriveRawAsBase64() throws ParseException { - String sd = joinLines( - "schema s {", - " raw-as-base64-in-summary", - " document s {", - " field raw_field type raw {", - " indexing: summary", - " }", - " }", - "}"); - Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); - SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); - assertEquals(SummaryClassField.Type.RAW, summary.fields().get("raw_field").getType()); - } - - @Test - public void deriveRawAsLegacy() throws ParseException { - String sd = joinLines( - "schema s {", - " document s {", - " field raw_field type raw {", - " indexing: summary", - " }", - " }", - "}"); - Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); - SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); - assertEquals(SummaryClassField.Type.DATA, summary.fields().get("raw_field").getType()); - } - - @Test - public void testDeriving() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); - SummaryClass summary = new SummaryClass(schema, schema.getSummary("default"), new BaseDeployLogger()); - assertEquals("default", summary.getName()); - - Iterator fields = summary.fields().values().iterator(); - - SummaryClassField field; - - assertEquals(13, summary.fields().size()); - - field = fields.next(); - assertEquals("exactemento", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("exact", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("title", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("description", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("dyndesc", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("longdesc", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("longstat", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("dynlong", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("dyndesc2", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - - field = fields.next(); - assertEquals("measurement", field.getName()); - assertEquals(SummaryClassField.Type.INTEGER, field.getType()); - - field = fields.next(); - assertEquals("rankfeatures", field.getName()); - assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); - - field = fields.next(); - assertEquals("summaryfeatures", field.getName()); - assertEquals(SummaryClassField.Type.FEATUREDATA, field.getType()); - - field = fields.next(); - assertEquals("documentid", field.getName()); - assertEquals(SummaryClassField.Type.LONGSTRING, field.getType()); - } - - @Test - public void reference_fields_can_be_part_of_summary_classes() throws ParseException { - Schema adSchema = buildCampaignAdModel(); - - SummaryClass defaultClass = new SummaryClass(adSchema, adSchema.getSummary("default"), new BaseDeployLogger()); - assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.fields().get("campaign_ref").getType()); - assertEquals(SummaryClassField.Type.LONGSTRING, defaultClass.fields().get("other_campaign_ref").getType()); - - SummaryClass myClass = new SummaryClass(adSchema, adSchema.getSummary("my_summary"), new BaseDeployLogger()); - assertNull(myClass.fields().get("campaign_ref")); - assertEquals(SummaryClassField.Type.LONGSTRING, myClass.fields().get("other_campaign_ref").getType()); - } - - private static Schema buildCampaignAdModel() throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema("search campaign { document campaign {} }"); - builder.addSchema(joinLines("search ad {", - " document ad {", - " field campaign_ref type reference {", - " indexing: summary | attribute", - " }", - " field other_campaign_ref type reference {", - " indexing: summary | attribute", - " }", - " }", - " document-summary my_summary {", - " summary other_campaign_ref type reference {}", - " }", - "}")); - builder.build(true); - return builder.getSchema("ad"); - } - - @Test - public void omit_summary_features_specified_for_document_summary() throws ParseException { - String sd = joinLines( - "schema test {", - " document test {", - " field foo type string { indexing: summary }", - " }", - " document-summary bar {", - " summary foo type string {}", - " omit-summary-features", - " }", - " document-summary baz {", - " summary foo type string {}", - " }", - "}"); - var search = ApplicationBuilder.createFromString(sd).getSchema(); - assertOmitSummaryFeatures(true, search, "bar"); - assertOmitSummaryFeatures(false, search, "baz"); - } - - private void assertOmitSummaryFeatures(boolean expected, Schema schema, String summaryName) { - var summary = new SummaryClass(schema, schema.getSummary(summaryName), new BaseDeployLogger()); - var config = new SummaryConfig.Classes(summary.getSummaryClassConfig()); - assertEquals(expected, config.omitsummaryfeatures()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java deleted file mode 100644 index 52d57bd0468..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.application.api.DeployLogger; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author bratseth - */ -public class TestableDeployLogger implements DeployLogger { - - private static final Logger log = Logger.getLogger("DeployLogger"); - - public List warnings = new ArrayList<>(); - public List info = new ArrayList<>(); - - @Override - public final void log(Level level, String message) { - log.log(level, message); - if (level.equals(Level.WARNING)) - warnings.add(message); - if (level.equals(Level.INFO)) - info.add(message); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java deleted file mode 100755 index 4b9d5386314..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author bratseh - */ -public class TokenizationTestCase extends AbstractExportingTestCase { - - @Test - public void testTokenizationScripts() throws IOException, ParseException { - assertCorrectDeriving("tokenization"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java deleted file mode 100644 index 4e629fcb4d8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Test structs for streaming with another unrelated .sd present - * - * @author arnej27959 - */ -public class TwoStreamingStructsTestCase extends AbstractExportingTestCase { - - @Test - public void testTwoStreamingStructsExporting() throws ParseException, IOException { - - String root = "src/test/derived/twostreamingstructs"; - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchemaFile(root + "/streamingstruct.sd"); - builder.addSchemaFile(root + "/whatever.sd"); - builder.build(true); - assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); - - builder = new ApplicationBuilder(); - builder.addSchemaFile(root + "/streamingstruct.sd"); - builder.addSchemaFile(root + "/whatever.sd"); - builder.build(true); - assertCorrectDeriving(builder, builder.getSchema("streamingstruct"), root); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java deleted file mode 100644 index 62a79e49146..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.util.Set; - -import static org.junit.Assert.assertFalse; -/** - * Tests automatic type conversion using multifield indices - * - * @author bratseth - */ -public class TypeConversionTestCase extends AbstractSchemaTestCase { - - /** Tests that exact-string stuff is not spilled over to the default index */ - @Test - public void testExactStringToStringTypeConversion() { - Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); - RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - SDDocumentType document = new SDDocumentType("test"); - schema.addDocument(document); - SDField a = new SDField(document, "a", DataType.STRING); - a.parseIndexingScript("{ index }"); - document.addField(a); - - new Processing().process(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), - true, false, Set.of()); - DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); - IndexInfo indexInfo = derived.getIndexInfo(); - assertFalse(indexInfo.hasCommand("default", "compact-to-term")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java deleted file mode 100644 index cee6d4f95c8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests deriving of various field types - * - * @author bratseth - */ -public class TypesTestCase extends AbstractExportingTestCase { - - @Test - public void testTypes() throws IOException, ParseException { - assertCorrectDeriving("types"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java deleted file mode 100644 index 9855ca30ebc..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; -import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author geirst - */ -public class VsmFieldsTestCase { - - @SuppressWarnings("deprecation") - @Test - public void reference_type_field_is_unsearchable() { - Schema schema = new Schema("test", MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); - var sdoc = new SDDocumentType("test"); - schema.addDocument(sdoc); - SDField refField = new TemporarySDField(sdoc, "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type")); - refField.parseIndexingScript("{ summary }"); - schema.getDocument().addField(refField); - - VsmFields vsmFields = new VsmFields(schema); - VsmfieldsConfig.Builder cfgBuilder = new VsmfieldsConfig.Builder(); - vsmFields.getConfig(cfgBuilder); - VsmfieldsConfig cfg = cfgBuilder.build(); - - assertEquals(1, cfg.fieldspec().size()); - VsmfieldsConfig.Fieldspec fieldSpec = cfg.fieldspec().get(0); - assertEquals("ref_field", fieldSpec.name()); - assertEquals(VsmfieldsConfig.Fieldspec.Searchmethod.NONE, fieldSpec.searchmethod()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java deleted file mode 100644 index 99692e70041..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class ComplexAttributeFieldUtilsTestCase { - - private static class FixtureBase { - - private final ImmutableSDField field; - - FixtureBase(String fieldName, String sdContent) throws ParseException { - Schema schema = ApplicationBuilder.createFromString(sdContent).getSchema(); - field = schema.getConcreteField(fieldName); - } - - public ImmutableSDField field() { - return field; - } - - boolean isSupportedComplexField() { - return ComplexAttributeFieldUtils.isSupportedComplexField(field()); - } - - boolean isArrayOfSimpleStruct() { - return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field()); - } - - boolean isMapOfSimpleStruct() { - return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field()); - } - - boolean isMapOfPrimitiveType() { - return ComplexAttributeFieldUtils.isMapOfPrimitiveType(field()); - } - - boolean isComplexFieldWithOnlyStructFieldAttributes() { - return ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes(field()); - } - } - - private static class Fixture extends FixtureBase { - - Fixture(String fieldName, String sdFieldContent) throws ParseException { - super(fieldName, joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weight type int {}", - " }", - sdFieldContent, - " }", - "}")); - } - } - - private static class ComplexFixture extends FixtureBase { - - ComplexFixture(String fieldName, String sdFieldContent) throws ParseException { - super(fieldName, joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weights type array {}", - " }", - sdFieldContent, - " }", - "}")); - } - } - - @Test - public void array_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - Fixture f = new Fixture("elem_array", - joinLines("field elem_array type array {", - " indexing: summary", - " struct-field name { indexing: attribute }", - " struct-field weight { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isArrayOfSimpleStruct()); - assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - - @Test - public void array_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { - Fixture f = new Fixture("elem_array", - joinLines("field elem_array type array {", - " indexing: summary", - " struct-field weight { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isArrayOfSimpleStruct()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - - @Test - public void map_of_struct_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - Fixture f = new Fixture("elem_map", - joinLines("field elem_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value.name { indexing: attribute }", - " struct-field value.weight { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - - @Test - public void map_of_struct_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { - { - Fixture f = new Fixture("elem_map", - joinLines("field elem_map type map {", - " indexing: summary", - " struct-field value.name { indexing: attribute }", - " struct-field value.weight { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - { - Fixture f = new Fixture("elem_map", - joinLines("field elem_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value.weight { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - } - - @Test - public void map_of_primitive_type_with_only_struct_field_attributes_is_tagged_as_such() throws ParseException { - Fixture f = new Fixture("str_map", - joinLines("field str_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isMapOfPrimitiveType()); - assertFalse(f.isMapOfSimpleStruct()); - assertTrue(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - - @Test - public void map_of_primitive_type_with_some_struct_field_attributes_is_tagged_as_such() throws ParseException { - { - Fixture f = new Fixture("int_map", - joinLines("field int_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isMapOfPrimitiveType()); - assertFalse(f.isMapOfSimpleStruct()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - { - Fixture f = new Fixture("int_map", - joinLines("field int_map type map {", - " indexing: summary", - " struct-field value { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isMapOfPrimitiveType()); - assertFalse(f.isMapOfSimpleStruct()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - } - - @Test - public void unsupported_complex_field_is_tagged_as_such() throws ParseException { - { - ComplexFixture f = new ComplexFixture("elem_array", - joinLines("field elem_array type array {", - " struct-field name { indexing: attribute }", - " struct-field weights { indexing: attribute }", - "}")); - assertFalse(f.isSupportedComplexField()); - assertFalse(f.isArrayOfSimpleStruct()); - assertFalse(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - { - ComplexFixture f = new ComplexFixture("elem_map", - joinLines("field elem_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value.weights { indexing: attribute }", - "}")); - assertFalse(f.isSupportedComplexField()); - assertFalse(f.isArrayOfSimpleStruct()); - assertFalse(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - } - - @Test - public void only_struct_field_attributes_are_considered_when_tagging_a_complex_field() throws ParseException { - { - ComplexFixture f = new ComplexFixture("elem_array", - joinLines("field elem_array type array {", - " struct-field name { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertTrue(f.isArrayOfSimpleStruct()); - assertFalse(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - { - ComplexFixture f = new ComplexFixture("elem_map", - joinLines("field elem_map type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value.name { indexing: attribute }", - "}")); - assertTrue(f.isSupportedComplexField()); - assertFalse(f.isArrayOfSimpleStruct()); - assertTrue(f.isMapOfSimpleStruct()); - assertFalse(f.isMapOfPrimitiveType()); - assertFalse(f.isComplexFieldWithOnlyStructFieldAttributes()); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java deleted file mode 100644 index d6ff8fa1067..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.document; - -import java.util.Optional; -import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.MatcherAssert.assertThat; - -public class HnswIndexParamsTestCase { - - @Test - public void override_from() throws Exception { - var empty = new HnswIndexParams(); - var builder = new HnswIndexParams.Builder(); - builder.setMaxLinksPerNode(7); - builder.setMultiThreadedIndexing(false); - var one = builder.build(); - builder.setNeighborsToExploreAtInsert(42); - var three = builder.build(); - builder.setMaxLinksPerNode(17); - builder.setNeighborsToExploreAtInsert(500); - builder.setMultiThreadedIndexing(true); - var four = builder.build(); - - assertThat(empty.maxLinksPerNode(), is(16)); - assertThat(empty.neighborsToExploreAtInsert(), is(200)); - assertThat(empty.multiThreadedIndexing(), is(true)); - - assertThat(one.maxLinksPerNode(), is(7)); - assertThat(one.multiThreadedIndexing(), is(false)); - assertThat(three.neighborsToExploreAtInsert(), is(42)); - - assertThat(four.maxLinksPerNode(), is(17)); - assertThat(four.neighborsToExploreAtInsert(), is(500)); - assertThat(four.multiThreadedIndexing(), is(true)); - - var five = four.overrideFrom(Optional.of(empty)); - assertThat(five.maxLinksPerNode(), is(17)); - assertThat(five.neighborsToExploreAtInsert(), is(500)); - assertThat(five.multiThreadedIndexing(), is(true)); - - var six = four.overrideFrom(Optional.of(one)); - assertThat(six.maxLinksPerNode(), is(7)); - assertThat(six.neighborsToExploreAtInsert(), is(500)); - // This is explicitly set to false in 'one' - assertThat(six.multiThreadedIndexing(), is(false)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java deleted file mode 100644 index 127f5184c4c..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import static com.yahoo.config.model.test.TestUtil.joinLines; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThrows; - -/** - * @author arnej - */ -public class ConvertIntermediateTestCase { - - @Test - public void can_convert_minimal_schema() throws Exception { - String input = joinLines - ("schema foo {", - " document foo {", - " }", - "}"); - var collection = new IntermediateCollection(); - ParsedSchema schema = collection.addSchemaFromString(input); - assertEquals("foo", schema.getDocument().name()); - var docMan = new DocumentTypeManager(); - var converter = new ConvertSchemaCollection(collection, docMan); - converter.convertTypes(); - var dt = docMan.getDocumentType("foo"); - assertTrue(dt != null); - } - - @Test - public void can_convert_schema_files() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); - collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); - collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); - assertEquals(collection.getParsedSchemas().size(), 3); - var docMan = new DocumentTypeManager(); - var converter = new ConvertSchemaCollection(collection, docMan); - converter.convertTypes(); - var dt = docMan.getDocumentType("child"); - assertTrue(dt != null); - dt = docMan.getDocumentType("parent"); - assertTrue(dt != null); - dt = docMan.getDocumentType("grandparent"); - assertTrue(dt != null); - } - - @Test - public void can_convert_structs_and_annotations() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromFile("src/test/converter/child.sd"); - collection.addSchemaFromFile("src/test/converter/other.sd"); - collection.addSchemaFromFile("src/test/converter/parent.sd"); - collection.addSchemaFromFile("src/test/converter/grandparent.sd"); - var docMan = new DocumentTypeManager(); - var converter = new ConvertSchemaCollection(collection, docMan); - converter.convertTypes(); - var dt = docMan.getDocumentType("child"); - assertTrue(dt != null); - for (var parent : dt.getInheritedTypes()) { - System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); - } - for (var field : dt.fieldSetAll()) { - System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); - } - dt = docMan.getDocumentType("parent"); - assertTrue(dt != null); - for (var parent : dt.getInheritedTypes()) { - System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); - } - for (var field : dt.fieldSetAll()) { - System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); - } - dt = docMan.getDocumentType("grandparent"); - assertTrue(dt != null); - for (var parent : dt.getInheritedTypes()) { - System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); - } - for (var field : dt.fieldSetAll()) { - System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); - } - dt = docMan.getDocumentType("other"); - assertTrue(dt != null); - for (var parent : dt.getInheritedTypes()) { - System.err.println("dt "+dt.getName()+" inherits from "+parent.getName()); - } - for (var field : dt.fieldSetAll()) { - System.err.println("dt "+dt.getName()+" contains field "+field.getName()+" of type "+field.getDataType()); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java deleted file mode 100644 index 743795e51ce..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import static com.yahoo.config.model.test.TestUtil.joinLines; - -import java.nio.charset.StandardCharsets; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.util.List; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThrows; - -/** - * @author arnej - */ -public class IntermediateCollectionTestCase { - - @Test - public void can_add_minimal_schema() throws Exception { - String input = joinLines - ("schema foo {", - " document foo {", - " }", - "}"); - var collection = new IntermediateCollection(); - ParsedSchema schema = collection.addSchemaFromString(input); - assertEquals("foo", schema.name()); - assertTrue(schema.hasDocument()); - assertEquals("foo", schema.getDocument().name()); - } - - @Test - public void names_may_differ() throws Exception { - String input = joinLines - ("schema foo_search {", - " document foo {", - " }", - "}"); - var collection = new IntermediateCollection(); - ParsedSchema schema = collection.addSchemaFromString(input); - assertEquals("foo_search", schema.name()); - assertTrue(schema.hasDocument()); - assertEquals("foo", schema.getDocument().name()); - } - - @Test - public void can_add_schema_files() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); - collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); - collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); - var schemes = collection.getParsedSchemas(); - assertEquals(schemes.size(), 3); - var schema = schemes.get("child"); - assertTrue(schema != null); - assertEquals(schema.name(), "child"); - schema = schemes.get("parent"); - assertTrue(schema != null); - assertEquals(schema.name(), "parent"); - schema = schemes.get("grandparent"); - assertTrue(schema != null); - assertEquals(schema.name(), "grandparent"); - } - - NamedReader readerOf(String fileName) throws Exception { - File f = new File(fileName); - FileReader fr = new FileReader(f, StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(fr); - return new NamedReader(fileName, br); - } - - @Test - public void can_add_schemas() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromReader(readerOf("src/test/derived/deriver/child.sd")); - collection.addSchemaFromReader(readerOf("src/test/derived/deriver/grandparent.sd")); - collection.addSchemaFromReader(readerOf("src/test/derived/deriver/parent.sd")); - var schemes = collection.getParsedSchemas(); - assertEquals(schemes.size(), 3); - var schema = schemes.get("child"); - assertTrue(schema != null); - assertEquals(schema.name(), "child"); - schema = schemes.get("parent"); - assertTrue(schema != null); - assertEquals(schema.name(), "parent"); - schema = schemes.get("grandparent"); - assertTrue(schema != null); - assertEquals(schema.name(), "grandparent"); - } - - ParsedRankProfile get(List all, String name) { - for (var rp : all) { - if (rp.name().equals(name)) return rp; - } - return null; - } - - @Test - public void can_add_extra_rank_profiles() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); - collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); - collection.addRankProfileFile("test", readerOf("src/test/derived/rankprofilemodularity/test/subdirectory/outside_schema2.profile")); - var schemes = collection.getParsedSchemas(); - assertEquals(schemes.size(), 1); - var schema = schemes.get("test"); - assertTrue(schema != null); - assertEquals(schema.name(), "test"); - var rankProfiles = schema.getRankProfiles(); - assertEquals(rankProfiles.size(), 7); - var outside = get(rankProfiles, "outside_schema1"); - assertTrue(outside != null); - assertEquals(outside.name(), "outside_schema1"); - var functions = outside.getFunctions(); - assertEquals(functions.size(), 1); - assertEquals(functions.get(0).name(), "fo1"); - outside = get(rankProfiles, "outside_schema2"); - assertTrue(outside != null); - assertEquals(outside.name(), "outside_schema2"); - functions = outside.getFunctions(); - assertEquals(functions.size(), 1); - assertEquals(functions.get(0).name(), "fo2"); - } - - @Test - public void name_mismatch_throws() throws Exception { - var collection = new IntermediateCollection(); - var ex = assertThrows(IllegalArgumentException.class, () -> - collection.addSchemaFromReader(readerOf("src/test/cfg/application/sdfilenametest/schemas/notmusic.sd"))); - assertEquals("The file containing schema 'music' must be named 'music.sd', was 'notmusic.sd'", - ex.getMessage()); - } - - @Test - public void bad_parse_throws() throws Exception { - var collection = new IntermediateCollection(); - var ex = assertThrows(ParseException.class, () -> - collection.addSchemaFromFile("src/test/examples/badparse.sd")); - assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/badparse.sd: Encountered")); - ex = assertThrows(ParseException.class, () -> - collection.addSchemaFromReader(readerOf("src/test/examples/badparse.sd"))); - assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/badparse.sd: Encountered")); - collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); - collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); - ex = assertThrows(ParseException.class, () -> - collection.addRankProfileFile("test", "src/test/examples/badparse.sd")); - assertTrue(ex.getMessage().startsWith("Failed parsing rank-profile from src/test/examples/badparse.sd: Encountered")); - } - - @Test - public void can_resolve_document_inheritance() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromFile("src/test/derived/deriver/child.sd"); - collection.addSchemaFromFile("src/test/derived/deriver/grandparent.sd"); - collection.addSchemaFromFile("src/test/derived/deriver/parent.sd"); - collection.resolveInternalConnections(); - var schemes = collection.getParsedSchemas(); - assertEquals(schemes.size(), 3); - var childDoc = schemes.get("child").getDocument(); - var inherits = childDoc.getResolvedInherits(); - assertEquals(inherits.size(), 1); - var parentDoc = inherits.get(0); - assertEquals(parentDoc.name(), "parent"); - inherits = parentDoc.getResolvedInherits(); - assertEquals(inherits.size(), 1); - assertEquals(inherits.get(0).name(), "grandparent"); - } - - @Test - public void can_detect_schema_inheritance_cycles() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromString("schema foo inherits bar { document foo {} }"); - collection.addSchemaFromString("schema bar inherits qux { document bar {} }"); - collection.addSchemaFromString("schema qux inherits foo { document qux {} }"); - assertEquals(collection.getParsedSchemas().size(), 3); - var ex = assertThrows(IllegalArgumentException.class, () -> - collection.resolveInternalConnections()); - assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for schemas: ")); - } - - @Test - public void can_detect_document_inheritance_cycles() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromString("schema foo { document foo inherits bar {} }"); - collection.addSchemaFromString("schema bar { document bar inherits qux {} }"); - collection.addSchemaFromString("schema qux { document qux inherits foo {} }"); - assertEquals(collection.getParsedSchemas().size(), 3); - var ex = assertThrows(IllegalArgumentException.class, () -> - collection.resolveInternalConnections()); - System.err.println("ex: "+ex.getMessage()); - assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); - } - - @Test - public void can_detect_missing_doc() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromString("schema foo { document foo inherits bar {} }"); - collection.addSchemaFromString("schema qux { document qux inherits foo {} }"); - assertEquals(collection.getParsedSchemas().size(), 2); - var ex = assertThrows(IllegalArgumentException.class, () -> - collection.resolveInternalConnections()); - assertEquals("document foo inherits from unavailable document bar", ex.getMessage()); - } - - @Test - public void can_detect_document_reference_cycle() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromString("schema foo { document foo { field oneref type reference {} } }"); - collection.addSchemaFromString("schema bar { document bar { field tworef type reference {} } }"); - assertEquals(collection.getParsedSchemas().size(), 2); - var ex = assertThrows(IllegalArgumentException.class, () -> - collection.resolveInternalConnections()); - System.err.println("ex: "+ex.getMessage()); - assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); - } - - @Test - public void can_detect_cycles_with_reference() throws Exception { - var collection = new IntermediateCollection(); - collection.addSchemaFromString("schema foo { document foodoc inherits bardoc {} }"); - collection.addSchemaFromString("schema bar { document bardoc { field myref type reference { } } }"); - collection.addSchemaFromString("schema qux inherits foo { document qux inherits foodoc {} }"); - assertEquals(collection.getParsedSchemas().size(), 3); - var ex = assertThrows(IllegalArgumentException.class, () -> - collection.resolveInternalConnections()); - System.err.println("ex: "+ex.getMessage()); - assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); - } - - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java deleted file mode 100644 index 9ae7b3aa4b8..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; - -/** - * @author arnej - */ -public class ParsedDocumentTestCase { - - @Test - public void fields_can_be_added_once() throws Exception { - var doc = new ParsedDocument("foo"); - var stringType = ParsedType.fromName("string"); - doc.addField(new ParsedField("bar1", stringType)); - doc.addField(new ParsedField("zap", stringType)); - doc.addField(new ParsedField("bar2", stringType)); - doc.addField(new ParsedField("bar3", stringType)); - var e = assertThrows(IllegalArgumentException.class, () -> - doc.addField(new ParsedField("zap", stringType))); - System.err.println("As expected: "+e); - assertEquals("document 'foo' error: Duplicate (case insensitively) field 'zap' in document type 'foo'", e.getMessage()); - e = assertThrows(IllegalArgumentException.class, () -> - doc.addField(new ParsedField("ZAP", stringType))); - assertEquals("document 'foo' error: Duplicate (case insensitively) field 'ZAP' in document type 'foo'", e.getMessage()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java deleted file mode 100644 index 2284acc705c..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.io.IOUtils; -import static com.yahoo.config.model.test.TestUtil.joinLines; - -import java.io.File; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThrows; - -/** - * @author arnej - */ -public class SchemaParserTestCase { - - ParsedSchema parseString(String input) throws Exception { - var deployLogger = new BaseDeployLogger(); - var modelProperties = new TestProperties(); - var stream = new SimpleCharStream(input); - try { - var parser = new SchemaParser(stream, deployLogger, modelProperties); - return parser.schema(); - } catch (ParseException pe) { - throw new ParseException(stream.formatException(pe.getMessage())); - } - } - - ParsedSchema parseFile(String fileName) throws Exception { - File file = new File(fileName); - return parseString(IOUtils.readFile(file)); - } - - @Test - public void minimal_schema_can_be_parsed() throws Exception { - String input = joinLines - ("schema foo {", - " document foo {", - " }", - "}"); - ParsedSchema schema = parseString(input); - assertEquals("foo", schema.name()); - assertTrue(schema.hasDocument()); - assertEquals("foo", schema.getDocument().name()); - } - - @Test - public void document_only_can_be_parsed() throws Exception { - String input = joinLines - ("document bar {", - "}"); - ParsedSchema schema = parseString(input); - assertEquals("bar", schema.name()); - assertTrue(schema.hasDocument()); - assertEquals("bar", schema.getDocument().name()); - } - - @Test - public void multiple_documents_disallowed() { - String input = joinLines - ("schema foo {", - " document foo {", - " }", - " document foo2 {", - " }", - "}"); - var e = assertThrows(IllegalArgumentException.class, () -> parseString(input)); - assertEquals("schema 'foo' error: already has document 'foo' so cannot add document 'foo2'", e.getMessage()); - } - - @Test - public void backwards_path_is_disallowed() { - assertEquals("'..' is not allowed in path", - assertThrows(IllegalArgumentException.class, - () -> parseString("schema foo {\n" + - " constant my_constant_tensor {\n" + - " file: foo/../bar\n" + - " type: tensor(x{},y{})\n" + - " }\n" + - "}\n")).getMessage()); - } - - void checkFileParses(String fileName) throws Exception { - var schema = parseFile(fileName); - assertNotNull(schema); - assertNotNull(schema.name()); - assertNotEquals("", schema.name()); - } - - // TODO: Many (all)? of the files below are parsed from other tests and can be removed from here - @Test - public void parse_various_old_sdfiles() throws Exception { - checkFileParses("src/test/cfg/search/data/travel/schemas/TTData.sd"); - checkFileParses("src/test/cfg/search/data/travel/schemas/TTEdge.sd"); - checkFileParses("src/test/cfg/search/data/travel/schemas/TTPOI.sd"); - checkFileParses("src/test/configmodel/types/other_doc.sd"); - checkFileParses("src/test/configmodel/types/types.sd"); - checkFileParses("src/test/configmodel/types/type_with_doc_field.sd"); - checkFileParses("src/test/derived/advanced/advanced.sd"); - checkFileParses("src/test/derived/annotationsimplicitstruct/annotationsimplicitstruct.sd"); - checkFileParses("src/test/derived/annotationsinheritance2/annotationsinheritance2.sd"); - checkFileParses("src/test/derived/annotationsinheritance/annotationsinheritance.sd"); - checkFileParses("src/test/derived/annotationsoutsideofdocument/annotationsoutsideofdocument.sd"); - checkFileParses("src/test/derived/annotationspolymorphy/annotationspolymorphy.sd"); - checkFileParses("src/test/derived/annotationsreference2/annotationsreference2.sd"); - checkFileParses("src/test/derived/annotationsreference/annotationsreference.sd"); - checkFileParses("src/test/derived/annotationssimple/annotationssimple.sd"); - checkFileParses("src/test/derived/annotationsstruct/annotationsstruct.sd"); - checkFileParses("src/test/derived/annotationsstructarray/annotationsstructarray.sd"); - checkFileParses("src/test/derived/array_of_struct_attribute/test.sd"); - checkFileParses("src/test/derived/arrays/arrays.sd"); - checkFileParses("src/test/derived/attributeprefetch/attributeprefetch.sd"); - checkFileParses("src/test/derived/attributerank/attributerank.sd"); - checkFileParses("src/test/derived/attributes/attributes.sd"); - checkFileParses("src/test/derived/combinedattributeandindexsearch/combinedattributeandindexsearch.sd"); - checkFileParses("src/test/derived/complex/complex.sd"); - checkFileParses("src/test/derived/deriver/child.sd"); - checkFileParses("src/test/derived/deriver/grandparent.sd"); - checkFileParses("src/test/derived/deriver/parent.sd"); - checkFileParses("src/test/derived/emptychild/child.sd"); - checkFileParses("src/test/derived/emptychild/parent.sd"); - checkFileParses("src/test/derived/emptydefault/emptydefault.sd"); - checkFileParses("src/test/derived/exactmatch/exactmatch.sd"); - checkFileParses("src/test/derived/fieldset/test.sd"); - checkFileParses("src/test/derived/flickr/flickrphotos.sd"); - checkFileParses("src/test/derived/function_arguments/test.sd"); - checkFileParses("src/test/derived/function_arguments_with_expressions/test.sd"); - checkFileParses("src/test/derived/gemini2/gemini.sd"); - checkFileParses("src/test/derived/hnsw_index/test.sd"); - checkFileParses("src/test/derived/id/id.sd"); - checkFileParses("src/test/derived/importedfields/child.sd"); - checkFileParses("src/test/derived/importedfields/grandparent.sd"); - checkFileParses("src/test/derived/imported_fields_inherited_reference/child_a.sd"); - checkFileParses("src/test/derived/imported_fields_inherited_reference/child_b.sd"); - checkFileParses("src/test/derived/imported_fields_inherited_reference/child_c.sd"); - checkFileParses("src/test/derived/imported_fields_inherited_reference/parent.sd"); - checkFileParses("src/test/derived/importedfields/parent_a.sd"); - checkFileParses("src/test/derived/importedfields/parent_b.sd"); - checkFileParses("src/test/derived/imported_position_field/child.sd"); - checkFileParses("src/test/derived/imported_position_field/parent.sd"); - checkFileParses("src/test/derived/imported_position_field_summary/child.sd"); - checkFileParses("src/test/derived/imported_position_field_summary/parent.sd"); - checkFileParses("src/test/derived/imported_struct_fields/child.sd"); - checkFileParses("src/test/derived/imported_struct_fields/parent.sd"); - checkFileParses("src/test/derived/indexinfo_fieldsets/indexinfo_fieldsets.sd"); - checkFileParses("src/test/derived/indexinfo_lowercase/indexinfo_lowercase.sd"); - checkFileParses("src/test/derived/indexschema/indexschema.sd"); - checkFileParses("src/test/derived/indexswitches/indexswitches.sd"); - checkFileParses("src/test/derived/inheritance/child.sd"); - checkFileParses("src/test/derived/inheritance/father.sd"); - checkFileParses("src/test/derived/inheritance/grandparent.sd"); - checkFileParses("src/test/derived/inheritance/mother.sd"); - checkFileParses("src/test/derived/inheritdiamond/child.sd"); - checkFileParses("src/test/derived/inheritdiamond/father.sd"); - checkFileParses("src/test/derived/inheritdiamond/grandparent.sd"); - checkFileParses("src/test/derived/inheritdiamond/mother.sd"); - checkFileParses("src/test/derived/inheritfromgrandparent/child.sd"); - checkFileParses("src/test/derived/inheritfromgrandparent/grandparent.sd"); - checkFileParses("src/test/derived/inheritfromgrandparent/parent.sd"); - checkFileParses("src/test/derived/inheritfromnull/inheritfromnull.sd"); - checkFileParses("src/test/derived/inheritfromparent/child.sd"); - checkFileParses("src/test/derived/inheritfromparent/parent.sd"); - checkFileParses("src/test/derived/inheritstruct/child.sd"); - checkFileParses("src/test/derived/inheritstruct/parent.sd"); - checkFileParses("src/test/derived/integerattributetostringindex/integerattributetostringindex.sd"); - checkFileParses("src/test/derived/language/language.sd"); - checkFileParses("src/test/derived/lowercase/lowercase.sd"); - checkFileParses("src/test/derived/mail/mail.sd"); - checkFileParses("src/test/derived/map_attribute/test.sd"); - checkFileParses("src/test/derived/map_of_struct_attribute/test.sd"); - checkFileParses("src/test/derived/mlr/mlr.sd"); - checkFileParses("src/test/derived/multiplesummaries/multiplesummaries.sd"); - checkFileParses("src/test/derived/music3/music3.sd"); - checkFileParses("src/test/derived/music/music.sd"); - checkFileParses("src/test/derived/namecollision/collision.sd"); - checkFileParses("src/test/derived/namecollision/collisionstruct.sd"); - checkFileParses("src/test/derived/nearestneighbor/test.sd"); - checkFileParses("src/test/derived/newrank/newrank.sd"); - checkFileParses("src/test/derived/nuwa/newsindex.sd"); - checkFileParses("src/test/derived/orderilscripts/orderilscripts.sd"); - checkFileParses("src/test/derived/position_array/position_array.sd"); - checkFileParses("src/test/derived/position_attribute/position_attribute.sd"); - checkFileParses("src/test/derived/position_extra/position_extra.sd"); - checkFileParses("src/test/derived/position_nosummary/position_nosummary.sd"); - checkFileParses("src/test/derived/position_summary/position_summary.sd"); - checkFileParses("src/test/derived/predicate_attribute/predicate_attribute.sd"); - checkFileParses("src/test/derived/prefixexactattribute/prefixexactattribute.sd"); - checkFileParses("src/test/derived/rankingexpression/rankexpression.sd"); - checkFileParses("src/test/derived/rankprofileinheritance/child.sd"); - checkFileParses("src/test/derived/rankprofileinheritance/parent1.sd"); - checkFileParses("src/test/derived/rankprofileinheritance/parent2.sd"); - checkFileParses("src/test/derived/rankprofilemodularity/test.sd"); - checkFileParses("src/test/derived/rankprofiles/rankprofiles.sd"); - checkFileParses("src/test/derived/rankproperties/rankproperties.sd"); - checkFileParses("src/test/derived/ranktypes/ranktypes.sd"); - checkFileParses("src/test/derived/reference_fields/ad.sd"); - checkFileParses("src/test/derived/reference_fields/campaign.sd"); - checkFileParses("src/test/derived/renamedfeatures/foo.sd"); - checkFileParses("src/test/derived/reserved_position/reserved_position.sd"); - checkFileParses("src/test/derived/schemainheritance/child.sd"); - checkFileParses("src/test/derived/schemainheritance/importedschema.sd"); - checkFileParses("src/test/derived/schemainheritance/parent.sd"); - checkFileParses("src/test/derived/slice/test.sd"); - checkFileParses("src/test/derived/streamingjuniper/streamingjuniper.sd"); - checkFileParses("src/test/derived/streamingstructdefault/streamingstructdefault.sd"); - checkFileParses("src/test/derived/streamingstruct/streamingstruct.sd"); - checkFileParses("src/test/derived/structandfieldset/test.sd"); - checkFileParses("src/test/derived/structanyorder/structanyorder.sd"); - checkFileParses("src/test/derived/structinheritance/bad.sd"); - checkFileParses("src/test/derived/structinheritance/simple.sd"); - checkFileParses("src/test/derived/tensor2/first.sd"); - checkFileParses("src/test/derived/tensor2/second.sd"); - checkFileParses("src/test/derived/tensor/tensor.sd"); - checkFileParses("src/test/derived/tokenization/tokenization.sd"); - checkFileParses("src/test/derived/twostreamingstructs/streamingstruct.sd"); - checkFileParses("src/test/derived/twostreamingstructs/whatever.sd"); - checkFileParses("src/test/derived/types/types.sd"); - checkFileParses("src/test/derived/uri_array/uri_array.sd"); - checkFileParses("src/test/derived/uri_wset/uri_wset.sd"); - checkFileParses("src/test/examples/arrays.sd"); - checkFileParses("src/test/examples/arraysweightedsets.sd"); - checkFileParses("src/test/examples/attributeposition.sd"); - checkFileParses("src/test/examples/attributesettings.sd"); - checkFileParses("src/test/examples/attributesexactmatch.sd"); - checkFileParses("src/test/examples/casing.sd"); - checkFileParses("src/test/examples/comment.sd"); - checkFileParses("src/test/examples/documentidinsummary.sd"); - checkFileParses("src/test/examples/fieldoftypedocument.sd"); - checkFileParses("src/test/examples/implicitsummaries_attribute.sd"); - checkFileParses("src/test/examples/implicitsummaryfields.sd"); - checkFileParses("src/test/examples/incorrectrankingexpressionfileref.sd"); - checkFileParses("src/test/examples/indexing_extra.sd"); - checkFileParses("src/test/examples/indexing_modify_field_no_output.sd"); - checkFileParses("src/test/examples/indexing.sd"); - checkFileParses("src/test/examples/indexrewrite.sd"); - checkFileParses("src/test/examples/indexsettings.sd"); - checkFileParses("src/test/examples/integerindex2attribute.sd"); - checkFileParses("src/test/examples/invalidimplicitsummarysource.sd"); - checkFileParses("src/test/examples/multiplesummaries.sd"); - checkFileParses("src/test/examples/music.sd"); - checkFileParses("src/test/examples/nextgen/boldedsummaryfields.sd"); - checkFileParses("src/test/examples/nextgen/dynamicsummaryfields.sd"); - checkFileParses("src/test/examples/nextgen/extrafield.sd"); - checkFileParses("src/test/examples/nextgen/implicitstructtypes.sd"); - checkFileParses("src/test/examples/nextgen/simple.sd"); - checkFileParses("src/test/examples/nextgen/summaryfield.sd"); - checkFileParses("src/test/examples/nextgen/toggleon.sd"); - checkFileParses("src/test/examples/nextgen/untransformedsummaryfields.sd"); - checkFileParses("src/test/examples/ngram.sd"); - checkFileParses("src/test/examples/outsidedoc.sd"); - checkFileParses("src/test/examples/outsidesummary.sd"); - checkFileParses("src/test/examples/position_array.sd"); - checkFileParses("src/test/examples/position_attribute.sd"); - checkFileParses("src/test/examples/position_base.sd"); - checkFileParses("src/test/examples/position_extra.sd"); - checkFileParses("src/test/examples/position_index.sd"); - checkFileParses("src/test/examples/position_inherited.sd"); - checkFileParses("src/test/examples/position_summary.sd"); - checkFileParses("src/test/examples/rankmodifier/literal.sd"); - checkFileParses("src/test/examples/rankpropvars.sd"); - checkFileParses("src/test/examples/reserved_words_as_field_names.sd"); - checkFileParses("src/test/examples/simple.sd"); - checkFileParses("src/test/examples/stemmingdefault.sd"); - checkFileParses("src/test/examples/stemmingsetting.sd"); - checkFileParses("src/test/examples/strange.sd"); - checkFileParses("src/test/examples/struct.sd"); - checkFileParses("src/test/examples/summaryfieldcollision.sd"); - checkFileParses("src/test/examples/weightedset-summaryto.sd"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java deleted file mode 100644 index 0adaba4cf68..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.ImportedSimpleField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import org.junit.Test; - -import java.util.Collections; - -import static org.junit.Assert.assertEquals; - -/** - * @author bjorncs - */ -public class AddAttributeTransformToSummaryOfImportedFieldsTest { - - private static final String IMPORTED_FIELD_NAME = "imported_myfield"; - private static final String DOCUMENT_NAME = "mydoc"; - private static final String SUMMARY_NAME = "mysummary"; - - @Test - public void attribute_summary_transform_applied_to_summary_field_of_imported_field() { - Schema schema = createSearchWithDocument(DOCUMENT_NAME); - schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME)); - schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, schema)); - - AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( - schema, null, null, null); - processor.process(true, false); - SummaryField summaryField = schema.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); - SummaryTransform actualTransform = summaryField.getTransform(); - assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); - } - - private static Schema createSearch(String documentType) { - return new Schema(documentType, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); - } - - private static Schema createSearchWithDocument(String documentName) { - Schema schema = createSearch(documentName); - SDDocumentType document = new SDDocumentType(documentName, schema); - schema.addDocument(document); - return schema; - } - - private static ImportedFields createSingleImportedField(String fieldName) { - Schema targetSchema = createSearchWithDocument("target_doc"); - var doc = targetSchema.getDocument(); - SDField targetField = new SDField(doc, "target_field", DataType.INT); - DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); - ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); - return new ImportedFields(Collections.singletonMap(fieldName, importedField)); - } - - private static DocumentSummary createDocumentSummary(String fieldName, Schema schema) { - DocumentSummary summary = new DocumentSummary("mysummary", schema); - summary.add(new SummaryField(fieldName, DataType.INT)); - return summary; - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java deleted file mode 100644 index 926e26451d7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -public class AdjustPositionSummaryFieldsTestCase { - - @Test - public void test_pos_summary() { - SearchModel model = new SearchModel(false); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); - model.resolve(); - model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "pos_zcurve"); - model.assertSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos_zcurve"); - model.assertSummaryField("my_pos.distance", DataType.INT, SummaryTransform.DISTANCE, "pos_zcurve"); - } - - @Test - public void test_imported_pos_summary() { - SearchModel model = new SearchModel(); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); - model.resolve(); - model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "my_pos_zcurve"); - model.assertSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "my_pos_zcurve"); - model.assertSummaryField("my_pos.distance", DataType.INT, SummaryTransform.DISTANCE, "my_pos_zcurve"); - } - - @Test - public void test_imported_pos_summary_bad_source() { - SearchModel model = new SearchModel(); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); - model.resolve(); - // SummaryFieldsMustHaveValidSource processing not run in this test. - model.assertSummaryField("my_pos", PositionDataType.INSTANCE, SummaryTransform.NONE, "pos"); - model.assertNoSummaryField("my_pos.position"); - model.assertNoSummaryField("my_pos.distance"); - } - - @Test - public void test_imported_pos_summary_bad_datatype() { - SearchModel model = new SearchModel(); - model.addSummaryField("my_pos", DataType.getArray(PositionDataType.INSTANCE), null, "pos"); - model.resolve(); - model.assertSummaryField("my_pos", DataType.getArray(PositionDataType.INSTANCE), SummaryTransform.NONE, "pos"); - model.assertNoSummaryField("my_pos.position"); - model.assertNoSummaryField("my_pos.distance"); - } - - @Test - public void test_pos_summary_no_attr_no_rename() { - SearchModel model = new SearchModel(false, false, false); - model.addSummaryField("pos", PositionDataType.INSTANCE, null, "pos"); - model.resolve(); - model.assertSummaryField("pos", PositionDataType.INSTANCE, SummaryTransform.NONE, "pos"); - model.assertNoSummaryField("pos.position"); - model.assertNoSummaryField("pos.distance"); - } - - @Test - public void test_pos_default_summary_no_attr_no_rename() { - SearchModel model = new SearchModel(false, false, false); - model.resolve(); - assertNull(model.childSchema.getSummary("default")); // ImplicitSummaries processing not run in this test - } - - @Test - public void test_pos_summary_no_rename() { - SearchModel model = new SearchModel(false, true, false); - model.addSummaryField("pos", PositionDataType.INSTANCE, null, "pos"); - model.resolve(); - model.assertSummaryField("pos", PositionDataType.INSTANCE, SummaryTransform.GEOPOS, "pos_zcurve"); - model.assertSummaryField("pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos_zcurve"); - model.assertSummaryField("pos.distance", DataType.INT, SummaryTransform.DISTANCE, "pos_zcurve"); - } - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void test_pos_summary_no_attr() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', field 'my_pos': No position attribute 'pos_zcurve'"); - SearchModel model = new SearchModel(false, false, false); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); - model.resolve(); - } - - @Test - public void test_pos_summary_bad_attr() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', field 'my_pos': No position attribute 'pos_zcurve'"); - SearchModel model = new SearchModel(false, false, true); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, "pos"); - model.resolve(); - } - - @Test - public void test_imported_pos_summary_no_attr() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', import field 'my_pos_zcurve': " - + "Field 'pos_zcurve' via reference field 'ref': Not found"); - SearchModel model = new SearchModel(true, false, false); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); - model.resolve(); - } - - @Test - public void test_imported_pos_summary_bad_attr() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', field 'my_pos': " - + "No position attribute 'my_pos_zcurve'"); - SearchModel model = new SearchModel(true, false, true); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); - model.resolve(); - } - - @Test - public void test_my_pos_position_summary_bad_datatype() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " - + "exists with type 'datatype string (code: 2)', should be of type 'datatype Array (code: -1486737430)"); - SearchModel model = new SearchModel(); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); - model.addSummaryField("my_pos.position", DataType.STRING, null, "pos"); - model.resolve(); - } - - @Test - public void test_my_pos_position_summary_bad_transform() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " - + "has summary transform 'none', should have transform 'positions'"); - SearchModel model = new SearchModel(); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); - model.addSummaryField("my_pos.position", DataType.getArray(DataType.STRING), null, "pos"); - model.resolve(); - } - - @Test - public void test_my_pos_position_summary_bad_source() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', field 'my_pos.position': " - + "has source '[source field 'pos']', should have source 'source field 'my_pos_zcurve''"); - SearchModel model = new SearchModel(); - model.addSummaryField("my_pos", PositionDataType.INSTANCE, null, null); - model.addSummaryField("my_pos.position", DataType.getArray(DataType.STRING), SummaryTransform.POSITIONS, "pos"); - model.resolve(); - } - - static class SearchModel extends ParentChildSearchModel { - - SearchModel() { - this(true); - } - - SearchModel(boolean importedPos) { - this(importedPos, true, false); - } - - SearchModel(boolean importedPos, boolean setupPosAttr, boolean setupBadAttr) { - super(); - if (importedPos) { - createPositionField(parentSchema, setupPosAttr, setupBadAttr); - } - addRefField(childSchema, parentSchema, "ref"); - if (importedPos) { - addImportedField("my_pos", "ref", "pos"); - } else { - createPositionField(childSchema, setupPosAttr, setupBadAttr); - } - } - - private void createPositionField(Schema schema, boolean setupPosAttr, boolean setupBadAttr) { - String ilScript = setupPosAttr ? "{ summary | attribute }" : "{ summary }"; - var doc = schema.getDocument(); - doc.addField(createField(doc, "pos", PositionDataType.INSTANCE, ilScript)); - if (setupBadAttr) { - doc.addField(createField(doc, "pos_zcurve", DataType.LONG, "{ attribute }")); - } - } - - void addSummaryField(String fieldName, DataType dataType, SummaryTransform transform, String source) { - addSummaryField("my_summary", fieldName, dataType, transform, source); - } - - public void addSummaryField(String summaryName, String fieldName, DataType dataType, SummaryTransform transform, String source) { - DocumentSummary summary = childSchema.getSummary(summaryName); - if (summary == null) { - summary = new DocumentSummary(summaryName, childSchema); - childSchema.addSummary(summary); - } - SummaryField summaryField = new SummaryField(fieldName, dataType); - if (source != null) { - summaryField.addSource(source); - } - if (transform != null) { - summaryField.setTransform(transform); - } - summary.add(summaryField); - } - - public void assertNoSummaryField(String fieldName) { - assertNoSummaryField("my_summary", fieldName); - } - - public void assertNoSummaryField(String summaryName, String fieldName) { - DocumentSummary summary = childSchema.getSummary(summaryName); - assertNotNull(summary); - SummaryField summaryField = summary.getSummaryField(fieldName); - assertNull(summaryField); - } - - public void assertSummaryField(String fieldName, DataType dataType, SummaryTransform transform, String source) { - assertSummaryField("my_summary", fieldName, dataType, transform, source); - } - - public void assertSummaryField(String summaryName, String fieldName, DataType dataType, SummaryTransform transform, String source) { - DocumentSummary summary = childSchema.getSummary(summaryName); - assertNotNull(summary); - SummaryField summaryField = summary.getSummaryField(fieldName); - assertNotNull(summaryField); - assertEquals(dataType, summaryField.getDataType()); - assertEquals(transform, summaryField.getTransform()); - if (source == null) { - assertEquals(0, summaryField.getSourceCount()); - } else { - assertEquals(1, summaryField.getSourceCount()); - assertEquals(source, summaryField.getSingleSource()); - } - } - - public void resolve() { - resolve(parentSchema); - resolve(childSchema); - } - - private static void resolve(Schema schema) { - new CreatePositionZCurve(schema, null, null, null).process(true, false); - assertNotNull(schema.temporaryImportedFields().get()); - assertFalse(schema.importedFields().isPresent()); - new ImportedFieldsResolver(schema, null, null, null).process(true, false); - assertNotNull(schema.importedFields().get()); - new AdjustPositionSummaryFields(schema, null, null, null).process(true, false); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java deleted file mode 100644 index 8a0c1e2fe71..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.IndexingScript; -import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import com.yahoo.vespa.indexinglanguage.parser.ParseException; - -import java.util.LinkedList; -import java.util.List; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author Simon Thoresen Hult - */ -public abstract class AssertIndexingScript { - - public static void assertIndexing(List expected, Schema schema) { - assertIndexing(expected, new IndexingScript(schema).expressions()); - } - - public static void assertIndexing(List expected, IndexingScript script) { - assertIndexing(expected, script.expressions()); - } - - public static void assertIndexing(List expected, Iterable actual) { - List parsedExpected = new LinkedList<>(); - for (String str : expected) { - try { - parsedExpected.add(Expression.fromString(str).toString()); - } catch (ParseException e) { - fail(e.getMessage()); - } - } - for (Expression actualExp : actual) { - String str = actualExp.toString(); - assertTrue("Unexpected: " + str, parsedExpected.remove(str)); - } - assertTrue("Missing: " + parsedExpected.toString(), parsedExpected.isEmpty()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java deleted file mode 100644 index 534c82157e4..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; - -import java.io.IOException; - -import static org.junit.Assert.*; - -/** - * @author Simon Thoresen Hult - */ -public abstract class AssertSearchBuilder { - - public static void assertBuilds(String searchDefinitionFileName) throws IOException, ParseException { - assertNotNull(ApplicationBuilder.buildFromFile(searchDefinitionFileName)); - } - - public static void assertBuildFails(String searchDefinitionFileName, String expectedException) - throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile(searchDefinitionFileName); - fail(searchDefinitionFileName); - } catch (IllegalArgumentException e) { - assertEquals(expectedException, e.getMessage()); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java deleted file mode 100644 index e30ac8f8e00..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -/** - * Attributes should be implicitly exact-match in some cases - * @author vegardh - * - */ -public class AttributesExactMatchTestCase extends AbstractSchemaTestCase { - @Test - public void testAttributesExactMatch() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); - assertEquals(schema.getConcreteField("color").getMatching().getType(), MatchType.EXACT); - assertEquals(schema.getConcreteField("artist").getMatching().getType(), MatchType.WORD); - assertEquals(schema.getConcreteField("drummer").getMatching().getType(), MatchType.WORD); - assertEquals(schema.getConcreteField("guitarist").getMatching().getType(), MatchType.TEXT); - assertEquals(schema.getConcreteField("saxophonist_arr").getMatching().getType(), MatchType.WORD); - assertEquals(schema.getConcreteField("flutist").getMatching().getType(), MatchType.TEXT); - - assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.EXACT)); - assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.EXACT)); - assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.EXACT)); - assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.WORD)); - assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.WORD)); - assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.WORD)); - - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java deleted file mode 100644 index efec9206ed9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class BoldingTestCase extends AbstractSchemaTestCase { - - private final String boldonnonstring = - "search boldnonstring {\n" + - " document boldnonstring {\n" + - " field title type string {\n" + - " indexing: summary | index\n" + - " }\n" + - "\n" + - " field year4 type int {\n" + - " indexing: summary | attribute\n" + - " bolding: on\n" + - " }\n" + - " }\n" + - "}\n"; - - @Test - public void testBoldOnNonString() throws ParseException { - try { - ApplicationBuilder.createFromString(boldonnonstring); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("'bolding: on' for non-text field 'year4' (datatype int (code: 0)) is not allowed", - e.getMessage()); - } - } - - private final String boldonarray = - "search boldonarray {\n" + - " document boldonarray {\n" + - " field myarray type array {\n" + - " indexing: summary | index\n" + - " bolding: on\n" + - " }\n" + - " }\n" + - "}\n"; - - @Test - public void testBoldOnArray() throws ParseException { - try { - ApplicationBuilder.createFromString(boldonarray); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("'bolding: on' for non-text field 'myarray' (datatype Array (code: -1486737430)) is not allowed", - e.getMessage()); - } - } - -} - - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java deleted file mode 100644 index db27bbbb84d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author geirst - */ -public class BoolAttributeValidatorTestCase { - - @Test - public void array_of_bool_attribute_is_not_supported() throws ParseException { - try { - createFromString(getSd("field b type array { indexing: attribute }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", - e.getMessage()); - } - } - - @Test - public void weigtedset_of_bool_attribute_is_not_supported() throws ParseException { - try { - createFromString(getSd("field b type weightedset { indexing: attribute }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported", - e.getMessage()); - } - } - - private String getSd(String field) { - return joinLines( - "schema test {", - " document test {", - " " + field, - " }", - "}"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java deleted file mode 100644 index c0124ca0386..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.test.TestUtil; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.config.search.AttributesConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -/** - * Test configuration of dictionary control. - * - * @author baldersheim - */ -public class DictionaryTestCase { - private static AttributesConfig getConfig(Schema schema) { - AttributeFields attributes = new AttributeFields(schema); - AttributesConfig.Builder builder = new AttributesConfig.Builder(); - attributes.getConfig(builder, AttributeFields.FieldSet.ALL, 130000, true); - return builder.build(); - } - private Schema createSearch(String def) throws ParseException { - ApplicationBuilder sb = ApplicationBuilder.createFromString(def); - return sb.getSchema(); - } - @Test - public void testDefaultDictionarySettings() throws ParseException { - String def = TestUtil.joinLines( - "search test {", - " document test {", - " field s1 type string {", - " indexing: attribute | summary", - " }", - " field n1 type int {", - " indexing: summary | attribute", - " }", - " }", - "}"); - Schema schema = createSearch(def); - assertNull(schema.getAttribute("s1").getDictionary()); - assertNull(schema.getAttribute("n1").getDictionary()); - assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, - getConfig(schema).attribute().get(0).dictionary().type()); - assertEquals(AttributesConfig.Attribute.Dictionary.Type.BTREE, - getConfig(schema).attribute().get(1).dictionary().type()); - assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, - getConfig(schema).attribute().get(0).dictionary().match()); - assertEquals(AttributesConfig.Attribute.Dictionary.Match.UNCASED, - getConfig(schema).attribute().get(1).dictionary().match()); - } - - Schema verifyDictionaryControl(Dictionary.Type expected, String type, String ... cfg) throws ParseException - { - String def = TestUtil.joinLines( - "search test {", - " document test {", - " field n1 type " + type + " {", - " indexing: summary | attribute", - " attribute:fast-search", - TestUtil.joinLines(cfg), - " }", - " }", - "}"); - Schema schema = createSearch(def); - AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig = toCfg(expected); - assertEquals(expected, schema.getAttribute("n1").getDictionary().getType()); - assertEquals(expectedConfig, getConfig(schema).attribute().get(0).dictionary().type()); - return schema; - } - - AttributesConfig.Attribute.Dictionary.Type.Enum toCfg(Dictionary.Type v) { - return (v == Dictionary.Type.HASH) - ? AttributesConfig.Attribute.Dictionary.Type.Enum.HASH - : (v == Dictionary.Type.BTREE) - ? AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE - : AttributesConfig.Attribute.Dictionary.Type.Enum.BTREE_AND_HASH; - } - AttributesConfig.Attribute.Dictionary.Match.Enum toCfg(Case v) { - return (v == Case.CASED) - ? AttributesConfig.Attribute.Dictionary.Match.Enum.CASED - : AttributesConfig.Attribute.Dictionary.Match.Enum.UNCASED; - } - - void verifyStringDictionaryControl(Dictionary.Type expectedType, Case expectedCase, Case matchCasing, - String ... cfg) throws ParseException - { - Schema schema = verifyDictionaryControl(expectedType, "string", cfg); - ImmutableSDField f = schema.getField("n1"); - AttributesConfig.Attribute.Dictionary.Match.Enum expectedCaseCfg = toCfg(expectedCase); - assertEquals(matchCasing, f.getMatching().getCase()); - assertEquals(expectedCase, schema.getAttribute("n1").getDictionary().getMatch()); - assertEquals(expectedCaseCfg, getConfig(schema).attribute().get(0).dictionary().match()); - } - - @Test - public void testCasedBtreeSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:cased"); - } - - @Test - public void testNumericBtreeSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE, "int", "dictionary:btree"); - } - @Test - public void testNumericHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.HASH, "int", "dictionary:hash"); - } - @Test - public void testNumericBtreeAndHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "int", "dictionary:btree", "dictionary:hash"); - } - @Test - public void testNumericArrayBtreeAndHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "array", "dictionary:btree", "dictionary:hash"); - } - @Test - public void testNumericWSetBtreeAndHashSettings() throws ParseException { - verifyDictionaryControl(Dictionary.Type.BTREE_AND_HASH, "weightedset", "dictionary:btree", "dictionary:hash"); - } - @Test - public void testStringBtreeSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary:btree"); - } - @Test - public void testStringBtreeUnCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.UNCASED, Case.UNCASED, "dictionary { btree\nuncased\n}"); - } - @Test - public void testStringBtreeCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE, Case.CASED, Case.CASED, "dictionary { btree\ncased\n}", "match:cased"); - } - @Test - public void testStringHashSettings() throws ParseException { - try { - verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary:hash"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'n1': hash dictionary require cased match", e.getMessage()); - } - } - @Test - public void testStringHashUnCasedSettings() throws ParseException { - try { - verifyStringDictionaryControl(Dictionary.Type.HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nuncased\n}"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'n1': hash dictionary require cased match", e.getMessage()); - } - } - @Test - public void testStringHashBothCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}", "match:cased"); - } - @Test - public void testStringHashCasedSettings() throws ParseException { - try { - verifyStringDictionaryControl(Dictionary.Type.HASH, Case.CASED, Case.CASED, "dictionary { hash\ncased\n}"); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); - } - } - @Test - public void testStringBtreeHashSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary{hash\nbtree\n}"); - } - @Test - public void testStringBtreeHashUnCasedSettings() throws ParseException { - verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.UNCASED, Case.UNCASED, "dictionary { hash\nbtree\nuncased\n}"); - } - @Test - public void testStringBtreeHashCasedSettings() throws ParseException { - try { - verifyStringDictionaryControl(Dictionary.Type.BTREE_AND_HASH, Case.CASED, Case.CASED, "dictionary { btree\nhash\ncased\n}"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'n1': Dictionary casing 'CASED' does not match field match casing 'UNCASED'", e.getMessage()); - } - } - @Test - public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { - String def = TestUtil.joinLines( - "schema test {", - " document test {", - " field n1 type bool {", - " indexing: summary | attribute", - " dictionary:btree", - " }", - " }", - "}"); - try { - ApplicationBuilder sb = ApplicationBuilder.createFromString(def); - fail("Controlling dictionary for non-numeric fields are not yet supported."); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'n1': You can only specify 'dictionary:' for numeric or string fields", e.getMessage()); - } - } - @Test - public void testNonFastSearchNumericFieldsFailsDictionaryControl() throws ParseException { - String def = TestUtil.joinLines( - "schema test {", - " document test {", - " field n1 type int {", - " indexing: summary | attribute", - " dictionary:btree", - " }", - " }", - "}"); - try { - ApplicationBuilder sb = ApplicationBuilder.createFromString(def); - fail("Controlling dictionary for non-fast-search fields are not allowed."); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); - } - } - - @Test - public void testCasingForNonFastSearch() throws ParseException { - String def = TestUtil.joinLines( - "schema test {", - " document test {", - " field s1 type string {", - " indexing: attribute | summary", - " }", - " field s2 type string {", - " indexing: attribute | summary", - " match:uncased", - " }", - " field s3 type string {", - " indexing: attribute | summary", - " match:cased", - " }", - " }", - "}"); - Schema schema = createSearch(def); - assertEquals(Case.UNCASED, schema.getAttribute("s1").getCase()); - assertEquals(Case.UNCASED, schema.getAttribute("s2").getCase()); - assertEquals(Case.CASED, schema.getAttribute("s3").getCase()); - assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(schema).attribute().get(0).match()); - assertEquals(AttributesConfig.Attribute.Match.UNCASED, getConfig(schema).attribute().get(1).match()); - assertEquals(AttributesConfig.Attribute.Match.CASED, getConfig(schema).attribute().get(2).match()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java deleted file mode 100644 index b03aff455c5..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -/** - * @author lesters - */ -public class DisallowComplexMapAndWsetKeyTypesTestCase { - - @Test(expected = IllegalArgumentException.class) - public void requireThatComplexTypesForMapKeysFail() throws ParseException { - testFieldType("map"); - } - - @Test(expected = IllegalArgumentException.class) - public void requireThatComplexTypesForWsetFail() throws ParseException { - testFieldType("weightedset"); - } - - @Test(expected = IllegalArgumentException.class) - public void requireThatNestedComplexTypesForMapFail() throws ParseException { - testFieldType("array>"); - } - - @Test - public void requireThatNestedComplexValuesForMapSucceed() throws ParseException { - testFieldType("array>"); - } - - @Test(expected = IllegalArgumentException.class) - public void requireThatNestedComplexTypesForWsetFail() throws ParseException { - testFieldType("array>"); - } - - @Test(expected = IllegalArgumentException.class) - public void requireThatDeepNestedComplexTypesForMapFail() throws ParseException { - testFieldType("map>"); - } - - private void testFieldType(String fieldType) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " struct mystruct {}\n" + - " field a type " + fieldType + " {}\n" + - " }\n" + - "}\n"); - builder.build(true); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java deleted file mode 100644 index 0c25cef49a1..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.test.TestUtil; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -/** - * @author bjorncs - */ -public class FastAccessValidatorTest { - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void throws_exception_on_incompatible_use_of_fastaccess() throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); - builder.addSchema( - TestUtil.joinLines( - "schema parent {", - " document parent {", - " field int_field type int { indexing: attribute }", - " }", - "}")); - builder.addSchema( - TestUtil.joinLines( - "schema test {", - " document test { ", - " field int_attribute type int { ", - " indexing: attribute ", - " attribute: fast-access", - " }", - " field predicate_attribute type predicate {", - " indexing: attribute ", - " attribute: fast-access", - " }", - " field tensor_attribute type tensor(x[5]) {", - " indexing: attribute ", - " attribute: fast-access", - " }", - " field reference_attribute type reference {", - " indexing: attribute ", - " attribute: fast-access", - " }", - " }", - "}")); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "For schema 'test': The following attributes have a type that is incompatible " + - "with fast-access: predicate_attribute, tensor_attribute, reference_attribute. " + - "Predicate, tensor and reference attributes are incompatible with fast-access."); - builder.build(true); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java deleted file mode 100644 index 833a6effe4a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { - - @Test - public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/extrafield.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertEquals(2, docType.getFieldCount()); - } - - @Test - public void testRequireThatSummaryFieldsAreIncluded() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/summaryfield.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertNotNull(docType.getField("cox")); - assertNotNull(docType.getField("mytags")); - assertNotNull(docType.getField("alltags")); - assertEquals(5, docType.getFieldCount()); - } - - @Test - public void testRequireThatBoldedSummaryFieldsAreIncluded() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/boldedsummaryfields.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertNotNull(docType.getField("baz")); - assertNotNull(docType.getField("cox")); - assertEquals(4, docType.getFieldCount()); - } - - @Test - public void testRequireThatUntransformedSummaryFieldsAreIgnored() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/untransformedsummaryfields.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertNotNull(docType.getField("baz")); - assertEquals(3, docType.getFieldCount()); - } - - @Test - public void testRequireThatDynamicSummaryFieldsAreIgnored() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/dynamicsummaryfields.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertEquals(2, docType.getFieldCount()); - } - - @Test - public void testRequireThatDerivedConfigurationWorks() throws IOException, ParseException { - ApplicationBuilder sb = new ApplicationBuilder(); - sb.addSchemaFile("src/test/examples/nextgen/simple.sd"); - sb.build(true); - assertNotNull(sb.getSchema()); - new DerivedConfiguration(sb.getSchema(), sb.getRankProfileRegistry()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java deleted file mode 100644 index 443eff07f41..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.document.*; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.*; -public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { - @Test - public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - assertStruct(docType, PositionDataType.INSTANCE); - } - @Test - public void testRequireThatImplicitStructsAreUsed() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/nextgen/implicitstructtypes.sd"); - assertNotNull(schema); - - SDDocumentType docType = schema.getDocument(); - assertNotNull(docType); - - assertField(docType, "doc_str", DataType.STRING); - assertField(docType, "doc_str_sum", DataType.STRING); - assertField(docType, "doc_uri", DataType.URI); - assertField(docType, "docsum_str", DataType.STRING); - } - - @SuppressWarnings({ "UnusedDeclaration" }) - private static void assertStruct(SDDocumentType docType, StructDataType expectedStruct) { - // TODO: When structs are refactored from a static register to a member of the owning document types, this test - // TODO: must be changed to retrieve struct type from the provided document type. - StructDataType structType = (StructDataType) docType.getType(expectedStruct.getName()).getStruct(); - assertNotNull(structType); - for (Field expectedField : expectedStruct.getFields()) { - Field field = structType.getField(expectedField.getName()); - assertNotNull(field); - assertEquals(expectedField.getDataType(), field.getDataType()); - } - assertEquals(expectedStruct.getFieldCount(), structType.getFieldCount()); - } - - private static void assertField(SDDocumentType docType, String fieldName, DataType type) { - Field field = getSecretField(docType, fieldName); // TODO: get rid of this stupidity - assertNotNull(field); - assertEquals(type, field.getDataType()); - assertTrue(field instanceof SDField); - } - - private static Field getSecretField(SDDocumentType docType, String fieldName) { - for (Field field : docType.fieldSet()) { - if (field.getName().equals(fieldName)) { - return field; - } - } - return null; - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java deleted file mode 100644 index cc2bc7d7bf6..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * @author Simon Thoresen Hult - */ -public class ImplicitSummariesTestCase { - - @Test - public void requireThatSummaryFromAttributeDoesNotWarn() throws IOException, ParseException { - LogHandler log = new LogHandler(); - Logger.getLogger("").addHandler(log); - - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaries_attribute.sd"); - assertNotNull(schema); - assertTrue(log.records.isEmpty()); - } - - private static class LogHandler extends Handler { - - final List records = new ArrayList<>(); - - @Override - public void publish(LogRecord record) { - if (record.getLevel() == Level.WARNING || - record.getLevel() == Level.SEVERE) - { - records.add(record); - } - } - - @Override - public void flush() { - - } - - @Override - public void close() throws SecurityException { - - } - } - - @Test - public void attribute_combiner_transform_is_set_on_array_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/array_of_struct_attribute/test.sd"); - assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array").getTransform()); - } - - @Test - public void attribute_combiner_transform_is_set_on_map_of_struct_with_only_struct_field_attributes() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); - assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("str_elem_map").getTransform()); - } - - @Test - public void attribute_combiner_transform_is_not_set_when_map_of_struct_has_some_struct_field_attributes() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/derived/map_of_struct_attribute/test.sd"); - assertEquals(SummaryTransform.NONE, schema.getSummaryField("int_elem_map").getTransform()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java deleted file mode 100644 index 175b8d6fe1e..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class ImplicitSummaryFieldsTestCase extends AbstractSchemaTestCase { - - @Test - public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/implicitsummaryfields.sd"); - assertNotNull(schema); - - DocumentSummary docsum = schema.getSummary("default"); - assertNotNull(docsum); - assertNotNull(docsum.getSummaryField("rankfeatures")); - assertNotNull(docsum.getSummaryField("summaryfeatures")); - assertEquals(2, docsum.getSummaryFields().size()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java deleted file mode 100644 index d1d3f4489ce..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.document.DataType; -import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableImportedSDField; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; -import com.yahoo.tensor.TensorType; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertSame; - -/** - * @author geirst - */ -public class ImportedFieldsResolverTestCase { - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - private void resolve_imported_field(String fieldName, String targetFieldName) { - SearchModel model = new SearchModel(); - model.addImportedField(fieldName, "ref", targetFieldName).resolve(); - - assertEquals(1, model.importedFields.fields().size()); - ImportedField myField = model.importedFields.fields().get(fieldName); - assertNotNull(myField); - assertEquals(fieldName, myField.fieldName()); - assertSame(model.childSchema.getConcreteField("ref"), myField.reference().referenceField()); - assertSame(model.parentSchema, myField.reference().targetSearch()); - ImmutableSDField targetField = model.parentSchema.getField(targetFieldName); - if (targetField instanceof SDField) { - assertSame(targetField, myField.targetField()); - } else { - assertSame(getImportedField(targetField), getImportedField(myField.targetField())); - } - } - - private static ImportedField getImportedField(ImmutableSDField field) { - return ((ImmutableImportedSDField) field).getImportedField(); - } - - @Test - public void valid_imported_fields_are_resolved() { - resolve_imported_field("my_attribute_field", "attribute_field"); - resolve_imported_field("my_tensor_field", "tensor_field"); - resolve_imported_field("my_ancient_field", "ancient_field"); - } - - @Test - public void resolver_fails_if_document_reference_is_not_found() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', import field 'my_attribute_field': " - + "Reference field 'not_ref' not found"); - new SearchModel().addImportedField("my_attribute_field", "not_ref", "budget").resolve(); - } - - @Test - public void resolver_fails_if_referenced_field_is_not_found() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', import field 'my_attribute_field': " - + "Field 'not_existing' via reference field 'ref': Not found"); - new SearchModel().addImportedField("my_attribute_field", "ref", "not_existing").resolve(); - } - - @Test - public void resolver_fails_if_imported_field_is_not_an_attribute() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'child', import field 'my_not_attribute': " - + "Field 'not_attribute' via reference field 'ref': Is not an attribute field. Only attribute fields supported"); - new SearchModel().addImportedField("my_not_attribute", "ref", "not_attribute").resolve(); - } - - @Test - public void resolver_fails_if_imported_field_is_indexing() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "For schema 'child', import field 'my_attribute_and_index': " + - "Field 'attribute_and_index' via reference field 'ref': Is an index field. Not supported"); - new SearchModel() - .addImportedField("my_attribute_and_index", "ref", "attribute_and_index") - .resolve(); - } - - @Test - public void resolver_fails_if_imported_field_is_of_type_predicate() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "For schema 'child', import field 'my_predicate_field': " + - "Field 'predicate_field' via reference field 'ref': Is of type 'predicate'. Not supported"); - new SearchModel().addImportedField("my_predicate_field", "ref", "predicate_field").resolve(); - } - - static class SearchModel extends ParentChildSearchModel { - - public final Schema grandParentSchema; - public ImportedFields importedFields; - - public SearchModel() { - super(); - grandParentSchema = createSearch("grandparent"); - var grandParentDoc = grandParentSchema.getDocument(); - grandParentDoc.addField(createField(grandParentDoc, "ancient_field", DataType.INT, "{ attribute }")); - var parentDoc = parentSchema.getDocument(); - parentDoc.addField(createField(parentDoc, "attribute_field", DataType.INT, "{ attribute }")); - parentDoc.addField(createField(parentDoc, "attribute_and_index", DataType.INT, "{ attribute | index }")); - parentDoc.addField(new TemporarySDField(parentDoc, "not_attribute", DataType.INT)); - parentDoc.addField(createField(parentDoc, "tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); - parentDoc.addField(createField(parentDoc, "predicate_field", DataType.PREDICATE, "{ attribute }")); - addRefField(parentSchema, grandParentSchema, "ref"); - addImportedField(parentSchema, "ancient_field", "ref", "ancient_field"); - - addRefField(childSchema, parentSchema, "ref"); - } - - - protected SearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) { - return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName); - } - - protected SearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) { - super.addImportedField(schema, fieldName, referenceFieldName, targetFieldName); - return this; - } - - public void resolve() { - resolve(grandParentSchema); - resolve(parentSchema); - importedFields = resolve(childSchema); - } - - private static ImportedFields resolve(Schema schema) { - assertNotNull(schema.temporaryImportedFields().get()); - assertFalse(schema.importedFields().isPresent()); - new ImportedFieldsResolver(schema, null, null, null).process(true, false); - assertNotNull(schema.importedFields().get()); - return schema.importedFields().get(); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java deleted file mode 100644 index b15b81b717d..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ /dev/null @@ -1,529 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.ImportedComplexField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.assertEquals; -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * @author geirst - */ -public class ImportedFieldsTestCase { - - @Test - public void fields_can_be_imported_from_referenced_document_types() throws ParseException { - Schema schema = buildAdSearch(joinLines( - "search ad {", - " document ad {", - " field campaign_ref type reference { indexing: attribute }", - " field person_ref type reference { indexing: attribute }", - " }", - " import field campaign_ref.budget as my_budget {}", - " import field person_ref.name as my_name {}", - "}")); - assertEquals(2, schema.importedFields().get().fields().size()); - assertSearchContainsImportedField("my_budget", "campaign_ref", "campaign", "budget", schema); - assertSearchContainsImportedField("my_name", "person_ref", "person", "name", schema); - } - - @SuppressWarnings("deprecation") - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void field_reference_spec_must_include_dot() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Illegal field reference spec 'campaignrefbudget': Does not include a single '.'"); - buildAdSearch(joinLines( - "search ad {", - " document ad {}", - " import field campaignrefbudget as budget {}", - "}")); - } - - @Test - public void fail_duplicate_import() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'ad', import field as 'my_budget': Field already imported"); - Schema schema = buildAdSearch(joinLines( - "schema ad {", - " document ad {", - " field campaign_ref type reference { indexing: attribute }", - " }", - " import field campaign_ref.budget as my_budget {}", - " import field campaign_ref.budget as my_budget {}", - "}")); - } - - private static Schema buildAdSearch(String sdContent) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "schema campaign {", - " document campaign {", - " field budget type int { indexing: attribute }", - " }", - "}")); - builder.addSchema(joinLines( - "schema person {", - " document person {", - " field name type string { indexing: attribute }", - " }", - "}")); - builder.addSchema(sdContent); - builder.build(true); - return builder.getSchema("ad"); - } - - private static void checkStructImport(AncestorStructSdBuilder parentBuilder) throws ParseException { - Schema schema = buildChildSearch(parentBuilder.build(), new ChildStructSdBuilder().build()); - checkImportedStructFields(schema, parentBuilder); - } - - private static void checkNestedStructImport(AncestorStructSdBuilder grandParentBuilder) throws ParseException { - Schema schema = buildChildSearch(grandParentBuilder.build(), - new IntermediateParentStructSdBuilder().build(), - new ChildStructSdBuilder().build()); - checkImportedStructFields(schema, grandParentBuilder); - } - - private static void checkImportedStructFields(Schema schema, AncestorStructSdBuilder ancestorBuilder) { - assertEquals(3, schema.importedFields().get().fields().size()); - checkImportedField("my_elem_array.name", "parent_ref", "parent", "elem_array.name", schema, ancestorBuilder.elem_array_name_attr); - checkImportedField("my_elem_array.weight", "parent_ref", "parent", "elem_array.weight", schema, ancestorBuilder.elem_array_weight_attr); - checkImportedField("my_elem_map.key", "parent_ref", "parent", "elem_map.key", schema, ancestorBuilder.elem_map_key_attr); - checkImportedField("my_elem_map.value.name", "parent_ref", "parent", "elem_map.value.name", schema, ancestorBuilder.elem_map_value_name_attr); - checkImportedField("my_elem_map.value.weight", "parent_ref", "parent", "elem_map.value.weight", schema, ancestorBuilder.elem_map_value_weight_attr); - checkImportedField("my_str_int_map.key", "parent_ref", "parent", "str_int_map.key", schema, ancestorBuilder.str_int_map_key_attr); - checkImportedField("my_str_int_map.value", "parent_ref", "parent", "str_int_map.value", schema, ancestorBuilder.str_int_map_value_attr); - checkImportedField("my_elem_array", "parent_ref", "parent", "elem_array", schema, true); - checkImportedField("my_elem_map", "parent_ref", "parent", "elem_map", schema, true); - checkImportedField("my_str_int_map", "parent_ref", "parent", "str_int_map", schema, true); - } - - @Test - public void check_struct_import() throws ParseException { - checkStructImport(new ParentStructSdBuilder()); - checkStructImport(new ParentStructSdBuilder().elem_array_weight_attr(false).elem_map_value_weight_attr(false)); - checkStructImport(new ParentStructSdBuilder().elem_array_name_attr(false).elem_map_value_name_attr(false)); - } - - @Test - public void check_nested_struct_import() throws ParseException { - checkNestedStructImport(new GrandParentStructSdBuilder()); - checkNestedStructImport(new GrandParentStructSdBuilder().elem_array_weight_attr(false).elem_map_value_weight_attr(false)); - checkNestedStructImport(new GrandParentStructSdBuilder().elem_array_name_attr(false).elem_map_value_name_attr(false)); - } - - @Test - public void check_illegal_struct_import_missing_array_of_struct_attributes() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'child', import field 'my_elem_array': Field 'elem_array' via reference field 'parent_ref': Is not a struct containing an attribute field."); - checkStructImport(new ParentStructSdBuilder().elem_array_name_attr(false).elem_array_weight_attr(false)); - } - - @Test - public void check_illegal_struct_import_missing_map_of_struct_key_attribute() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'child', import field 'my_elem_map' (nested to 'my_elem_map.key'): Field 'elem_map.key' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); - checkStructImport(new ParentStructSdBuilder().elem_map_key_attr(false)); - } - - @Test - public void check_illegal_struct_import_missing_map_of_struct_value_attributes() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'child', import field 'my_elem_map' (nested to 'my_elem_map.value'): Field 'elem_map.value' via reference field 'parent_ref': Is not a struct containing an attribute field."); - checkStructImport(new ParentStructSdBuilder().elem_map_value_name_attr(false).elem_map_value_weight_attr(false)); - } - - @Test - public void check_illegal_struct_import_missing_map_of_primitive_key_attribute() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'child', import field 'my_str_int_map' (nested to 'my_str_int_map.key'): Field 'str_int_map.key' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); - checkStructImport(new ParentStructSdBuilder().str_int_map_key_attr(false)); - } - - @Test - public void check_illegal_struct_import_missing_map_of_primitive_value_attribute() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'child', import field 'my_str_int_map' (nested to 'my_str_int_map.value'): Field 'str_int_map.value' via reference field 'parent_ref': Is not an attribute field. Only attribute fields supported"); - checkStructImport(new ParentStructSdBuilder().str_int_map_value_attr(false)); - } - - private static class NamedSdBuilder { - protected String name; - private String fieldPrefix; - - public NamedSdBuilder(String name, String fieldPrefix) { - this.name = name; - this.fieldPrefix = fieldPrefix; - } - - protected String prefixedFieldName(String name) { - return fieldPrefix + name; - } - } - - private static class AncestorStructSdBuilder extends NamedSdBuilder { - private boolean elem_array_name_attr; - private boolean elem_array_weight_attr; - private boolean elem_map_key_attr; - private boolean elem_map_value_name_attr; - private boolean elem_map_value_weight_attr; - private boolean str_int_map_key_attr; - private boolean str_int_map_value_attr; - - public AncestorStructSdBuilder(String name, String fieldPrefix) { - super(name, fieldPrefix); - elem_array_name_attr = true; - elem_array_weight_attr = true; - elem_map_key_attr = true; - elem_map_value_name_attr = true; - elem_map_value_weight_attr = true; - str_int_map_key_attr = true; - str_int_map_value_attr = true; - } - - public AncestorStructSdBuilder elem_array_name_attr(boolean v) { elem_array_name_attr = v; return this; } - public AncestorStructSdBuilder elem_array_weight_attr(boolean v) { elem_array_weight_attr = v; return this; } - public AncestorStructSdBuilder elem_map_key_attr(boolean v) { elem_map_key_attr = v; return this; } - public AncestorStructSdBuilder elem_map_value_name_attr(boolean v) { elem_map_value_name_attr = v; return this; } - public AncestorStructSdBuilder elem_map_value_weight_attr(boolean v) { elem_map_value_weight_attr = v; return this; } - public AncestorStructSdBuilder str_int_map_key_attr(boolean v) { str_int_map_key_attr = v; return this; } - public AncestorStructSdBuilder str_int_map_value_attr(boolean v) { str_int_map_value_attr = v; return this; } - - public String build() { - return joinLines("search " + name + " {", - " document " + name + " {", - " struct elem {", - " field name type string {}", - " field weight type int {}", - " }", - " field " + prefixedFieldName("elem_array") + " type array {", - " indexing: summary", - " struct-field name {", - structFieldSpec(elem_array_name_attr), - " }", - " struct-field weight {", - structFieldSpec(elem_array_weight_attr), - " }", - " }", - " field " + prefixedFieldName("elem_map") + " type map {", - " indexing: summary", - " struct-field key {", - structFieldSpec(elem_map_key_attr), - " }", - " struct-field value.name {", - structFieldSpec(elem_map_value_name_attr), - " }", - " struct-field value.weight {", - structFieldSpec(elem_map_value_weight_attr), - " }", - " }", - " field " + prefixedFieldName("str_int_map") + " type map {", - " indexing: summary", - " struct-field key {", - structFieldSpec(str_int_map_key_attr), - " }", - " struct-field value {", - structFieldSpec(str_int_map_value_attr), - " }", - " }", - " }", - "}"); - } - - private static String structFieldSpec(boolean isAttribute) { - return isAttribute ? " indexing: attribute" : ""; - } - } - - private static class ParentStructSdBuilder extends AncestorStructSdBuilder { - ParentStructSdBuilder() { - super("parent", ""); - } - } - - private static class GrandParentStructSdBuilder extends AncestorStructSdBuilder { - GrandParentStructSdBuilder() { - super("grandparent", "gp_"); - } - } - - private static class DescendantSdBuilder extends NamedSdBuilder { - protected String parentName; - private String parentFieldPrefix; - - public DescendantSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { - super(name, fieldPrefix); - this.parentName = parentName; - this.parentFieldPrefix = parentFieldPrefix; - } - - protected String parentRef() { - return parentName + "_ref"; - } - - protected String importParentField(String fieldName) { - return " import field " + parentRef() + "." + parentFieldPrefix + fieldName + " as " + prefixedFieldName(fieldName) + " {}"; - } - } - - private static class DescendantStructSdBuilder extends DescendantSdBuilder { - public DescendantStructSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { - super(name, fieldPrefix, parentName, parentFieldPrefix); - } - - public String build() { - return joinLines("search " + name + " {", - " document " + name + " {", - " field " + parentRef() + " type reference<" + parentName + "> {", - " indexing: attribute | summary", - " }", - " }", - importParentField("elem_array"), - importParentField("elem_map"), - importParentField("str_int_map"), - "}"); - } - } - - private static class ChildStructSdBuilder extends DescendantStructSdBuilder { - public ChildStructSdBuilder() { - super("child", "my_", "parent", ""); - } - } - - private static class IntermediateParentStructSdBuilder extends DescendantStructSdBuilder { - public IntermediateParentStructSdBuilder() { - super("parent", "", "grandparent", "gp_"); - } - } - - private static Schema buildChildSearch(String parentSdContent, String sdContent) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(parentSdContent); - builder.addSchema(sdContent); - builder.build(true); - return builder.getSchema("child"); - } - - private static Schema buildChildSearch(String grandParentSdContent, String parentSdContent, String sdContent) throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(grandParentSdContent); - builder.addSchema(parentSdContent); - builder.addSchema(sdContent); - builder.build(true); - return builder.getSchema("child"); - } - - private static class AncestorPosSdBuilder extends NamedSdBuilder { - public AncestorPosSdBuilder(String name, String fieldPrefix) { - super(name, fieldPrefix); - } - - public String build() { - return joinLines("search " + name + " {", - " document " + name + " {", - "field " + prefixedFieldName("pos") + " type position {", - "indexing: attribute | summary", - " }", - " }", - "}"); - } - } - - private static class ParentPosSdBuilder extends AncestorPosSdBuilder { - public ParentPosSdBuilder() { super("parent", ""); } - } - - private static class GrandParentPosSdBuilder extends AncestorPosSdBuilder { - public GrandParentPosSdBuilder() { super("grandparent", "gp_"); } - } - - private static class DescendantPosSdBuilder extends DescendantSdBuilder { - private boolean import_pos_zcurve_before; - - public DescendantPosSdBuilder(String name, String fieldPrefix, String parentName, String parentFieldPrefix) { - super(name, fieldPrefix, parentName, parentFieldPrefix); - import_pos_zcurve_before = false; - } - - DescendantPosSdBuilder import_pos_zcurve_before(boolean v) { import_pos_zcurve_before = v; return this; } - - public String build() { - return joinLines("search " + name + " {", - " document " + name + " {", - " field " + parentRef() + " type reference<" + parentName + "> {", - " indexing: attribute | summary", - " }", - " }", - importPosZCurve(import_pos_zcurve_before), - importParentField("pos"), - "}"); - } - - private static String importPosZCurve(boolean doImport) { - return doImport ? "import field parent_ref.pos_zcurve as my_pos_zcurve {}" : ""; - } - } - - private static class ChildPosSdBuilder extends DescendantPosSdBuilder { - public ChildPosSdBuilder() { - super("child", "my_", "parent", ""); - } - } - - private static class IntermediateParentPosSdBuilder extends DescendantPosSdBuilder { - public IntermediateParentPosSdBuilder() { - super("parent", "", "grandparent", "gp_"); - } - } - - private static void checkPosImport(ParentPosSdBuilder parentBuilder, DescendantPosSdBuilder childBuilder) throws ParseException { - Schema schema = buildChildSearch(parentBuilder.build(), childBuilder.build()); - checkImportedPosFields(schema); - } - - private static void checkNestedPosImport(GrandParentPosSdBuilder grandParentBuilder, DescendantPosSdBuilder childBuilder) throws ParseException { - Schema schema = buildChildSearch(grandParentBuilder.build(), new IntermediateParentPosSdBuilder().build(), childBuilder.build()); - checkImportedPosFields(schema); - } - - private static void checkImportedPosFields(Schema schema) { - assertEquals(2, schema.importedFields().get().fields().size()); - assertSearchContainsImportedField("my_pos_zcurve", "parent_ref", "parent", "pos_zcurve", schema); - assertSearchContainsImportedField("my_pos", "parent_ref", "parent", "pos", schema); - } - - @Test - public void check_pos_import() throws ParseException { - checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder()); - } - - @Test - public void check_nested_pos_import() throws ParseException { - checkNestedPosImport(new GrandParentPosSdBuilder(), new ChildPosSdBuilder()); - } - - @Test - public void check_pos_import_after_pos_zcurve_import() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For schema 'child', import field 'my_pos_zcurve': Field 'pos_zcurve' via reference field 'parent_ref': Field already imported"); - checkPosImport(new ParentPosSdBuilder(), new ChildPosSdBuilder().import_pos_zcurve_before(true)); - } - - private static ImportedField getImportedField(String name, Schema schema) { - if (name.contains(".")) { - assertNull(schema.importedFields().get().fields().get(name)); - String superFieldName = name.substring(0,name.indexOf(".")); - String subFieldName = name.substring(name.indexOf(".")+1); - ImportedField superField = schema.importedFields().get().fields().get(superFieldName); - if (superField != null && superField instanceof ImportedComplexField) { - return ((ImportedComplexField)superField).getNestedField(subFieldName); - } - return null; - } - return schema.importedFields().get().fields().get(name); - } - - private static void assertSearchNotContainsImportedField(String fieldName, Schema schema) { - ImportedField importedField = getImportedField(fieldName, schema); - assertNull(importedField); - } - - private static void assertSearchContainsImportedField(String fieldName, - String referenceFieldName, - String referenceDocType, - String targetFieldName, - Schema schema) { - ImportedField importedField = getImportedField(fieldName, schema); - assertNotNull(importedField); - assertEquals(fieldName, importedField.fieldName()); - assertEquals(referenceFieldName, importedField.reference().referenceField().getName()); - assertEquals(referenceDocType, importedField.reference().targetSearch().getName()); - assertEquals(targetFieldName, importedField.targetField().getName()); - } - - private static void checkImportedField(String fieldName, String referenceFieldName, String referenceDocType, - String targetFieldName, Schema schema, boolean present) { - if (present) { - assertSearchContainsImportedField(fieldName, referenceFieldName, referenceDocType, targetFieldName, schema); - } else { - assertSearchNotContainsImportedField(fieldName, schema); - } - } - - @Test - public void field_with_struct_field_attributes_can_be_imported_from_parents_that_use_inheritance() throws ParseException { - var builder = buildParentsUsingInheritance(); - - assertParentContainsEntriesAttributes(builder.getSchema("parent_a")); - assertParentContainsEntriesAttributes(builder.getSchema("parent_b")); - - var child = builder.getSchema("child"); - checkImportedField("entries_from_a", "ref_parent_a", "parent_a", "entries", child, true); - checkImportedField("entries_from_a.key", "ref_parent_a", "parent_a", "entries.key", child, true); - checkImportedField("entries_from_a.value", "ref_parent_a", "parent_a", "entries.value", child, true); - - checkImportedField("entries_from_b", "ref_parent_b", "parent_b", "entries", child, true); - checkImportedField("entries_from_b.key", "ref_parent_b", "parent_b", "entries.key", child, true); - checkImportedField("entries_from_b.value", "ref_parent_b", "parent_b", "entries.value", child, true); - } - - private void assertParentContainsEntriesAttributes(Schema parent) { - var attrs = new AttributeFields(parent); - assertTrue(attrs.containsAttribute("entries.key")); - assertTrue(attrs.containsAttribute("entries.value")); - } - - private ApplicationBuilder buildParentsUsingInheritance() throws ParseException { - var builder = new ApplicationBuilder(); - builder.addSchema(joinLines("schema parent_a {", - "document parent_a {", - " struct Entry {", - " field key type string {}", - " field value type string {}", - " }", - " field entries type array {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - " }", - "}", - "}")); - - builder.addSchema(joinLines("schema parent_b {", - "document parent_b inherits parent_a {", - "}", - "}")); - - builder.addSchema(joinLines("schema child {", - "document child {", - " field ref_parent_a type reference {", - " indexing: attribute", - " }", - " field ref_parent_b type reference {", - " indexing: attribute", - " }", - "}", - "import field ref_parent_a.entries as entries_from_a {}", - "import field ref_parent_b.entries as entries_from_b {}", - "}")); - - builder.build(true); - return builder; - } - - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java deleted file mode 100644 index 725b1b17ff0..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; - -/** - * @author Simon Thoresen Hult - */ -public class IndexingInputsTestCase { - - @Test - public void requireThatExtraFieldInputExtraFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", - "For schema 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + - "to field 'bar' which does not exist in document type " + - "'indexing_extra_field_input_extra_field', and is not a mutable attribute."); - } - - @Test - public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", - "For schema 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + - "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit', and is not a mutable attribute."); - } - - @Test - public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", - "For schema 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + - "'foo' which does not exist in document type 'indexing_extra_field_input_null', and is not a mutable attribute."); - } - - @Test - public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", - "For schema 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + - "'foo' which does not exist in document type 'indexing_extra_field_input_self', and is not a mutable attribute."); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java deleted file mode 100644 index 5ea4b37991a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; - - -/** - * @author Simon Thoresen Hult - */ -public class IndexingOutputsTestCase { - - @Test - public void requireThatOutputOtherFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_output_other_field.sd", - "For schema 'indexing_output_other_field', field 'foo': Indexing expression 'index bar' " + - "attempts to write to a field other than 'foo'."); - } - - @Test - public void requireThatOutputConflictThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_output_conflict.sd", - "For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + - "to assign conflicting values to field 'bar'."); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java deleted file mode 100644 index 9e4ba1c6728..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.util.Arrays; -import java.util.OptionalDouble; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; - -import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; -import static org.junit.Assert.assertEquals; - -/** - * @author Simon Thoresen Hult - */ -public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { - - @Test - public void testSetLanguageRewriting() { - assertIndexingScript("{ input test | set_language; }", - createField("test", DataType.STRING, "{ set_language }")); - } - - @Test - public void testSummaryRewriting() { - assertIndexingScript("{ input test | summary test; }", - createField("test", DataType.STRING, "{ summary }")); - } - - @Test - public void testDynamicSummaryRewriting() { - SDField field = createField("test", DataType.STRING, "{ summary }"); - field.addSummaryField(createDynamicSummaryField(field, "dyn")); - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn; }", field); - } - - @Test - public void testSummaryRewritingWithIndexing() { - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", - createField("test", DataType.STRING, "{ summary | index }")); - } - - @Test - public void testDynamicAndStaticSummariesRewritingWithIndexing() { - SDField field = createField("test", DataType.STRING, "{ summary | index }"); - field.addSummaryField(createDynamicSummaryField(field, "dyn")); - field.addSummaryField(createStaticSummaryField(field, "test")); - field.addSummaryField(createStaticSummaryField(field, "other")); - field.addSummaryField(createDynamicSummaryField(field, "dyn2")); - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn | summary dyn2 | summary other | " + - "summary test | index test; }", field); - } - - @Test - public void testIntSummaryRewriting() { - assertIndexingScript("{ input test | summary test | attribute test; }", - createField("test", DataType.INT, "{ summary | index }")); - } - - @Test - public void testStringAttributeSummaryRewriting() { - assertIndexingScript("{ input test | summary test | attribute test; }", - createField("test", DataType.STRING, "{ summary | attribute }")); - } - - @Test - public void testMultiblockTokenize() { - SDField field = createField("test", DataType.STRING, - "{ input test | tokenize | { summary test; }; }"); - assertIndexingScript("{ input test | tokenize | { summary test; }; }", field); - } - - @Test - public void requireThatOutputDefaultsToCurrentField() { - assertIndexingScript("{ input test | attribute test; }", - createField("test", DataType.STRING, "{ attribute; }")); - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | index test; }", - createField("test", DataType.STRING, "{ index; }")); - assertIndexingScript("{ input test | summary test; }", - createField("test", DataType.STRING, "{ summary; }")); - } - - @Test - public void testTokenizeComparisonDisregardsConfig() { - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", - createField("test", DataType.STRING, "{ summary | tokenize | index; }")); - } - - @Test - public void testDerivingFromSimple() throws Exception { - assertIndexing(Arrays.asList("clear_state | guard { input access | attribute access; }", - "clear_state | guard { input category | split \";\" | attribute category_arr; }", - "clear_state | guard { input category | tokenize | index category; }", - "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", - "clear_state | guard { input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", - "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }", - "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | summary dyndesc | index description; }", - "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", - "clear_state | guard { input longdesc | tokenize normalize stem:\"BEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }", - "clear_state | guard { input measurement | attribute measurement | summary measurement; }", - "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", - "clear_state | guard { input popularity | attribute popularity; }", - "clear_state | guard { input popularity * input measurement | attribute popsiness; }", - "clear_state | guard { input smallattribute | attribute smallattribute; }", - "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", - "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), - ApplicationBuilder.buildFromFile("src/test/examples/simple.sd")); - } - - @Test - public void testIndexRewrite() throws Exception { - assertIndexing( - Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + - " tokenize | index title; }", - "clear_state | guard { input title_src | summary title_s; }"), - ApplicationBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); - } - - @Test - public void requireThatPredicateFieldsGetOptimization() { - assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | attribute test; }; }", - createPredicateField( - "test", DataType.PREDICATE, "{ attribute; }", 10, OptionalLong.empty(), OptionalLong.empty())); - assertIndexingScript("{ 10 | set_var arity | { input test | optimize_predicate | summary test | attribute test; }; }", - createPredicateField( - "test", DataType.PREDICATE, "{ summary | attribute ; }", 10, OptionalLong.empty(), OptionalLong.empty())); - assertIndexingScript( - "{ 2 | set_var arity | 0L | set_var lower_bound | 1023L | set_var upper_bound | " + - "{ input test | optimize_predicate | attribute test; }; }", - createPredicateField("test", DataType.PREDICATE, "{ attribute; }", 2, OptionalLong.of(0L), OptionalLong.of(1023L))); - } - - private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { - assertEquals(expectedScript, - processField(unprocessedField).toString()); - } - - private static ScriptExpression processField(SDField unprocessedField) { - SDDocumentType sdoc = new SDDocumentType("test"); - sdoc.addField(unprocessedField); - Schema schema = new Schema("test", MockApplicationPackage.createEmpty()); - schema.addDocument(sdoc); - new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), - new QueryProfiles(), true, false, Set.of()); - return unprocessedField.getIndexingScript(); - } - - private static SDField createField(String name, DataType type, String script) { - SDField field = new SDField(null, name, type); - field.parseIndexingScript(script); - return field; - } - - private static SDField createPredicateField( - String name, DataType type, String script, int arity, OptionalLong lower_bound, OptionalLong upper_bound) { - SDField field = new SDField(null, name, type); - field.parseIndexingScript(script); - Index index = new Index("foo"); - index.setBooleanIndexDefiniton(new BooleanIndexDefinition( - OptionalInt.of(arity), lower_bound, upper_bound, OptionalDouble.empty())); - field.addIndex(index); - return field; - } - - private static SummaryField createDynamicSummaryField(SDField field, String name) { - return createSummaryField(field, name, true); - } - - private static SummaryField createStaticSummaryField(SDField field, String name) { - return createSummaryField(field, name, false); - } - - private static SummaryField createSummaryField(SDField field, String name, boolean dynamic) { - SummaryField summaryField = new SummaryField(name, field.getDataType()); - if (dynamic) { - summaryField.setTransform(SummaryTransform.DYNAMICTEASER); - } - summaryField.addDestination("default"); - summaryField.addSource(field.getName()); - return summaryField; - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java deleted file mode 100644 index 74a8cdfdb6a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; - -import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; - -/** - * @author Simon Thoresen Hult - */ -public class IndexingValidationTestCase extends AbstractExportingTestCase { - - @Test - public void testAttributeChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_attribute_changed.sd", - "For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + - "Attempting to assign conflicting values to field 'foo'."); - } - - @Test - public void testAttributeOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_attribute_other.sd", - "For schema 'indexing_attribute_other', field 'foo': Indexing expression 'attribute bar' " + - "attempts to write to a field other than 'foo'."); - } - - @Test - public void testIndexChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_index_changed.sd", - "For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + - "Attempting to assign conflicting values to field 'foo'."); - } - - @Test - public void testIndexOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_index_other.sd", - "For schema 'indexing_index_other', field 'foo': Indexing expression 'index bar' " + - "attempts to write to a field other than 'foo'."); - } - - @Test - public void testSummaryChanged() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_summary_changed.sd", - "For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + - "to assign conflicting values to field 'foo'."); - } - - @Test - public void testSummaryOther() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_summary_other.sd", - "For schema 'indexing_summary_other', field 'foo': Indexing expression 'summary bar' " + - "attempts to write to a field other than 'foo'."); - } - - @Test - public void testExtraField() throws IOException, ParseException { - assertIndexing( - Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", - "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), - ApplicationBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); - } - - @Test - public void requireThatMultilineOutputConflictThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_multiline_output_conflict.sd", - "For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + - "Attempting to assign conflicting values to field 'cox'."); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java deleted file mode 100644 index 25d966b1324..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuilds; - -/** - * @author Simon Thoresen Hult - */ -public class IndexingValuesTestCase { - - @Test - public void requireThatModifyFieldNoOutputDoesNotThrow() throws IOException, ParseException { - assertBuilds("src/test/examples/indexing_modify_field_no_output.sd"); - } - - @Test - public void requireThatInputOtherFieldThrows() throws IOException, ParseException { - assertBuildFails("src/test/examples/indexing_input_other_field.sd", - "For schema 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + - "attempts to modify the value of the document field 'bar'. " + - "Use a field outside the document block instead."); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java deleted file mode 100644 index 2465839b72b..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author baldersheim - */ -public class IntegerIndex2AttributeTestCase extends AbstractSchemaTestCase { - - @Test - public void testIntegerIndex2Attribute() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/integerindex2attribute.sd"); - new IntegerIndex2Attribute(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); - - SDField f; - f = schema.getConcreteField("s1"); - assertTrue(f.getAttributes().isEmpty()); - assertTrue(f.existsIndex("s1")); - f = schema.getConcreteField("s2"); - assertEquals(f.getAttributes().size(), 1); - assertTrue(f.existsIndex("s2")); - - f = schema.getConcreteField("as1"); - assertTrue(f.getAttributes().isEmpty()); - assertTrue(f.existsIndex("as1")); - f = schema.getConcreteField("as2"); - assertEquals(f.getAttributes().size(), 1); - assertTrue(f.existsIndex("as2")); - - f = schema.getConcreteField("i1"); - assertEquals(f.getAttributes().size(), 1); - assertFalse(f.existsIndex("i1")); - - f = schema.getConcreteField("i2"); - assertEquals(f.getAttributes().size(), 1); - assertFalse(f.existsIndex("i2")); - - f = schema.getConcreteField("ai1"); - assertEquals(schema.getConcreteField("ai1").getAttributes().size(), 1); - assertFalse(schema.getConcreteField("ai1").existsIndex("ai1")); - f = schema.getConcreteField("ai2"); - assertEquals(f.getAttributes().size(), 1); - assertFalse(f.existsIndex("ai2")); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java deleted file mode 100644 index 05d816ed716..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import org.junit.Test; - -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; - -public class MatchPhaseSettingsValidatorTestCase { - - private static String getMessagePrefix() { - return "In search definition 'test', rank-profile 'default': match-phase attribute 'foo' "; - } - - @Test - public void requireThatAttributeMustExists() throws Exception { - assertBuildFails("src/test/examples/matchphase/non_existing_attribute.sd", - getMessagePrefix() + "does not exists"); - } - - @Test - public void requireThatAttributeMustBeNumeric() throws Exception { - assertBuildFails("src/test/examples/matchphase/wrong_data_type_attribute.sd", - getMessagePrefix() + "must be single value numeric, but it is 'string'"); - } - - @Test - public void requireThatAttributeMustBeSingleValue() throws Exception { - assertBuildFails("src/test/examples/matchphase/wrong_collection_type_attribute.sd", - getMessagePrefix() + "must be single value numeric, but it is 'Array'"); - } - - @Test - public void requireThatAttributeMustHaveFastSearch() throws Exception { - assertBuildFails("src/test/examples/matchphase/non_fast_search_attribute.sd", - getMessagePrefix() + "must be fast-search, but it is not"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java deleted file mode 100644 index b48927d58a3..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; - -/** - * @author geirst - */ -public class MatchedElementsOnlyResolverTestCase { - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void complex_field_with_some_struct_field_attributes_gets_default_transform() throws ParseException { - assertSummaryField(joinLines("field my_field type map {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field key { indexing: attribute }", - "}"), - "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); - - assertSummaryField(joinLines("field my_field type map {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field key { indexing: attribute }", - "}"), - "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); - - assertSummaryField(joinLines("field my_field type array {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field name { indexing: attribute }", - "}"), - "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER); - } - - @Test - public void complex_field_with_only_struct_field_attributes_gets_attribute_transform() throws ParseException { - assertSummaryField(joinLines("field my_field type map {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - "}"), - "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - - assertSummaryField(joinLines("field my_field type map {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field key { indexing: attribute }", - " struct-field value.name { indexing: attribute }", - " struct-field value.weight { indexing: attribute }", - "}"), - "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - - assertSummaryField(joinLines("field my_field type array {", - " indexing: summary", - " summary: matched-elements-only", - " struct-field name { indexing: attribute }", - " struct-field weight { indexing: attribute }", - "}"), - "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - } - - @Test - public void explicit_complex_summary_field_can_use_filter_transform_with_reference_to_source_field() throws ParseException { - String documentSummary = joinLines("document-summary my_summary {", - " summary my_filter_field type map {", - " source: my_field", - " matched-elements-only", - " }", - "}"); - { - var search = buildSearch(joinLines("field my_field type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - "}"), - documentSummary); - assertSummaryField(search.getSummaryField("my_filter_field"), - SummaryTransform.MATCHED_ELEMENTS_FILTER, "my_field"); - assertSummaryField(search.getSummaryField("my_field"), - SummaryTransform.NONE, "my_field"); - } - { - var search = buildSearch(joinLines("field my_field type map {", - " indexing: summary", - " struct-field key { indexing: attribute }", - " struct-field value { indexing: attribute }", - "}"), - documentSummary); - assertSummaryField(search.getSummaryField("my_filter_field"), - SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field"); - assertSummaryField(search.getSummaryField("my_field"), - SummaryTransform.ATTRIBUTECOMBINER, "my_field"); - } - } - - @Test - public void primitive_array_attribute_field_gets_attribute_transform() throws ParseException { - assertSummaryField(joinLines("field my_field type array {", - " indexing: attribute | summary", - " summary: matched-elements-only", - "}"), - "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - } - - @Test - public void primitive_weighted_set_attribute_field_gets_attribute_transform() throws ParseException { - assertSummaryField(joinLines("field my_field type weightedset {", - " indexing: attribute | summary", - " summary: matched-elements-only", - "}"), - "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); - } - - @Test - public void explicit_summary_field_can_use_filter_transform_with_reference_to_attribute_source_field() throws ParseException { - String documentSummary = joinLines("document-summary my_summary {", - " summary my_filter_field type array {", - " source: my_field", - " matched-elements-only", - " }", - "}"); - - var search = buildSearch(joinLines( - "field my_field type array {", - " indexing: attribute | summary", - "}"), - documentSummary); - assertSummaryField(search.getSummaryField("my_filter_field"), - SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field"); - assertSummaryField(search.getSummaryField("my_field"), - SummaryTransform.ATTRIBUTE, "my_field"); - } - - @Test - public void unsupported_field_type_throws() throws ParseException { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For schema 'test', document summary 'default', summary field 'my_field': " + - "'matched-elements-only' is not supported for this field type. " + - "Supported field types are: array of primitive, weighted set of primitive, " + - "array of simple struct, map of primitive type to simple struct, " + - "and map of primitive type to primitive type"); - buildSearch(joinLines("field my_field type string {", - " indexing: summary", - " summary: matched-elements-only", - "}")); - } - - private void assertSummaryField(String fieldContent, String fieldName, SummaryTransform expTransform) throws ParseException { - var search = buildSearch(fieldContent); - assertSummaryField(search.getSummaryField(fieldName), expTransform, fieldName); - } - - private void assertSummaryField(SummaryField field, SummaryTransform expTransform, String expSourceField) { - assertEquals(expTransform, field.getTransform()); - assertEquals(expSourceField, field.getSingleSource()); - } - - private Schema buildSearch(String field) throws ParseException { - return buildSearch(field, ""); - } - - private Schema buildSearch(String field, String summary) throws ParseException { - var builder = new ApplicationBuilder(new RankProfileRegistry()); - builder.addSchema(joinLines("search test {", - " document test {", - " struct elem {", - " field name type string {}", - " field weight type int {}", - " }", - field, - " }", - summary, - "}")); - builder.build(true); - return builder.getSchema(); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java deleted file mode 100644 index ab78297bcf9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class NGramTestCase extends AbstractSchemaTestCase { - - @Test - public void testNGram() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/ngram.sd"); - assertNotNull(schema); - - SDField gram1 = schema.getConcreteField("gram_1"); - assertEquals(MatchType.GRAM, gram1.getMatching().getType()); - assertEquals(1, gram1.getMatching().getGramSize()); - - SDField gram2 = schema.getConcreteField("gram_2"); - assertEquals(MatchType.GRAM, gram2.getMatching().getType()); - assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly - - SDField gram3= schema.getConcreteField("gram_3"); - assertEquals(MatchType.GRAM,gram3.getMatching().getType()); - assertEquals(3, gram3.getMatching().getGramSize()); - - assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); - assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); - assertEquals("input gram_3 | ngram 3 | index gram_3", gram3.getIndexingScript().iterator().next().toString()); - - assertFalse(gram1.getNormalizing().doRemoveAccents()); - assertEquals(Stemming.NONE, gram1.getStemming()); - - List queryCommands = gram1.getQueryCommands(); - assertEquals(2, queryCommands.size()); - assertEquals("ngram 1", queryCommands.get(1)); - } - - @Test - public void testInvalidNGramSetting1() throws IOException, ParseException { - try { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram1.sd"); - fail("Should cause an exception"); - } - catch (IllegalArgumentException e) { - assertEquals("gram-size can only be set when the matching mode is 'gram'", e.getMessage()); - } - } - - @Test - public void testInvalidNGramSetting2() throws IOException, ParseException { - try { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram2.sd"); - fail("Should cause an exception"); - } - catch (IllegalArgumentException e) { - assertEquals("gram-size can only be set when the matching mode is 'gram'", e.getMessage()); - } - } - - @Test - public void testInvalidNGramSetting3() throws IOException, ParseException { - try { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/invalidngram3.sd"); - fail("Should cause an exception"); - } - catch (IllegalArgumentException e) { - assertEquals("gram matching is not supported with attributes, use 'index' in indexing", e.getMessage()); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java deleted file mode 100644 index 4eeab15fdd2..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.util.Optional; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromStrings; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class PagedAttributeValidatorTestCase { - - @Test - public void dense_tensor_attribute_supports_paged_setting() throws ParseException { - assertPagedSupported("tensor(x[2],y[2])"); - } - - @Test - public void primitive_attribute_types_support_paged_setting() throws ParseException { - assertPagedSupported("int"); - assertPagedSupported("array"); - assertPagedSupported("weightedset"); - - assertPagedSupported("string"); - assertPagedSupported("array"); - assertPagedSupported("weightedset"); - } - - @Test - public void struct_field_attributes_support_paged_setting() throws ParseException { - var sd = joinLines("schema test {", - " document test {", - " struct elem {", - " field first type int {}", - " field second type string {}", - " }", - " field foo type array {", - " indexing: summary", - " struct-field first {", - " indexing: attribute", - " attribute: paged", - " }", - " struct-field second {", - " indexing: attribute", - " attribute: paged", - " }", - " }", - " }", - "}"); - - var appBuilder = createFromString(sd); - var field = appBuilder.getSchema().getField("foo"); - assertTrue(field.getStructField("first").getAttribute().isPaged()); - assertTrue(field.getStructField("second").getAttribute().isPaged()); - } - - private void assertPagedSupported(String fieldType) throws ParseException { - var appBuilder = createFromString(getSd(fieldType)); - var attribute = appBuilder.getSchema().getAttribute("foo"); - assertTrue(attribute.isPaged()); - } - - @Test - public void non_dense_tensor_attribute_does_not_support_paged_setting() throws ParseException { - assertPagedSettingNotSupported("tensor(x{},y[2])"); - } - - @Test - public void predicate_attribute_does_not_support_paged_setting() throws ParseException { - assertPagedSettingNotSupported("predicate"); - } - - @Test - public void reference_attribute_does_not_support_paged_setting() throws ParseException { - assertPagedSettingNotSupported("reference", Optional.of(getSd("parent", "int"))); - } - - private void assertPagedSettingNotSupported(String fieldType) throws ParseException { - assertPagedSettingNotSupported(fieldType, Optional.empty()); - } - - private void assertPagedSettingNotSupported(String fieldType, Optional parentSd) throws ParseException { - try { - if (parentSd.isPresent()) { - createFromStrings(new BaseDeployLogger(), parentSd.get(), getSd(fieldType)); - } else { - createFromString(getSd(fieldType)); - } - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'foo': The 'paged' attribute setting is not supported for non-dense tensor, predicate and reference types", - e.getMessage()); - } - } - - private String getSd(String fieldType) { - return getSd("test", fieldType); - } - - private String getSd(String docType, String fieldType) { - return joinLines( - "schema " + docType + " {", - " document " + docType + " {", - " field foo type " + fieldType + "{", - " indexing: attribute", - " attribute: paged", - " }", - " }", - "}"); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java deleted file mode 100644 index 6ddacd066b1..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.google.common.collect.ImmutableMap; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; -import com.yahoo.searchdefinition.document.TemporarySDField; - -/* - * Fixture class used for ImportedFieldsResolverTestCase and AdjustPositionSummaryFieldsTestCase. - */ -public class ParentChildSearchModel { - - public Schema parentSchema; - public Schema childSchema; - - ParentChildSearchModel() { - parentSchema = createSearch("parent"); - childSchema = createSearch("child"); - } - - protected Schema createSearch(String name) { - Schema result = new Schema(name, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties()); - result.addDocument(new SDDocumentType(name)); - return result; - } - - protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { - TemporarySDField result = new TemporarySDField(repo, name, dataType); - result.parseIndexingScript(indexingScript); - return result; - } - - @SuppressWarnings("deprecation") - protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) { - return new TemporarySDField(repo, fieldName, NewDocumentReferenceDataType.forDocumentName(parentType)); - } - - protected static void addRefField(Schema child, Schema parent, String fieldName) { - SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName); - child.getDocument().addField(refField); - child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(), - new DocumentReference(refField, parent)))); - } - - protected ParentChildSearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) { - return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName); - } - - protected ParentChildSearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) { - schema.temporaryImportedFields().get().add(new TemporaryImportedField(fieldName, referenceFieldName, targetFieldName)); - return this; - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java deleted file mode 100644 index a3af7706fb9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; - -import org.junit.Test; - -import java.util.Arrays; -import java.util.Iterator; - -import static org.junit.Assert.*; - -/** - * Test Position processor. - * - * @author hmusum - */ -public class PositionTestCase { - - @Test - public void inherited_position_zcurve_field_is_not_added_to_document_fieldset() throws Exception { - ApplicationBuilder sb = ApplicationBuilder.createFromFiles(Arrays.asList( - "src/test/examples/position_base.sd", - "src/test/examples/position_inherited.sd")); - - Schema schema = sb.getSchema("position_inherited"); - FieldSet fieldSet = schema.getDocument().getFieldSets().builtInFieldSets().get(DocumentType.DOCUMENT); - assertFalse(fieldSet.getFieldNames().contains(PositionDataType.getZCurveFieldName("pos"))); - } - - @Test - public void requireThatPositionCanBeAttribute() throws Exception { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_attribute.sd"); - assertNull(schema.getAttribute("pos")); - assertNull(schema.getAttribute("pos.x")); - assertNull(schema.getAttribute("pos.y")); - - assertPositionAttribute(schema, "pos", Attribute.CollectionType.SINGLE); - assertPositionSummary(schema, "pos", false); - } - - @Test - public void requireThatPositionCanNotBeIndex() throws Exception { - try { - ApplicationBuilder.buildFromFile("src/test/examples/position_index.sd"); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'position_index', field 'pos': Indexing of data type 'position' is not " + - "supported, replace 'index' statement with 'attribute'.", e.getMessage()); - } - } - - @Test - public void requireThatSummaryAloneDoesNotCreateZCurve() throws Exception { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_summary.sd"); - assertNull(schema.getAttribute("pos")); - assertNull(schema.getAttribute("pos.x")); - assertNull(schema.getAttribute("pos.y")); - assertNull(schema.getAttribute("pos.zcurve")); - - SummaryField summary = schema.getSummaryField("pos"); - assertNotNull(summary); - assertEquals(2, summary.getSourceCount()); - Iterator it = summary.getSources().iterator(); - assertEquals("pos.x", it.next().getName()); - assertEquals("pos.y", it.next().getName()); - assertEquals(SummaryTransform.NONE, summary.getTransform()); - - assertNull(schema.getSummaryField("pos_ext.distance")); - } - - @Test - public void requireThatExtraFieldCanBePositionAttribute() throws Exception { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_extra.sd"); - assertNull(schema.getAttribute("pos_ext")); - assertNull(schema.getAttribute("pos_ext.x")); - assertNull(schema.getAttribute("pos_ext.y")); - - assertPositionAttribute(schema, "pos_ext", Attribute.CollectionType.SINGLE); - assertPositionSummary(schema, "pos_ext", false); - } - - @Test - public void requireThatPositionArrayIsSupported() throws Exception { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/position_array.sd"); - assertNull(schema.getAttribute("pos")); - assertNull(schema.getAttribute("pos.x")); - assertNull(schema.getAttribute("pos.y")); - - assertPositionAttribute(schema, "pos", Attribute.CollectionType.ARRAY); - assertPositionSummary(schema, "pos", true); - } - - private static void assertPositionAttribute(Schema schema, String fieldName, Attribute.CollectionType type) { - Attribute attribute = schema.getAttribute(PositionDataType.getZCurveFieldName(fieldName)); - assertNotNull(attribute); - assertTrue(attribute.isPosition()); - assertEquals(attribute.getCollectionType(), type); - assertEquals(attribute.getType(), Attribute.Type.LONG); - } - - private static void assertPositionSummary(Schema schema, String fieldName, boolean isArray) { - assertSummaryField(schema, - fieldName, - PositionDataType.getZCurveFieldName(fieldName), - (isArray ? DataType.getArray(PositionDataType.INSTANCE) : PositionDataType.INSTANCE), - SummaryTransform.GEOPOS); - assertNull(schema.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); - assertNull(schema.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); - } - - private static void assertSummaryField(Schema schema, String fieldName, String sourceName, DataType dataType, - SummaryTransform transform) - { - SummaryField summary = schema.getSummaryField(fieldName); - assertNotNull(summary); - assertEquals(1, summary.getSourceCount()); - assertEquals(sourceName, summary.getSingleSource()); - assertEquals(dataType, summary.getDataType()); - assertEquals(transform, summary.getTransform()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java deleted file mode 100644 index 3a0ceebcb0a..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -/** - * Tests for the field "rank {" shortcut - * @author vegardh - * - */ -public class RankModifierTestCase extends AbstractSchemaTestCase { - @Test - public void testLiteral() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java deleted file mode 100644 index 2d0bdb58122..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.google.common.collect.ImmutableList; -import com.yahoo.config.application.api.ApplicationPackage; -import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.path.Path; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; -import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; -import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; -import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Helper class for setting up and asserting over a Search instance with a rank profile given literally - * in the search definition language. - * - * @author geirst - */ -class RankProfileSearchFixture { - - private final ImmutableList importers = ImmutableList.of(new TensorFlowImporter(), - new OnnxImporter(), - new LightGBMImporter(), - new XGBoostImporter()); - private final RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - private final QueryProfileRegistry queryProfileRegistry; - private final Schema schema; - private final Map compiledRankProfiles = new HashMap<>(); - private final ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - - public RankProfileRegistry getRankProfileRegistry() { - return rankProfileRegistry; - } - - public QueryProfileRegistry getQueryProfileRegistry() { - return queryProfileRegistry; - } - - RankProfileSearchFixture(String rankProfiles) throws ParseException { - this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); - } - - RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, - String rankProfiles) throws ParseException { - this(applicationpackage, queryProfileRegistry, rankProfiles, null, null); - } - - RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, - String rankProfiles, String constant, String field) - throws ParseException { - this.queryProfileRegistry = queryProfileRegistry; - ApplicationBuilder builder = new ApplicationBuilder(applicationpackage, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); - String sdContent = "search test {\n" + - " " + (constant != null ? constant : "") + "\n" + - " document test {\n" + - " " + (field != null ? field : "") + "\n" + - " }\n" + - rankProfiles + - "\n" + - "}"; - builder.addSchema(sdContent); - builder.build(true); - schema = builder.getSchema(); - } - - public void assertFirstPhaseExpression(String expExpression, String rankProfile) { - assertEquals(expExpression, compiledRankProfile(rankProfile).getFirstPhaseRanking().getRoot().toString()); - } - - public void assertSecondPhaseExpression(String expExpression, String rankProfile) { - assertEquals(expExpression, compiledRankProfile(rankProfile).getSecondPhaseRanking().getRoot().toString()); - } - - public void assertRankProperty(String expValue, String name, String rankProfile) { - List rankPropertyList = compiledRankProfile(rankProfile).getRankPropertyMap().get(name); - assertEquals(1, rankPropertyList.size()); - assertEquals(expValue, rankPropertyList.get(0).getValue()); - } - - public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { - assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); - } - - public RankProfile compileRankProfile(String rankProfile) { - return compileRankProfile(rankProfile, Path.fromString("nonexistinng")); - } - - public RankProfile compileRankProfile(String rankProfile, Path applicationDir) { - RankProfile compiled = rankProfileRegistry.get(schema, rankProfile) - .compile(queryProfileRegistry, - new ImportedMlModels(applicationDir.toFile(), executor, importers)); - compiledRankProfiles.put(rankProfile, compiled); - return compiled; - } - - /** Returns the given uncompiled profile */ - public RankProfile rankProfile(String rankProfile) { - return rankProfileRegistry.get(schema, rankProfile); - } - - /** Returns the given compiled profile, or null if not compiled yet or not present at all */ - public RankProfile compiledRankProfile(String rankProfile) { - return compiledRankProfiles.get(rankProfile); - } - - public Schema search() { return schema; } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java deleted file mode 100644 index 0ab7406b5b1..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile.RankProperty; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.fail; - -public class RankPropertyVariablesTestCase extends AbstractSchemaTestCase { - - @Test - public void testRankPropVariables() throws IOException, ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/rankpropvars.sd", - new BaseDeployLogger(), - rankProfileRegistry, - new QueryProfileRegistry()); - assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar1", "foo"); - assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvar_2", "bar"); - assertRankPropEquals(rankProfileRegistry.get(schema, "other").getRankProperties(), "$testvarOne23", "baz"); - assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$Testvar1", "1"); - assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$Testvar_4", "4"); - assertRankPropEquals(rankProfileRegistry.get(schema, "another").getRankProperties(), "$testvarFour23", "234234.234"); - } - - private void assertRankPropEquals(List props, String key, String val) { - for (RankProperty prop : props) { - if (prop.getName().equals(key)) { - if (prop.getValue().equals(val)) { - return; - } - } - } - fail(key+":"+val+ " not found in rank properties."); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java deleted file mode 100644 index 9b32d97ef75..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ /dev/null @@ -1,521 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.search.query.profile.types.TensorFieldType; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.tensor.TensorType; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.stream.Collectors; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class RankingExpressionTypeResolverTestCase { - - @Test - public void tensorFirstPhaseMustProduceDouble() throws Exception { - try { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10],y[3]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: attribute(a)", - " }", - " }", - "}" - )); - builder.build(true); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", - Exceptions.toMessageString(expected)); - } - } - - - @Test - public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { - try { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "schema test {", - " document test { ", - " field a type tensor(d0[3]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " function my_func() {", - " expression: x_tensor*2.0", - " }", - " function inline other_func() {", - " expression: z_tensor+3.0", - " }", - " first-phase {", - " expression: reduce(attribute(a),sum,d0)+y_tensor+my_func+other_func", - " }", - " constants {", - " x_tensor {", // legacy form - " type: tensor(x{})", - " value: { {x:bar}:17 }", - " }", - " y_tensor tensor(y{}):{{y:foo}:42 }", - " z_tensor tensor(z{}):{qux:666}", - " }", - " }", - "}" - )); - builder.build(true); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x{},y{},z{})", - Exceptions.toMessageString(expected)); - } - } - - - - @Test - public void tensorSecondPhaseMustProduceDouble() throws Exception { - try { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10],y[3]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(attribute(a))", - " }", - " second-phase {", - " expression: attribute(a)", - " }", - " }", - "}" - )); - builder.build(true); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In schema 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { - try { - ApplicationBuilder schemaBuilder = new ApplicationBuilder(); - schemaBuilder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10],y[5]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(if(1>0, attribute(a), attribute(b)))", - " }", - " }", - "}" - )); - schemaBuilder.build(true); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In schema 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[10],y[5]) while the 'false' type is tensor(z[10])" + - "\n'true' branch: attribute(a)" + - "\n'false' branch: attribute(b)", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void testFunctionInvocationTypes() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10],y[3]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " function macro1(attribute_to_use) {", - " expression: attribute(attribute_to_use)", - " }", - " summary-features {", - " macro1(a)", - " macro1(b)", - " }", - " }", - "}" - )); - builder.build(true); - RankProfile profile = - builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), - summaryFeatures(profile).get("macro1(a)").type(profile.typeContext(builder.getQueryProfileRegistry()))); - assertEquals(TensorType.fromSpec("tensor(z[10])"), - summaryFeatures(profile).get("macro1(b)").type(profile.typeContext(builder.getQueryProfileRegistry()))); - } - - @Test - public void testTensorFunctionInvocationTypes_Nested() throws Exception { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10],y[1]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " function return_a() {", - " expression: return_first(attribute(a), attribute(b))", - " }", - " function return_b() {", - " expression: return_second(attribute(a), attribute(b))", - " }", - " function return_first(e1, e2) {", - " expression: e1", - " }", - " function return_second(e1, e2) {", - " expression: return_first(e2, e1)", - " }", - " summary-features {", - " return_a", - " return_b", - " }", - " }", - "}" - )); - builder.build(true); - RankProfile profile = - builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), - summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); - assertEquals(TensorType.fromSpec("tensor(z[10])"), - summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); - } - - @Test - public void testAttributeInvocationViaBoundIdentifier() throws Exception { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search newsarticle {", - " document newsarticle {", - " field title type string {", - " indexing {", - " input title | index", - " }", - " weight: 30", - " }", - " field usstaticrank type int {", - " indexing: summary | attribute", - " }", - " field eustaticrank type int {", - " indexing: summary | attribute", - " }", - " }", - " rank-profile default {", - " macro newsboost() { ", - " expression: 200 * matches(title)", - " }", - " macro commonboost(mystaticrank) { ", - " expression: attribute(mystaticrank) + newsboost", - " }", - " macro commonfirstphase(mystaticrank) { ", - " expression: nativeFieldMatch(title) + commonboost(mystaticrank) ", - " }", - " first-phase { expression: commonfirstphase(usstaticrank) }", - " }", - " rank-profile eurank inherits default {", - " first-phase { expression: commonfirstphase(eustaticrank) }", - " }", - "}")); - builder.build(true); - RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "eurank"); - } - - @Test - public void testTensorFunctionInvocationTypes_NestedSameName() throws Exception { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[10],y[1]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " function return_a() {", - " expression: return_first(attribute(a), attribute(b))", - " }", - " function return_b() {", - " expression: return_second(attribute(a), attribute(b))", - " }", - " function return_first(e1, e2) {", - " expression: just_return(e1)", - " }", - " function just_return(e1) {", - " expression: e1", - " }", - " function return_second(e1, e2) {", - " expression: return_first(e2+0, e1)", - " }", - " summary-features {", - " return_a", - " return_b", - " }", - " }", - "}" - )); - builder.build(true); - RankProfile profile = - builder.getRankProfileRegistry().get(builder.getSchema(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), - summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); - assertEquals(TensorType.fromSpec("tensor(z[10])"), - summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); - } - - @Test - public void testTensorFunctionInvocationTypes_viaFuncWithExpr() throws Exception { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search test {", - " document test {", - " field t1 type tensor(y{}) { indexing: attribute | summary }", - " field t2 type tensor(x{}) { indexing: attribute | summary }", - " }", - " rank-profile test {", - " function my_func(t) { expression: sum(t, x) + 1 }", - " function test_func_via_func_with_expr() { expression: call_func_with_expr( attribute(t1), attribute(t2) ) }", - " function call_func_with_expr(a, b) { expression: my_func( a * b ) }", - " summary-features { test_func_via_func_with_expr }", - " }", - "}")); - builder.build(true); - RankProfile profile = builder.getRankProfileRegistry().get(builder.getSchema(), "test"); - assertEquals(TensorType.fromSpec("tensor(y{})"), - summaryFeatures(profile).get("test_func_via_func_with_expr").type(profile.typeContext(builder.getQueryProfileRegistry()))); - } - - @Test - public void importedFieldsAreAvailable() throws Exception { - ApplicationBuilder builder = new ApplicationBuilder(); - builder.addSchema(joinLines( - "search parent {", - " document parent {", - " field a type tensor(x[5],y[1000]) {", - " indexing: attribute", - " }", - " }", - "}" - )); - builder.addSchema(joinLines( - "search child {", - " document child { ", - " field ref type reference {", - "indexing: attribute | summary", - " }", - " }", - " import field ref.a as imported_a {}", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(attribute(imported_a))", - " }", - " }", - "}" - )); - builder.build(true); - } - - @Test - public void undeclaredQueryFeaturesAreAccepted() throws Exception { - InspectableDeployLogger logger = new InspectableDeployLogger(); - ApplicationBuilder builder = new ApplicationBuilder(logger); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field anyfield type double {" + - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: query(foo) + f() + sum(attribute(anyfield))", - " }", - " function f() {", - " expression: query(bar) + query(baz)", - " }", - " }", - "}" - )); - builder.build(true); - String message = logger.findMessage("The following query features"); - assertNull(message); - } - - @Test - public void undeclaredQueryFeaturesAreNotAcceptedWhenStrict() throws Exception { - try { - InspectableDeployLogger logger = new InspectableDeployLogger(); - ApplicationBuilder builder = new ApplicationBuilder(logger); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field anyfield type double {" + - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " strict: true" + - " first-phase {", - " expression: query(foo) + f() + sum(attribute(anyfield))", - " }", - " function f() {", - " expression: query(bar) + query(baz)", - " }", - " }", - "}" - )); - builder.build(true); - } - catch (IllegalArgumentException e) { - assertEquals("In schema 'test', rank profile 'my_rank_profile': rank profile 'my_rank_profile' is strict but is missing a query profile type declaration of features [query(bar), query(baz), query(foo)]", - Exceptions.toMessageString(e)); - } - } - - @Test - public void undeclaredQueryFeaturesAreAcceptedWithWarningWhenUsingTensors() throws Exception { - InspectableDeployLogger logger = new InspectableDeployLogger(); - ApplicationBuilder builder = new ApplicationBuilder(logger); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field anyfield type tensor(d[2]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: query(foo) + f() + sum(attribute(anyfield))", - " }", - " function f() {", - " expression: query(bar) + query(baz)", - " }", - " }", - "}" - )); - builder.build(true); - String message = logger.findMessage("The following query features"); - assertNotNull(message); - assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared in query profile types and " + - "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", - message); - } - - @Test - public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception { - InspectableDeployLogger logger = new InspectableDeployLogger(); - ApplicationBuilder builder = new ApplicationBuilder(logger); - QueryProfileType myType = new QueryProfileType("mytype"); - myType.addField(new FieldDescription("rank.feature.query(foo)", - new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), - builder.getQueryProfileRegistry().getTypeRegistry()); - myType.addField(new FieldDescription("rank.feature.query(bar)", - new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), - builder.getQueryProfileRegistry().getTypeRegistry()); - myType.addField(new FieldDescription("rank.feature.query(baz)", - new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), - builder.getQueryProfileRegistry().getTypeRegistry()); - builder.getQueryProfileRegistry().getTypeRegistry().register(myType); - builder.addSchema(joinLines( - "search test {", - " document test { ", - " field anyfield type tensor(d[2]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(query(foo) + f() + sum(attribute(anyfield)))", - " }", - " function f() {", - " expression: query(bar) + query(baz)", - " }", - " }", - "}" - )); - builder.build(true); - String message = logger.findMessage("The following query features"); - assertNull(message); - } - - private Map summaryFeatures(RankProfile profile) { - return profile.getSummaryFeatures().stream().collect(Collectors.toMap(f -> f.toString(), f -> f)); - } - - private static class InspectableDeployLogger implements DeployLogger { - - private List messages = new ArrayList<>(); - - @Override - public void log(Level level, String message) { - messages.add(level + ": " + message); - } - - /** Returns the first message containing the given string, or null if none */ - public String findMessage(String substring) { - return messages.stream().filter(message -> message.contains(substring)).findFirst().orElse(null); - } - - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java deleted file mode 100644 index adaf1ef78f9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.After; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author lesters - */ -public class RankingExpressionWithLightGBMTestCase { - - private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); - - private final static String lightGBMExpression = - "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; - - @After - public void removeGeneratedModelFiles() { - IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - } - - @Test - public void testLightGBMReference() { - RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); - search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); - } - - @Test - public void testNestedLightGBMReference() { - RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); - search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); - } - - @Test - public void testImportingFromStoredExpressions() throws IOException { - RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); - search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); - try { - storedApplicationDirectory.toFile().mkdirs(); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - RankingExpressionWithOnnxTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); - searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); - } - finally { - IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); - } - } - - private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { - return fixtureWith(firstPhaseExpression, null, null, - new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(applicationDir)); - } - - private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, - String constant, - String field, - RankingExpressionWithOnnxTestCase.StoringApplicationPackage application) { - try { - RankProfileSearchFixture fixture = new RankProfileSearchFixture( - application, - application.getQueryProfiles(), - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: " + firstPhaseExpression + - " }\n" + - " }", - constant, - field); - fixture.compileRankProfile("my_profile", applicationDir.append("models")); - return fixture; - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - -} - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java deleted file mode 100644 index 6820a8d9678..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.DocumentDatabase; -import com.yahoo.vespa.model.search.IndexedSearchCluster; -import org.junit.After; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class RankingExpressionWithOnnxModelTestCase { - - private final Path applicationDir = Path.fromString("src/test/integration/onnx-model/"); - - @After - public void removeGeneratedModelFiles() { - IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - } - - @Test - public void testOnnxModelFeature() throws Exception { - VespaModel model = loadModel(applicationDir); - assertTransformedFeature(model); - assertGeneratedConfig(model); - - Path storedApplicationDir = applicationDir.append("copy"); - try { - storedApplicationDir.toFile().mkdirs(); - IOUtils.copy(applicationDir.append("services.xml").toString(), storedApplicationDir.append("services.xml").toString()); - IOUtils.copyDirectory(applicationDir.append("schemas").toFile(), storedApplicationDir.append("schemas").toFile()); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - - VespaModel storedModel = loadModel(storedApplicationDir); - assertTransformedFeature(storedModel); - assertGeneratedConfig(storedModel); - } - finally { - IOUtils.recursiveDeleteDir(storedApplicationDir.toFile()); - } - } - - private VespaModel loadModel(Path path) throws Exception { - FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); - DeployState state = new DeployState.Builder().applicationPackage(applicationPackage).build(); - return new VespaModel(state); - } - - private void assertGeneratedConfig(VespaModel vespaModel) { - DocumentDatabase db = ((IndexedSearchCluster)vespaModel.getSearchClusters().get(0)).getDocumentDbs().get(0); - - RankingConstantsConfig.Builder rankingConstantsConfigBuilder = new RankingConstantsConfig.Builder(); - db.getConfig(rankingConstantsConfigBuilder); - var rankingConstantsConfig = rankingConstantsConfigBuilder.build(); - assertEquals(1, rankingConstantsConfig.constant().size()); - assertEquals("my_constant", rankingConstantsConfig.constant(0).name()); - assertEquals("tensor(d0[2])", rankingConstantsConfig.constant(0).type()); - assertEquals("files/constant.json", rankingConstantsConfig.constant(0).fileref().value()); - - OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); - ((OnnxModelsConfig.Producer) db).getConfig(builder); - OnnxModelsConfig config = new OnnxModelsConfig(builder); - assertEquals(6, config.model().size()); - for (OnnxModelsConfig.Model model : config.model()) { - assertTrue(model.dry_run_on_setup()); - } - - OnnxModelsConfig.Model model = config.model(0); - assertEquals("my_model", model.name()); - assertEquals(3, model.input().size()); - assertEquals("second/input:0", model.input(0).name()); - assertEquals("constant(my_constant)", model.input(0).source()); - assertEquals("first_input", model.input(1).name()); - assertEquals("attribute(document_field)", model.input(1).source()); - assertEquals("third_input", model.input(2).name()); - assertEquals("rankingExpression(my_function)", model.input(2).source()); - assertEquals(3, model.output().size()); - assertEquals("path/to/output:0", model.output(0).name()); - assertEquals("out", model.output(0).as()); - assertEquals("path/to/output:1", model.output(1).name()); - assertEquals("path_to_output_1", model.output(1).as()); - assertEquals("path/to/output:2", model.output(2).name()); - assertEquals("path_to_output_2", model.output(2).as()); - - model = config.model(1); - assertEquals("dynamic_model", model.name()); - assertEquals(1, model.input().size()); - assertEquals(1, model.output().size()); - assertEquals("rankingExpression(my_function)", model.input(0).source()); - - model = config.model(2); - assertEquals("unbound_model", model.name()); - assertEquals(1, model.input().size()); - assertEquals(1, model.output().size()); - assertEquals("rankingExpression(my_function)", model.input(0).source()); - - model = config.model(3); - assertEquals("files_model_onnx", model.name()); - assertEquals(3, model.input().size()); - assertEquals(3, model.output().size()); - assertEquals("path/to/output:0", model.output(0).name()); - assertEquals("path_to_output_0", model.output(0).as()); - assertEquals("path/to/output:1", model.output(1).name()); - assertEquals("path_to_output_1", model.output(1).as()); - assertEquals("path/to/output:2", model.output(2).name()); - assertEquals("path_to_output_2", model.output(2).as()); - assertEquals("files_model_onnx", model.name()); - - model = config.model(4); - assertEquals("another_model", model.name()); - assertEquals("third_input", model.input(2).name()); - assertEquals("rankingExpression(another_function)", model.input(2).source()); - - model = config.model(5); - assertEquals("files_summary_model_onnx", model.name()); - assertEquals(3, model.input().size()); - assertEquals(3, model.output().size()); - } - - private void assertTransformedFeature(VespaModel model) { - DocumentDatabase db = ((IndexedSearchCluster)model.getSearchClusters().get(0)).getDocumentDbs().get(0); - RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); - ((RankProfilesConfig.Producer) db).getConfig(builder); - RankProfilesConfig config = new RankProfilesConfig(builder); - assertEquals(10, config.rankprofile().size()); - - assertEquals("test_model_config", config.rankprofile(2).name()); - assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(2).fef().property(0).name()); - assertEquals("vespa.rank.firstphase", config.rankprofile(2).fef().property(2).name()); - assertEquals("rankingExpression(firstphase)", config.rankprofile(2).fef().property(2).value()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(2).fef().property(3).name()); - assertEquals("onnxModel(my_model).out{d0:1}", config.rankprofile(2).fef().property(3).value()); - - assertEquals("test_generated_model_config", config.rankprofile(3).name()); - assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(3).fef().property(0).name()); - assertEquals("rankingExpression(first_input).rankingScript", config.rankprofile(3).fef().property(2).name()); - assertEquals("rankingExpression(second_input).rankingScript", config.rankprofile(3).fef().property(4).name()); - assertEquals("rankingExpression(third_input).rankingScript", config.rankprofile(3).fef().property(6).name()); - assertEquals("vespa.rank.firstphase", config.rankprofile(3).fef().property(8).name()); - assertEquals("rankingExpression(firstphase)", config.rankprofile(3).fef().property(8).value()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(3).fef().property(9).name()); - assertEquals("onnxModel(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value()); - - assertEquals("test_summary_features", config.rankprofile(4).name()); - assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(0).name()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(3).name()); - assertEquals("1", config.rankprofile(4).fef().property(3).value()); - assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(4).name()); - assertEquals("onnxModel(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(4).value()); - assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(5).name()); - assertEquals("onnxModel(another_model).out", config.rankprofile(4).fef().property(5).value()); - - assertEquals("test_dynamic_model", config.rankprofile(5).name()); - assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(5).fef().property(0).name()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(5).fef().property(3).name()); - assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:1}", config.rankprofile(5).fef().property(3).value()); - - assertEquals("test_dynamic_model_2", config.rankprofile(6).name()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(6).fef().property(5).name()); - assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:2}", config.rankprofile(6).fef().property(5).value()); - - assertEquals("test_dynamic_model_with_transformer_tokens", config.rankprofile(7).name()); - assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(7).fef().property(1).name()); - assertEquals("tensor(d0[1],d1[10])((if (d1 < 1.0 + rankingExpression(__token_length@-1993461420) + 1.0, 0.0, if (d1 < 1.0 + rankingExpression(__token_length@-1993461420) + 1.0 + rankingExpression(__token_length@-1993461420) + 1.0, 1.0, 0.0))))", config.rankprofile(7).fef().property(1).value()); - - assertEquals("test_unbound_model", config.rankprofile(8).name()); - assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(8).fef().property(0).name()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(8).fef().property(3).name()); - assertEquals("onnxModel(unbound_model).my_output{d0:0, d1:1}", config.rankprofile(8).fef().property(3).value()); - - - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java deleted file mode 100644 index bd6310ffd48..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.path.Path; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.tensor.TensorType; -import com.yahoo.yolean.Exceptions; -import org.junit.After; -import org.junit.Test; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -public class RankingExpressionWithOnnxTestCase { - - private final Path applicationDir = Path.fromString("src/test/integration/onnx/"); - - private final static String name = "mnist_softmax"; - private final static String vespaExpression = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(mnist_softmax_layer_Variable), f(a,b)(a * b)), sum, d2) * 1.0, constant(mnist_softmax_layer_Variable_1) * 1.0, f(a,b)(a + b))"; - - @After - public void removeGeneratedModelFiles() { - IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - } - - @Test - public void testOnnxReferenceWithConstantFeature() { - RankProfileSearchFixture search = fixtureWith("constant(mytensor)", - "onnx_vespa('mnist_softmax.onnx')", - "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", - null); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - @Test - public void testOnnxReferenceWithQueryFeature() { - String queryProfile = ""; - String queryProfileType = - "" + - " " + - ""; - StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, - queryProfile, - queryProfileType); - RankProfileSearchFixture search = fixtureWith("query(mytensor)", - "onnx_vespa('mnist_softmax.onnx')", - null, - null, - "Placeholder", - application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - @Test - public void testOnnxReferenceWithDocumentFeature() { - StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); - RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", - "onnx_vespa('mnist_softmax.onnx')", - null, - "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", - "Placeholder", - application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - - @Test - public void testOnnxReferenceWithFeatureCombination() { - String queryProfile = ""; - String queryProfileType = - "" + - " " + - ""; - StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, queryProfileType); - RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", - "onnx_vespa('mnist_softmax.onnx')", - "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", - "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", - "Placeholder", - application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - - @Test - public void testNestedOnnxReference() { - RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "5 + sum(onnx_vespa('mnist_softmax.onnx'))"); - search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); - } - - @Test - public void testOnnxReferenceWithSpecifiedOutput() { - RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx', 'layer_add')"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - @Test - public void testOnnxReferenceWithSpecifiedOutputAndSignature() { - RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx', 'default.layer_add')"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - @Test - public void testOnnxReferenceMissingFunction() throws ParseException { - try { - RankProfileSearchFixture search = new RankProfileSearchFixture( - new StoringApplicationPackage(applicationDir), - new QueryProfileRegistry(), - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: onnx_vespa('mnist_softmax.onnx')" + - " }\n" + - " }"); - search.compileRankProfile("my_profile", applicationDir.append("models")); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - fail("Expecting exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + - "onnx_vespa(\"mnist_softmax.onnx\"): " + - "Model refers input 'Placeholder' of type tensor(d0[1],d1[784]) but this function is " + - "not present in rank profile 'my_profile'", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void testOnnxReferenceWithWrongFunctionType() { - try { - RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d5[10])(0.0)", - "onnx_vespa('mnist_softmax.onnx')"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - fail("Expecting exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + - "onnx_vespa(\"mnist_softmax.onnx\"): " + - "Model refers input 'Placeholder'. The required type of this is tensor(d0[1],d1[784]), " + - "but this function returns tensor(d0[1],d5[10])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void testOnnxReferenceSpecifyingNonExistingOutput() { - try { - RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx', 'y')"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - fail("Expecting exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + - "onnx_vespa(\"mnist_softmax.onnx\",\"y\"): " + - "No expressions named 'y' in model 'mnist_softmax.onnx'. Available expressions: default.layer_add", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void testImportingFromStoredExpressions() throws IOException { - RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", - "onnx_vespa(\"mnist_softmax.onnx\")"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); - try { - storedApplicationDirectory.toFile().mkdirs(); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = fixtureWith("tensor(d0[2],d1[784])(0.0)", - "onnx_vespa('mnist_softmax.onnx')", - null, - null, - "Placeholder", - storedApplication); - searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); - // Verify that the constants exists, but don't verify the content as we are not - // simulating file distribution in this test - } - finally { - IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); - } - } - - @Test - public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { - String rankProfile = - " rank-profile my_profile {\n" + - " function Placeholder() {\n" + - " expression: tensor(d0[1],d1[784])(0.0)\n" + - " }\n" + - " function " + name + "_layer_Variable() {\n" + - " expression: tensor(d1[10],d2[784])(0.0)\n" + - " }\n" + - " first-phase {\n" + - " expression: onnx_vespa('mnist_softmax.onnx')" + - " }\n" + - " }" + - " rank-profile my_profile_child inherits my_profile {\n" + - " }"; - - String vespaExpressionWithoutConstant = - "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), " + name + "_layer_Variable, f(a,b)(a * b)), sum, d2) * 1.0, constant(" + name + "_layer_Variable_1) * 1.0, f(a,b)(a + b))"; - RankProfileSearchFixture search = uncompiledFixtureWith(rankProfile, new StoringApplicationPackage(applicationDir)); - search.compileRankProfile("my_profile", applicationDir.append("models")); - search.compileRankProfile("my_profile_child", applicationDir.append("models")); - search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - - assertNull("Constant overridden by function is not added", - search.search().constants().get(name + "_Variable")); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); - try { - storedApplicationDirectory.toFile().mkdirs(); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); - searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); - searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); - searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by function is not added", - searchFromStored.search().constants().get(name + "_Variable")); - } finally { - IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); - } - } - - @Test - public void testFunctionGeneration() { - final String name = "small_constants_and_functions"; - final String rankProfiles = - " rank-profile my_profile {\n" + - " function input() {\n" + - " expression: tensor(d0[3])(0.0)\n" + - " }\n" + - " first-phase {\n" + - " expression: onnx_vespa('" + name + ".onnx')" + - " }\n" + - " }"; - final String functionName = "imported_ml_function_" + name + "_exp_output"; - final String expression = "join(" + functionName + ", reduce(join(join(reduce(" + functionName + ", sum, d0), tensor(d0[1])(1.0), f(a,b)(a * b)), constant(" + name + "_epsilon), f(a,b)(a + b)), sum, d0), f(a,b)(a / b))"; - final String functionExpression = "map(input, f(a)(exp(a)))"; - - RankProfileSearchFixture search = uncompiledFixtureWith(rankProfiles, new StoringApplicationPackage(applicationDir)); - search.compileRankProfile("my_profile", applicationDir.append("models")); - search.assertFirstPhaseExpression(expression, "my_profile"); - search.assertFunction(functionExpression, functionName, "my_profile"); - } - - @Test - public void testImportingFromStoredExpressionsWithSmallConstantsAndInheritance() throws IOException { - final String name = "small_constants_and_functions"; - final String rankProfiles = - " rank-profile my_profile {\n" + - " function input() {\n" + - " expression: tensor(d0[3])(0.0)\n" + - " }\n" + - " first-phase {\n" + - " expression: onnx_vespa('" + name + ".onnx')" + - " }\n" + - " }" + - " rank-profile my_profile_child inherits my_profile {\n" + - " }"; - final String functionName = "imported_ml_function_" + name + "_exp_output"; - final String expression = "join(" + functionName + ", reduce(join(join(reduce(" + functionName + ", sum, d0), tensor(d0[1])(1.0), f(a,b)(a * b)), constant(" + name + "_epsilon), f(a,b)(a + b)), sum, d0), f(a,b)(a / b))"; - final String functionExpression = "map(input, f(a)(exp(a)))"; - - RankProfileSearchFixture search = uncompiledFixtureWith(rankProfiles, new StoringApplicationPackage(applicationDir)); - search.compileRankProfile("my_profile", applicationDir.append("models")); - search.compileRankProfile("my_profile_child", applicationDir.append("models")); - search.assertFirstPhaseExpression(expression, "my_profile"); - search.assertFirstPhaseExpression(expression, "my_profile_child"); - assertSmallConstant(name + "_epsilon", TensorType.fromSpec("tensor()"), search); - search.assertFunction(functionExpression, functionName, "my_profile"); - search.assertFunction(functionExpression, functionName, "my_profile_child"); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); - try { - storedApplicationDirectory.toFile().mkdirs(); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - StoringApplicationPackage storedApplication = new StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfiles, storedApplication); - searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); - searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); - searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); - searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); - assertSmallConstant(name + "_epsilon", TensorType.fromSpec("tensor()"), search); - searchFromStored.assertFunction(functionExpression, functionName, "my_profile"); - searchFromStored.assertFunction(functionExpression, functionName, "my_profile_child"); - } - finally { - IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); - } - } - - private void assertSmallConstant(String name, TensorType type, RankProfileSearchFixture search) { - var value = search.compiledRankProfile("my_profile").constants().get(FeatureNames.asConstantFeature(name)); - assertNotNull(value); - assertEquals(type, value.type()); - } - - private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression) { - return fixtureWith(placeholderExpression, firstPhaseExpression, null, null, "Placeholder", - new StoringApplicationPackage(applicationDir)); - } - - private RankProfileSearchFixture fixtureWith(String placeholderExpression, String firstPhaseExpression, - String constant, String field) { - return fixtureWith(placeholderExpression, firstPhaseExpression, constant, field, "Placeholder", - new StoringApplicationPackage(applicationDir)); - } - - private RankProfileSearchFixture uncompiledFixtureWith(String rankProfile, StoringApplicationPackage application) { - try { - return new RankProfileSearchFixture(application, application.getQueryProfiles(), - rankProfile, null, null); - } - catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - private RankProfileSearchFixture fixtureWith(String functionExpression, - String firstPhaseExpression, - String constant, - String field, - String functionName, - StoringApplicationPackage application) { - try { - RankProfileSearchFixture fixture = new RankProfileSearchFixture( - application, - application.getQueryProfiles(), - " rank-profile my_profile {\n" + - " function " + functionName + "() {\n" + - " expression: " + functionExpression + - " }\n" + - " first-phase {\n" + - " expression: " + firstPhaseExpression + - " }\n" + - " }", - constant, - field); - fixture.compileRankProfile("my_profile", applicationDir.append("models")); - return fixture; - } - catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - static class StoringApplicationPackage extends MockApplicationPackage { - - StoringApplicationPackage(Path applicationPackageWritableRoot) { - this(applicationPackageWritableRoot, null, null); - } - - StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { - super(new File(applicationPackageWritableRoot.toString()), - null, null, List.of(), Map.of(), null, - null, null, false, queryProfile, queryProfileType); - } - - @Override - public ApplicationFile getFile(Path file) { - return new MockApplicationFile(file, Path.fromString(root().toString())); - } - - @Override - public List getFiles(Path path, String suffix) { - File[] files = getFileReference(path).listFiles(); - if (files == null) return List.of(); - List readers = new ArrayList<>(); - for (File file : files) { - if ( ! file.getName().endsWith(suffix)) continue; - try { - readers.add(new NamedReader(file.getName(), new FileReader(file))); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - return readers; - } - - } - - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java deleted file mode 100644 index 5bf2115c3f7..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author geirst - */ -public class RankingExpressionWithTensorTestCase { - - @Test - public void requireThatSingleLineConstantMappedTensorCanBeParsed() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x{},y{}):{ {x:1,y:2}:1, {x:2,y:1}:2 }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatSingleLineConstantIndexedTensorCanBeParsed() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x[3]):{ {x:0}:1, {x:1}:2, {x:2}:3 }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatSingleLineConstantIndexedTensorShortFormCanBeParsed() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x[3]):[1, 2, 3]\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: sum(constant(my_tensor))\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x{},y{}):{{x:1,y:2}:1, {x:2,y:1}:2}\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatMultiLineConstantTensorAndTypeCanBeParsed() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x{},y{}):\n" + - " { {x:1,y:2}:1,\n" + - " {x:2,y:1}:2 }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0, {x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatConstantTensorsCanBeUsedInSecondPhaseExpression() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " second-phase {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x{}):{ {x:1}:1 }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertSecondPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatConstantTensorsCanBeUsedInInheritedRankProfile() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile parent {\n" + - " constants {\n" + - " my_tensor {\n" + - " value: { {x:1}:1 }\n" + - " }\n" + - " }\n" + - " }\n" + - " rank-profile my_profile inherits parent {\n" + - " first-phase {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " function my_macro() {\n" + - " expression: sum(my_tensor)\n" + - " }\n" + - " first-phase {\n" + - " expression: 5.0 + my_macro\n" + - " }\n" + - " constants {\n" + - " my_tensor tensor(x{}):{ {x:1}:1 }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); - f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); - f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatCombinationOfConstantTensorsAndConstantValuesCanBeUsed() throws ParseException { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: my_number_1 + sum(my_tensor) + my_number_2\n" + - " }\n" + - " constants {\n" + - " my_number_1 double: 3.0\n" + - " my_tensor tensor(x{}):{ {x:1}:1 }\n" + - " my_number_2 double: 5.0\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - f.assertFirstPhaseExpression("3.0 + reduce(constant(my_tensor), sum) + 5.0", "my_profile"); - f.assertRankProperty("tensor(x{}):{1:1.0}", "constant(my_tensor).value", "my_profile"); - f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); - } - - @Test - public void requireThatInvalidTensorTypeSpecThrowsException() throws ParseException { - try { - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " constants {\n" + - " my_tensor tensor(x):{ {x:1}:1 }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertStartsWith("Type of constant(my_tensor): Illegal tensor type spec: A tensor type spec must be on the form", - e.getMessage()); - } - } - - private void assertStartsWith(String prefix, String string) { - assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java deleted file mode 100644 index ffc073ed434..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; -import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.tensor.Tensor; -import org.junit.Test; - -import java.util.Collections; - -import static org.junit.Assert.assertEquals; - -public class RankingExpressionWithTransformerTokensTestCase { - - @Test - public void testTokenInputIds() throws Exception { - String expected = "tensor(d0[1],d1[12]):[101,1,2,102,3,4,5,102,6,7,102,0]"; - String a = "tensor(d0[2]):[1,2]"; - String b = "tensor(d0[3]):[3,4,5]"; - String c = "tensor(d0[2]):[6,7]"; - String expression = "tokenInputIds(12, a, b, c)"; - Tensor result = evaluateExpression(expression, a, b, c); - assertEquals(Tensor.from(expected), result); - } - - @Test - public void testTokenTypeIds() throws Exception { - String expected = "tensor(d0[1],d1[10]):[0,0,0,0,1,1,1,1,0,0]"; - String a = "tensor(d0[2]):[1,2]"; - String b = "tensor(d0[3]):[3,4,5]"; - String expression = "tokenTypeIds(10, a, b)"; - Tensor result = evaluateExpression(expression, a, b); - assertEquals(Tensor.from(expected), result); - } - - @Test - public void testAttentionMask() throws Exception { - String expected = "tensor(d0[1],d1[10]):[1,1,1,1,1,1,1,1,0,0]"; - String a = "tensor(d0[2]):[1,2]"; - String b = "tensor(d0[3]):[3,4,5]"; - String expression = "tokenAttentionMask(10, a, b)"; - Tensor result = evaluateExpression(expression, a, b); - assertEquals(Tensor.from(expected), result); - } - - private Tensor evaluateExpression(String expression, String a, String b) throws Exception { - return evaluateExpression(expression, a, b, null, null); - } - - private Tensor evaluateExpression(String expression, String a, String b, String c) throws Exception { - return evaluateExpression(expression, a, b, c, null); - } - - private Tensor evaluateExpression(String expression, String a, String b, String c, String d) throws Exception { - MapContext context = new MapContext(); - if (a != null) context.put("a", new TensorValue(Tensor.from(a))); - if (b != null) context.put("b", new TensorValue(Tensor.from(b))); - if (c != null) context.put("c", new TensorValue(Tensor.from(c))); - if (d != null) context.put("d", new TensorValue(Tensor.from(d))); - var transformContext = createTransformContext(); - var rankingExpression = new RankingExpression(expression); - var transformed = new TokenTransformer().transform(rankingExpression, transformContext); - for (var entry : transformContext.rankProfile().getFunctions().entrySet()) { - context.put(entry.getKey(), entry.getValue().function().getBody().evaluate(context).asDouble()); - } - return transformed.evaluate(context).asTensor(); - } - - private RankProfileTransformContext createTransformContext() throws ParseException { - MockApplicationPackage application = (MockApplicationPackage) MockApplicationPackage.createEmpty(); - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfileRegistry = application.getQueryProfiles(); - String sdContent = "search test {\n" + - " document test {}\n" + - " rank-profile my_profile inherits default {}\n" + - "}"; - ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); - schemaBuilder.addSchema(sdContent); - schemaBuilder.build(true); - Schema schema = schemaBuilder.getSchema(); - RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); - return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXGBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXGBoostTestCase.java deleted file mode 100644 index 0c163954038..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXGBoostTestCase.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.After; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author grace-lam - * @author bratseth - */ -public class RankingExpressionWithXGBoostTestCase { - - private final Path applicationDir = Path.fromString("src/test/integration/xgboost/"); - - private final static String vespaExpression = - "if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + " + - "if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)"; - - @After - public void removeGeneratedModelFiles() { - IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - } - - @Test - public void testXGBoostReference() { - RankProfileSearchFixture search = fixtureWith("xgboost('xgboost.2.2.json')"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - - @Test - public void testNestedXGBoostReference() { - RankProfileSearchFixture search = fixtureWith("5 + sum(xgboost('xgboost.2.2.json'))"); - search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); - } - - @Test - public void testImportingFromStoredExpressions() throws IOException { - RankProfileSearchFixture search = fixtureWith("xgboost('xgboost.2.2.json')"); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); - try { - storedApplicationDirectory.toFile().mkdirs(); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - RankingExpressionWithOnnxTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(storedApplicationDirectory); - RankProfileSearchFixture searchFromStored = fixtureWith("xgboost('xgboost.2.2.json')"); - searchFromStored.assertFirstPhaseExpression(vespaExpression, "my_profile"); - } - finally { - IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); - } - } - - private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { - return fixtureWith(firstPhaseExpression, null, null, - new RankingExpressionWithOnnxTestCase.StoringApplicationPackage(applicationDir)); - } - - private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, - String constant, - String field, - RankingExpressionWithOnnxTestCase.StoringApplicationPackage application) { - try { - RankProfileSearchFixture fixture = new RankProfileSearchFixture( - application, - application.getQueryProfiles(), - " rank-profile my_profile {\n" + - " first-phase {\n" + - " expression: " + firstPhaseExpression + - " }\n" + - " }", - constant, - field); - fixture.compileRankProfile("my_profile", applicationDir.append("models")); - return fixture; - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - -} - diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java deleted file mode 100644 index b42f932ef98..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.collections.Pair; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class RankingExpressionsTestCase extends AbstractSchemaTestCase { - - private static Schema createSearch(String dir, ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return ApplicationBuilder.createFromDirectory(dir, new MockFileRegistry(), new TestableDeployLogger(), deployProperties, rankProfileRegistry).getSchema(); - } - - @Test - public void testFunctions() throws IOException, ParseException { - ModelContext.Properties deployProperties = new TestProperties(); - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Schema schema = createSearch("src/test/examples/rankingexpressionfunction", deployProperties, rankProfileRegistry); - RankProfile functionsRankProfile = rankProfileRegistry.get(schema, "macros"); - Map functions = functionsRankProfile.getFunctions(); - assertEquals(2, functions.get("titlematch$").function().arguments().size()); - assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); - assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); - assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); - assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", - functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); - assertEquals("78 + closeness(distance)", - functions.get("artistmatch").function().getBody().getRoot().toString()); - assertEquals(0, functions.get("artistmatch").function().arguments().size()); - - RawRankProfile rawRankProfile = new RawRankProfile(functionsRankProfile, new LargeRankExpressions(new MockFileRegistry()), new QueryProfileRegistry(), - new ImportedMlModels(), new AttributeFields(schema), deployProperties); - List> rankProperties = rawRankProfile.configProperties(); - assertEquals(6, rankProperties.size()); - - assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); - - assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); - assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); - } - - @Test(expected = IllegalArgumentException.class) - public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { - RankProfileRegistry registry = new RankProfileRegistry(); - Schema schema = createSearch("src/test/examples/rankingexpressioninfile", new TestProperties(), registry); - new DerivedConfiguration(schema, registry); // rank profile parsing happens during deriving - } - - private void verifyProfile(RankProfile profile, List expectedFunctions, List> rankProperties, - LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels models, - AttributeFields attributes, ModelContext.Properties properties) { - var functions = profile.getFunctions(); - assertEquals(expectedFunctions.size(), functions.size()); - for (String func : expectedFunctions) { - assertTrue(functions.containsKey(func)); - } - - RawRankProfile raw = new RawRankProfile(profile, largeExpressions, queryProfiles, models, attributes, properties); - assertEquals(rankProperties.size(), raw.configProperties().size()); - for (int i = 0; i < rankProperties.size(); i++) { - assertEquals(rankProperties.get(i).getFirst(), raw.configProperties().get(i).getFirst()); - assertEquals(rankProperties.get(i).getSecond(), raw.configProperties().get(i).getSecond()); - } - } - - private void verifySearch(Schema schema, RankProfileRegistry rankProfileRegistry, LargeRankExpressions largeExpressions, - QueryProfileRegistry queryProfiles, ImportedMlModels models, ModelContext.Properties properties) - { - AttributeFields attributes = new AttributeFields(schema); - - verifyProfile(rankProfileRegistry.get(schema, "base"), Arrays.asList("large_f", "large_m"), - Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), - largeExpressions, queryProfiles, models, attributes, properties); - for (String child : Arrays.asList("child_a", "child_b")) { - verifyProfile(rankProfileRegistry.get(schema, child), Arrays.asList("large_f", "large_m", "large_local_f", "large_local_m"), - Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), - new Pair<>("rankingExpression(large_local_f).expressionName", child + ".large_local_f"), new Pair<>("rankingExpression(large_local_m).expressionName", child + ".large_local_m"), - new Pair<>("vespa.rank.firstphase", "rankingExpression(firstphase)"), new Pair<>("rankingExpression(firstphase).expressionName", child + ".firstphase")), - largeExpressions, queryProfiles, models, attributes, properties); - } - } - - @Test - public void testLargeInheritedFunctions() throws IOException, ParseException { - ModelContext.Properties properties = new TestProperties(); - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry(), 50); - QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); - ImportedMlModels models = new ImportedMlModels(); - Schema schema = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); - verifySearch(schema, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); - // Need to verify that second derivation works as that will happen if same sd is used in multiple content clusters - verifySearch(schema, rankProfileRegistry, largeExpressions, queryProfiles, models, properties); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java deleted file mode 100644 index 780e87b01f0..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * @author bjorncs - */ -public class ReferenceFieldTestCase { - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void reference_fields_are_parsed_from_search_definition() throws ParseException { - ApplicationBuilder builder = new ApplicationBuilder(); - String campaignSdContent = - "schema campaign {\n" + - " document campaign {\n" + - " }\n" + - "}"; - String salespersonSdContent = - "schema salesperson {\n" + - " document salesperson {\n" + - " }\n" + - "}"; - String adSdContent = - "schema ad {\n" + - " document ad {\n" + - " field campaign_ref type reference { indexing: attribute }\n" + - " field salesperson_ref type reference { indexing: attribute }\n" + - " }\n" + - "}"; - builder.addSchema(campaignSdContent); - builder.addSchema(salespersonSdContent); - builder.addSchema(adSdContent); - builder.build(true); - Schema schema = builder.getSchema("ad"); - assertSearchContainsReferenceField("campaign_ref", "campaign", schema.getDocument()); - assertSearchContainsReferenceField("salesperson_ref", "salesperson", schema.getDocument()); - } - - @Test - public void cyclic_document_dependencies_are_detected() throws ParseException { - var builder = new ApplicationBuilder(new TestProperties()); - String campaignSdContent = - "schema campaign {\n" + - " document campaign {\n" + - " field ad_ref type reference { indexing: attribute }\n" + - " }\n" + - "}"; - String adSdContent = - "schema ad {\n" + - " document ad {\n" + - " field campaign_ref type reference { indexing: attribute }\n" + - " }\n" + - "}"; - builder.addSchema(campaignSdContent); - builder.addSchema(adSdContent); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("reference cycle for documents"); - builder.build(true); - } - - private static void assertSearchContainsReferenceField(String expectedFieldname, - String referencedDocType, - SDDocumentType documentType) { - Field field = documentType.getDocumentType().getField(expectedFieldname); - assertNotNull("Field does not exist in document type: " + expectedFieldname, field); - DataType dataType = field.getDataType(); - assertTrue(dataType instanceof NewDocumentReferenceDataType); - NewDocumentReferenceDataType refField = (NewDocumentReferenceDataType) dataType; - assertEquals(referencedDocType, refField.getTargetTypeName()); - assertTrue(! refField.isTemporary()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java deleted file mode 100644 index 6d722138871..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author Simon Thoresen Hult - */ -public class ReservedDocumentNamesTestCase extends AbstractExportingTestCase { - - @Test - public void requireThatPositionIsAReservedDocumentName() throws IOException, ParseException { - try { - assertCorrectDeriving("reserved_position"); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'position': Document name 'position' is reserved.", e.getMessage()); - } - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java deleted file mode 100644 index d3d79fd7798..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.util.logging.Level; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author lesters - */ -public class ReservedRankingExpressionFunctionNamesTestCase { - - @Test - public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { - TestDeployLogger deployLogger = new TestDeployLogger(); - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(deployLogger, rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test_rank_profile {\n" + - " function not_a_reserved_name(x) {\n" + - " expression: x + x\n" + - " }\n" + - " function sigmoid(x) {\n" + - " expression: x * x\n" + - " }\n" + - " first-phase {\n" + - " expression: sigmoid(2) + not_a_reserved_name(1)\n" + - " }\n" + - " }\n" + - " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + - " function sin(x) {\n" + - " expression: x * x\n" + - " }\n" + - " first-phase {\n" + - " expression: sigmoid(2) + sin(1)\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - - assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile")); - assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile_2")); - assertTrue(deployLogger.log.contains("sin") && deployLogger.log.contains("test_rank_profile_2")); - assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile")); - assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile_2")); - - } - - public static class TestDeployLogger implements DeployLogger { - public String log = ""; - @Override - public void log(Level level, String message) { - log += message; - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java deleted file mode 100644 index 4cfe4c09052..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.fail; - -/** - * @author hmusum - */ -public class SchemaMustHaveDocumentTest { - - @Test - public void requireErrorWhenMissingDocument() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalid_sd_missing_document.sd"); - fail("SD without document"); - } catch (IllegalArgumentException e) { - if (!e.getMessage() - .contains("For schema 'imageconfig': A search specification must have an equally named document inside of it.")) { - throw e; - } - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java deleted file mode 100644 index f224dc0688e..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import org.junit.Test; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; - -public class SummaryConsistencyTestCase { - - @Test - public void attribute_combiner_transform_is_set_when_source_is_array_of_struct_with_only_struct_field_attributes() throws ParseException { - String sd = joinLines( - "search structmemorysummary {", - " document structmemorysummary {", - " struct elem {", - " field name type string {}", - " field weight type int {}\n", - " }", - " field elem_array type array {", - " indexing: summary", - " struct-field name {", - " indexing: attribute", - " }", - " struct-field weight {", - " indexing: attribute", - " }", - " }", - " }", - " document-summary unfiltered {", - " summary elem_array_unfiltered type array {", - " source: elem_array", - " }", - " }", - "", - "}" - ); - Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); - assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); - } -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java deleted file mode 100644 index 1eb5e66df93..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.BaseDeployLogger; - -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class SummaryFieldsMustHaveValidSourceTestCase extends AbstractSchemaTestCase { - - @Test - public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalidsummarysource.sd"); - fail("This should throw and never get here"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); - } - } - - @Test - public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalidimplicitsummarysource.sd"); - fail("This should throw and never get here"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); - } - } - - @Test - public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { - try { - ApplicationBuilder.buildFromFile("src/test/examples/invalidselfreferringsummary.sd"); - fail("This should throw and never get here"); - } catch (IllegalArgumentException e) { - assertEquals("For schema 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); - } - } - - @Test - public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/documentidinsummary.sd"); - BaseDeployLogger deployLogger = new BaseDeployLogger(); - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - new SummaryFieldsMustHaveValidSource(schema, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); - assertEquals("documentid", schema.getSummary("withid").getSummaryField("w").getSingleSource()); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java deleted file mode 100644 index b9e6bfe7dc9..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - - -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author geirst - */ -public class TensorFieldTestCase { - - @Test - public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { - try { - createFromString(getSd("field f1 type array {}")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead.", - e.getMessage()); - } - } - - @Test - public void requireThatTensorFieldCannotBeIndexField() throws ParseException { - try { - createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + - "Currently, only tensors with 1 indexed dimension supports that.", - e.getMessage()); - } - } - - @Test - public void requireThatIndexedTensorAttributeCannotBeFastSearch() throws ParseException { - try { - createFromString(getSd("field f1 type tensor(x[3]) { indexing: attribute \n attribute: fast-search }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'.", e.getMessage()); - } - } - - @Test - public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { - try { - createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertStartsWith("Field type: Illegal tensor type spec:", e.getMessage()); - } - } - - @Test - public void hnsw_index_is_default_turned_off() throws ParseException { - var attr = getAttributeFromSd("field t1 type tensor(x[64]) { indexing: attribute }", "t1"); - assertFalse(attr.hnswIndexParams().isPresent()); - } - - @Test - public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { - assertHnswIndexParams("", 16, 200); - assertHnswIndexParams("index: hnsw", 16, 200); - } - - @Test - public void hnsw_index_parameters_can_be_specified() throws ParseException { - assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); - assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); - assertHnswIndexParams(joinLines("index {", - " hnsw {", - " max-links-per-node: 32", - " neighbors-to-explore-at-insert: 300", - " }", - "}"), - 32, 300); - } - - @Test - public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { - try { - createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); - } - } - - @Test - public void tensor_with_hnsw_index_parameters_must_be_an_index() throws ParseException { - try { - createFromString(getSd(joinLines( - "field t1 type tensor(x[64]) {", - " indexing: attribute ", - " index {", - " hnsw { max-links-per-node: 32 }", - " }", - "}"))); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("For schema 'test', field 't1': " + - "A tensor that specifies hnsw index parameters must also specify 'index' in 'indexing'", - e.getMessage()); - } - } - - @Test - public void tensors_with_at_least_one_mapped_dimension_can_be_direct() throws ParseException { - assertTrue(getAttributeFromSd( - "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); - assertTrue(getAttributeFromSd( - "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch()); - } - - @Test - public void tensors_with_at_least_one_mapped_dimension_can_be_fast_rank() throws ParseException { - assertTrue(getAttributeFromSd( - "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank()); - assertTrue(getAttributeFromSd( - "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank()); - } - - private static String getSd(String field) { - return joinLines("search test {", - " document test {", - " " + field, - " }", - "}"); - } - - private Attribute getAttributeFromSd(String fieldSpec, String attrName) throws ParseException { - return createFromString(getSd(fieldSpec)).getSchema().getAttribute(attrName); - } - - private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { - var sd = getSdWithIndexSpec(indexSpec); - var search = createFromString(sd).getSchema(); - var attr = search.getAttribute("t1"); - var params = attr.hnswIndexParams(); - assertTrue(params.isPresent()); - assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); - assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); - } - - private String getSdWithIndexSpec(String indexSpec) { - return getSd(joinLines("field t1 type tensor(x[64]) {", - " indexing: attribute | index", - " " + indexSpec, - "}")); - } - - private void assertStartsWith(String prefix, String string) { - assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java deleted file mode 100644 index 26b108e2b45..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.collections.Pair; -import com.yahoo.component.ComponentId; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.FieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class TensorTransformTestCase extends AbstractSchemaTestCase { - - @Test - public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { - assertTransformedExpression("max(1.0,2.0)", - "max(1.0,2.0)"); - assertTransformedExpression("min(attribute(double_field),x)", - "min(attribute(double_field),x)"); - assertTransformedExpression("max(attribute(double_field),attribute(double_array_field))", - "max(attribute(double_field),attribute(double_array_field))"); - assertTransformedExpression("min(attribute(tensor_field_1),attribute(double_field))", - "min(attribute(tensor_field_1),attribute(double_field))"); - assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)", - "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)"); - assertTransformedExpression("min(constant(test_constant_tensor),1.0)", - "min(test_constant_tensor,1.0)"); - assertTransformedExpression("max(constant(base_constant_tensor),1.0)", - "max(base_constant_tensor,1.0)"); - assertTransformedExpression("min(constant(file_constant_tensor),1.0)", - "min(constant(file_constant_tensor),1.0)"); - assertTransformedExpression("max(query(q),1.0)", - "max(query(q),1.0)"); - assertTransformedExpression("max(query(n),1.0)", - "max(query(n),1.0)"); - } - - @Test - public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { - assertTransformedExpression("reduce(attribute(tensor_field_1),max,x)", - "max(attribute(tensor_field_1),x)"); - assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)", - "1 + max(attribute(tensor_field_1),x)"); - assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),reduce(attribute(tensor_field_1),sum,x))", - "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),reduce(attribute(tensor_field_1), sum, x))"); - assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)", - "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)"); - assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)", - "max(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),x)"); - assertTransformedExpression("max(reduce(attribute(tensor_field_1),max,x),x)", - "max(max(attribute(tensor_field_1),x),x)"); // will result in deploy error. - assertTransformedExpression("reduce(reduce(attribute(tensor_field_2),max,x),max,y)", - "max(max(attribute(tensor_field_2),x),y)"); - } - - @Test - public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(constant(test_constant_tensor),max,x)", - "max(test_constant_tensor,x)"); - assertTransformedExpression("reduce(constant(base_constant_tensor),max,x)", - "max(base_constant_tensor,x)"); - assertTransformedExpression("reduce(constant(file_constant_tensor),min,x)", - "min(constant(file_constant_tensor),x)"); - } - - @Test - public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(attribute(double_field)+attribute(tensor_field_1),min,x)", - "min(attribute(double_field) + attribute(tensor_field_1),x)"); - assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)", - "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)"); - assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)", - "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); - assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", - "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) - assertTransformedExpression("reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)", - "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); - } - - @Test - public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { - assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)", - "max(tensorFromLabels(attribute(double_array_field)),double_array_field)"); - assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field),x),max,x)", - "max(tensorFromLabels(attribute(double_array_field),x),x)"); - assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)", - "max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)"); - assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)", - "max(tensorFromWeightedSet(attribute(weightedset_field),x),x)"); - } - - @Test - public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { - assertTransformedExpression("reduce(query(q),max,x)", "max(query(q),x)"); - assertTransformedExpression("max(query(n),x)", "max(query(n),x)"); - } - - @Test - public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", - "max(returns_tensor,x)"); - assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", - "max(wraps_returns_tensor,x)"); - assertTransformedExpression("reduce(rankingExpression(tensor_inheriting),max,x)", - "max(tensor_inheriting,x)"); - assertTransformedExpression("reduce(rankingExpression(returns_tensor_with_arg@),max,x)", - "max(returns_tensor_with_arg(attribute(tensor_field_1)),x)"); - } - - private void assertTransformedExpression(String expected, String original) throws ParseException { - for (Pair rankPropertyExpression : buildSearch(original)) { - String rankProperty = rankPropertyExpression.getFirst(); - if (rankProperty.equals("rankingExpression(testexpression).rankingScript")) { - String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); - assertEquals(expected, rankExpression); - return; - } - } - fail("No 'rankingExpression(testexpression).rankingScript' property produced"); - } - - private List> buildSearch(String expression) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfiles); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field double_field type double { \n" + - " indexing: summary | attribute \n" + - " }\n" + - " field double_array_field type array { \n" + - " indexing: summary | attribute \n" + - " }\n" + - " field weightedset_field type weightedset { \n" + - " indexing: summary | attribute \n" + - " }\n" + - " field tensor_field_1 type tensor(x{}) { \n" + - " indexing: summary | attribute \n" + - " }\n" + - " field tensor_field_2 type tensor(x[3],y[3]) { \n" + - " indexing: summary | attribute \n" + - " }\n" + - " }\n" + - " constant file_constant_tensor {\n" + - " file: constants/tensor.json\n" + - " type: tensor(x{})\n" + - " }\n" + - " rank-profile base {\n" + - " constants {\n" + - " base_constant_tensor tensor(x[1]):[0]\n"+ - " }\n" + - " function base_tensor() {\n" + - " expression: constant(base_constant_tensor)\n" + - " }\n" + - " }\n" + - " rank-profile test inherits base {\n" + - " constants {\n" + - " test_constant_tensor tensor(x[1]):[1]" + - " }\n" + - " function returns_tensor_with_arg(arg1) {\n" + - " expression: 2.0 * arg1\n" + - " }\n" + - " function wraps_returns_tensor() {\n" + - " expression: returns_tensor\n" + - " }\n" + - " function returns_tensor() {\n" + - " expression: attribute(tensor_field_2)\n" + - " }\n" + - " function tensor_inheriting() {\n" + - " expression: base_tensor\n" + - " }\n" + - " function testexpression() {\n" + - " expression: " + expression + "\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(queryProfiles, new ImportedMlModels()); - List> testRankProperties = new RawRankProfile(test, - new LargeRankExpressions(new MockFileRegistry()), - queryProfiles, - new ImportedMlModels(), - new AttributeFields(s), new TestProperties()).configProperties(); - return testRankProperties; - } - - private static QueryProfileRegistry setupQueryProfileTypes() { - QueryProfileRegistry registry = new QueryProfileRegistry(); - QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); - QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); - type.addField(new FieldDescription("ranking.features.query(q)", - FieldType.fromString("tensor(x{})", typeRegistry)), typeRegistry); - type.addField(new FieldDescription("ranking.features.query(n)", - FieldType.fromString("integer", typeRegistry)), typeRegistry); - typeRegistry.register(type); - return registry; - } - - private String censorBindingHash(String s) { - StringBuilder b = new StringBuilder(); - boolean areInHash = false; - for (int i = 0; i < s.length() ; i++) { - char current = s.charAt(i); - if ( ! Character.isLetterOrDigit(current)) // end of hash - areInHash = false; - if ( ! areInHash) - b.append(current); - if (current == '@') // start of hash - areInHash = true; - } - return b.toString(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java deleted file mode 100644 index 8c801d9deaf..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.ImportedSimpleField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.Collections; - -/** - * @author bjorncs - */ -public class ValidateFieldTypesTest { - - private static final String IMPORTED_FIELD_NAME = "imported_myfield"; - private static final String DOCUMENT_NAME = "my_doc"; - - @SuppressWarnings("deprecation") - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void throws_exception_if_type_of_document_field_does_not_match_summary_field() { - Schema schema = createSearchWithDocument(DOCUMENT_NAME); - schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME, DataType.INT)); - schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, DataType.STRING, schema)); - - ValidateFieldTypes validator = new ValidateFieldTypes(schema, null, null, null); - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "For schema '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + - "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); - validator.process(true, false); - } - - private static Schema createSearch(String documentType) { - return new Schema(documentType, - MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - new TestableDeployLogger(), - new TestProperties()); - } - - private static Schema createSearchWithDocument(String documentName) { - Schema schema = createSearch(documentName); - SDDocumentType document = new SDDocumentType(documentName, schema); - schema.addDocument(document); - return schema; - } - - private static ImportedFields createSingleImportedField(String fieldName, DataType dataType) { - Schema targetSchema = createSearchWithDocument("target_doc"); - SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); - DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); - ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); - return new ImportedFields(Collections.singletonMap(fieldName, importedField)); - } - - private static DocumentSummary createDocumentSummary(String fieldName, DataType dataType, Schema schema) { - DocumentSummary summary = new DocumentSummary("mysummary", schema); - summary.add(new SummaryField(fieldName, dataType)); - return summary; - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java deleted file mode 100644 index 4446f01aa95..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.ml.ImportedModelTester; -import org.junit.After; -import org.junit.Test; - -import java.io.IOException; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -/** - * Tests adding Vespa ranking expression based models in the models/ dir - * - * @author bratseth - */ -public class VespaMlModelTestCase { - - private final Path applicationDir = Path.fromString("src/test/integration/vespa/"); - - private final String expectedRankConfig = - "constant(constant1).type : tensor(x[3])\n" + - "constant(constant1).value : tensor(x[3]):[0.5, 1.5, 2.5]\n" + - "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" + - "rankingExpression(foo1).input2.type : tensor(x[3])\n" + - "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" + - "rankingExpression(foo2).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1asLarge), max, x) * 3.0\n" + - "rankingExpression(foo2).input2.type : tensor(x[3])\n" + - "rankingExpression(foo2).input1.type : tensor(name{},x[3])\n"; - - /** The model name */ - private final String name = "example"; - - @After - public void removeGeneratedModelFiles() { - IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - } - - @Test - public void testGlobalVespaModel() throws IOException { - ImportedModelTester tester = new ImportedModelTester(name, applicationDir); - VespaModel model = tester.createVespaModel(); - tester.assertLargeConstant("constant1asLarge", model, Optional.of(3L)); - assertEquals(expectedRankConfig, rankConfigOf("example", model)); - - // At this point the expression is stored - copy application to another location which do not have a models dir - Path storedAppDir = applicationDir.append("copy"); - try { - storedAppDir.toFile().mkdirs(); - IOUtils.copy(applicationDir.append("services.xml").toString(), storedAppDir.append("services.xml").toString()); - IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), - storedAppDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); - ImportedModelTester storedTester = new ImportedModelTester(name, storedAppDir); - VespaModel storedModel = storedTester.createVespaModel(); - storedTester.assertLargeConstant("constant1asLarge", model, Optional.of(3L)); - assertEquals(expectedRankConfig, rankConfigOf("example", storedModel)); - } - finally { - IOUtils.recursiveDeleteDir(storedAppDir.toFile()); - } - } - - private String rankConfigOf(String rankProfileName, VespaModel model) { - StringBuilder b = new StringBuilder(); - RawRankProfile profile = model.rankProfileList().getRankProfiles().get(rankProfileName); - for (var property : profile.configProperties()) - b.append(property.getFirst()).append(" : ").append(property.getSecond()).append("\n"); - return b.toString(); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java deleted file mode 100644 index e56d6ccf343..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.assertNotNull; - -/** @author bratseth */ -public class WeightedSetSummaryToTestCase extends AbstractSchemaTestCase { - - @Test - public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { - Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/weightedset-summaryto.sd"); - assertNotNull(schema); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java index c0d74cd02d3..768bf408250 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; +import com.yahoo.schema.AbstractSchemaTestCase; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java index 33d84cf4313..faf4b67430c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index ebb4156f3c0..3c2193ca4df 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index dc88590a198..d345c776db3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 86385b2f80d..859194fae79 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -8,7 +8,7 @@ import com.yahoo.document.Field; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.FieldSets; +import com.yahoo.schema.FieldSets; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 977c10d0828..0d51e4b6e74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java index ab0b2f0d92c..e73eb905566 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java @@ -8,14 +8,13 @@ import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.QueryProfileVariant; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; +import com.yahoo.schema.derived.TestableDeployLogger; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index 52923779602..1d15f973d21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -6,11 +6,11 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.container.QrSearchersConfig; import com.yahoo.document.DataType; import com.yahoo.search.config.ClusterConfig; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index c68560d16b7..189949ceca9 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; -import com.yahoo.compress.CompressionType; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.Processing; import com.yahoo.docproc.proxy.ProxyDocument; @@ -40,7 +39,7 @@ import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.tensor.Tensor; import com.yahoo.vespa.document.NodeImpl; import com.yahoo.vespa.document.dom.DocumentImpl; diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 0b0e7c5b647..1fcb639589f 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -18,10 +18,10 @@ import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.parser.ParseException; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index 3879067be2e..c619a1ff56b 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa; import com.yahoo.document.DataType; import com.yahoo.document.StructDataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import org.junit.Test; import java.io.File; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index eaed95b97e1..9f3c536a59a 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -47,7 +47,7 @@ import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.TraceNode; import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.slime.Inspector; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.SlimeUtils; -- cgit v1.2.3