From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- .../yahoo/component/chain/dependencies/.gitignore | 0 .../com/yahoo/component/provider/test/.gitignore | 0 .../test/java/com/yahoo/component/test/.gitignore | 0 .../yahoo/container/core/config/basic/.gitignore | 0 .../container/core/config/configurable/.gitignore | 0 .../com/yahoo/container/http/fileserver/.gitignore | 0 .../com/yahoo/container/logging/test/.gitignore | 0 .../com/yahoo/container/test/ConstantFile.java | 6 + .../java/com/yahoo/container/test/globalpackages | 3 + .../container/test/globalpackageservice/.gitignore | 0 .../test/httpheaders/container-config/.gitignore | 0 .../test/logging/container-config/.gitignore | 0 .../test/methods/container-config/.gitignore | 0 .../test/minimal/container-config/.gitignore | 0 .../com/yahoo/fs4/PacketQueryTracerTestCase.java | 113 + .../java/com/yahoo/fs4/mplex/BackendTestCase.java | 224 ++ .../java/com/yahoo/fs4/test/FastHitTestCase.java | 27 + .../yahoo/fs4/test/GetDocSumsPacketTestCase.java | 116 + .../yahoo/fs4/test/HexByteIteratorTestCase.java | 37 + .../com/yahoo/fs4/test/PacketDecoderTestCase.java | 184 ++ .../java/com/yahoo/fs4/test/PacketTestCase.java | 225 ++ .../com/yahoo/fs4/test/QueryResultTestCase.java | 202 ++ .../java/com/yahoo/fs4/test/QueryTestCase.java | 294 +++ .../com/yahoo/fs4/test/RankFeaturesTestCase.java | 115 + .../test/java/com/yahoo/osgi/test/Calculator.java | 13 + .../test/calculatorservice/CalculatorService.java | 31 + .../yahoo/osgi/test/calculatorservice/Manifest.MF | 13 + .../java/com/yahoo/osgi/test/client/Client.java | 33 + .../java/com/yahoo/osgi/test/client/Manifest.MF | 12 + .../osgi/test/counterservice/CounterService.java | 15 + .../test/counterservice/CounterServiceImpl.java | 35 + .../com/yahoo/osgi/test/counterservice/Manifest.MF | 13 + .../java/com/yahoo/prelude/IndexFactsFactory.java | 31 + .../yahoo/prelude/cache/test/CacheTestCase.java | 171 ++ .../prelude/cluster/ClusterSearcherTestCase.java | 593 +++++ .../yahoo/prelude/cluster/test/HasherTestCase.java | 128 + .../prelude/fastsearch/DocsumFieldTestCase.java | 67 + .../yahoo/prelude/fastsearch/FieldsTestCase.java | 322 +++ .../prelude/fastsearch/JsonFieldTestCase.java | 97 + .../prelude/fastsearch/SlimeSummaryTestCase.java | 172 ++ .../java/com/yahoo/prelude/fastsearch/summary.cfg | 30 + .../prelude/fastsearch/test/CacheKeyTestCase.java | 29 + .../prelude/fastsearch/test/DispatchThread.java | 101 + .../fastsearch/test/DocsumDefinitionTestCase.java | 394 +++ .../fastsearch/test/FastSearcherTestCase.java | 633 +++++ .../prelude/fastsearch/test/MockFDispatch.java | 211 ++ .../fastsearch/test/PacketCacheTestCase.java | 181 ++ .../fastsearch/test/PacketWrapperTestCase.java | 408 ++++ .../fastsearch/test/PartialFillTestCase.java | 164 ++ .../yahoo/prelude/fastsearch/test/category.enum | 20 + .../prelude/fastsearch/test/documentdb-info.cfg | 351 +++ .../fastsearch/test/updated-qr-summary-dummy.cfg | 15 + .../yahoo/prelude/grouping/legacy/test/.gitignore | 0 .../prelude/hitfield/XmlRendererTestCase.java | 76 + .../prelude/hitfield/test/HitFieldTestCase.java | 78 + .../prelude/hitfield/test/JSONStringTestCase.java | 862 +++++++ .../hitfield/test/TokenFieldIteratorTestCase.java | 90 + .../yahoo/prelude/query/ItemHelperTestCase.java | 67 + .../com/yahoo/prelude/query/ItemLabelTestCase.java | 86 + .../prelude/query/ItemsCommonStuffTestCase.java | 398 ++++ .../yahoo/prelude/query/TaggableItemsTestCase.java | 158 ++ .../query/WordAlternativesItemTestCase.java | 74 + .../prelude/query/parser/TestLinguistics.java | 70 + .../yahoo/prelude/query/parser/TestSegmenter.java | 57 + .../query/parser/UnicodePropertyDumpTestCase.java | 46 + .../test/ExactMatchAndDefaultIndexTestCase.java | 54 + .../prelude/query/parser/test/ParseTestCase.java | 2507 ++++++++++++++++++++ .../prelude/query/parser/test/ParsingTester.java | 139 ++ .../query/parser/test/SubstringTestCase.java | 48 + .../query/parser/test/TokenizerTestCase.java | 765 ++++++ .../query/parser/test/WashPhrasesTestCase.java | 101 + .../prelude/query/parser/test/parseindexinfo.cfg | 111 + .../prelude/query/parser/test/replacingtokens.cfg | 12 + .../prelude/query/parser/test/specialtokens.cfg | 15 + .../prelude/query/test/DotProductItemTestCase.java | 32 + .../yahoo/prelude/query/test/IntItemTestCase.java | 27 + .../prelude/query/test/ItemEncodingTestCase.java | 319 +++ .../prelude/query/test/PhraseItemTestCase.java | 100 + .../query/test/PredicateQueryItemTestCase.java | 136 ++ .../test/QueryCanonicalizerMicroBenchmark.java | 44 + .../query/test/QueryCanonicalizerTestCase.java | 329 +++ .../prelude/query/test/QueryLanguageTestCase.java | 106 + .../yahoo/prelude/query/test/QueryTestCase.java | 70 + .../prelude/query/test/RangeItemTestCase.java | 39 + .../prelude/query/test/SegmentItemTestCase.java | 31 + .../yahoo/prelude/query/test/WandItemTestCase.java | 85 + .../query/test/WeightedSetItemTestCase.java | 111 + .../test/TextualQueryRepresentationTestCase.java | 121 + .../query/textualrepresentation/test/basic.txt | 3 + .../query/textualrepresentation/test/composite.txt | 8 + .../querytransform/test/CJKSearcherTestCase.java | 72 + .../test/CollapsePhraseSearcherTestCase.java | 119 + .../test/IndexCombinatorTestCase.java | 165 ++ .../test/LiteralBoostSearcherTestCase.java | 112 + .../test/NoRankingSearcherTestCase.java | 39 + .../test/NonPhrasingSearcherTestCase.java | 75 + .../test/NormalizingSearcherTestCase.java | 165 ++ .../querytransform/test/PhraseMatcherTestCase.java | 252 ++ .../test/PhrasingSearcherTestCase.java | 178 ++ .../querytransform/test/QueryRewriteTestCase.java | 132 ++ .../test/RecallSearcherTestCase.java | 98 + .../test/StemmingSearcherTestCase.java | 156 ++ .../test/accent-removal-index-info.cfg | 47 + .../prelude/querytransform/test/cjk-index-info.cfg | 19 + .../prelude/querytransform/test/container-http.cfg | 3 + .../prelude/querytransform/test/emptyindexinfo.cfg | 2 + .../prelude/querytransform/test/index-info.cfg | 47 + .../querytransform/test/indexcombinator.cfg | 29 + .../test/proximity-phrases-input.txt | 5 + .../querytransform/test/proximity-phrases.fsa | Bin 0 -> 1852 bytes .../test/proximity-segments-input.txt | 6 + .../querytransform/test/proximity-segments.fsa | Bin 0 -> 1916 bytes .../test/proximity-stop-words-input.txt | 9 + .../querytransform/test/proximity-stop-words.fsa | Bin 0 -> 1762 bytes .../prelude/querytransform/test/test-fsa-input.txt | 4 + .../yahoo/prelude/querytransform/test/test-fsa.fsa | Bin 0 -> 1757 bytes .../querytransform/test/testindexinfonoboost.cfg | 15 + .../searcher/test/BlendingSearcherTestCase.java | 480 ++++ .../searcher/test/CachingSearcherTestCase.java | 96 + .../searcher/test/ErrorHitRenderTestCase.java | 33 + .../test/FieldCollapsingSearcherTestCase.java | 479 ++++ .../searcher/test/JSONDebugSearcherTestCase.java | 91 + .../searcher/test/JuniperSearcherTestCase.java | 263 ++ .../searcher/test/KeyValueSearcherTest.java | 184 ++ .../searcher/test/MultipleResultsTestCase.java | 142 ++ .../prelude/searcher/test/PosSearcherTestCase.java | 192 ++ .../test/QuerySnapshotSearcherTestCase.java | 49 + .../test/QueryValidatingSearcherTestCase.java | 80 + .../searcher/test/QuotingSearcherTestCase.java | 140 ++ .../test/ValidatePredicateSearcherTestCase.java | 66 + .../test/ValidateSortingSearcherTestCase.java | 120 + .../yahoo/prelude/searcher/test/qr-searchers.cfg | 21 + .../com/yahoo/prelude/searcher/test/qr-summary.cfg | 349 +++ .../searcher/test/testdynteaserfieldinfo.cfg | 11 + .../prelude/searcher/test/testdynteaserquoting.cfg | 16 + .../yahoo/prelude/searcher/test/testfieldinfo.cfg | 19 + .../com/yahoo/prelude/searcher/test/testhit.xml | 29 + .../yahoo/prelude/searcher/test/testindexinfo.cfg | 28 + .../prelude/searcher/test/testlazymapping.cfg | 5 + .../prelude/searcher/test/testphysicalmapping.cfg | 98 + .../yahoo/prelude/searcher/test/testquoting.cfg | 10 + .../prelude/searcher/test/validate_sorting.cfg | 17 + .../semantics/compatibility/test/.gitignore | 0 .../config/test/RuleConfigDeriverTestCase.java | 77 + .../parser/test/SemanticsParserTestCase.java | 59 + .../yahoo/prelude/semantics/parser/test/rules.sr | 32 + .../prelude/semantics/parser/test/semantics.fsa | Bin 0 -> 2539 bytes .../prelude/semantics/test/AlibabaTestCase.java | 31 + .../prelude/semantics/test/AnchorTestCase.java | 51 + .../semantics/test/AutomataNotTestCase.java | 22 + .../prelude/semantics/test/AutomataTestCase.java | 71 + .../semantics/test/BacktrackingTestCase.java | 103 + .../prelude/semantics/test/BlendingTestCase.java | 35 + .../yahoo/prelude/semantics/test/CJKTestCase.java | 25 + .../prelude/semantics/test/ComparisonTestCase.java | 41 + .../semantics/test/ComparisonsTestCase.java | 20 + .../prelude/semantics/test/ConditionTestCase.java | 78 + .../semantics/test/ConfigurationTestCase.java | 133 ++ .../semantics/test/DuplicateRuleTestCase.java | 31 + .../prelude/semantics/test/Ellipsis2TestCase.java | 19 + .../prelude/semantics/test/EllipsisTestCase.java | 42 + .../prelude/semantics/test/ExactMatchTestCase.java | 26 + .../semantics/test/ExactMatchTrickTestCase.java | 30 + .../semantics/test/InheritanceTestCase.java | 168 ++ .../semantics/test/LabelMatchingTestCase.java | 44 + .../prelude/semantics/test/MatchAllTestCase.java | 27 + .../test/MatchOnlyIfNotOnlyTermTestCase.java | 27 + .../prelude/semantics/test/NoStemmingTestCase.java | 30 + .../yahoo/prelude/semantics/test/NotTestCase.java | 20 + .../prelude/semantics/test/NumbersTestCase.java | 25 + .../semantics/test/NumericTermsTestCase.java | 23 + .../prelude/semantics/test/OrPhraseTestCase.java | 22 + .../prelude/semantics/test/Parameter2TestCase.java | 30 + .../prelude/semantics/test/ParameterTestCase.java | 77 + .../semantics/test/PhraseMatchTestCase.java | 21 + .../semantics/test/ProductionRuleTestCase.java | 55 + .../semantics/test/RuleBaseAbstractTestCase.java | 84 + .../test/SegmentSubstitutionTestCase.java | 55 + .../semantics/test/SemanticSearcherTestCase.java | 164 ++ .../prelude/semantics/test/StemmingTestCase.java | 31 + .../prelude/semantics/test/StopwordTestCase.java | 28 + .../yahoo/prelude/semantics/test/UrlTestCase.java | 20 + .../prelude/semantics/test/WeightingTestCase.java | 22 + .../prelude/semantics/test/rulebases/alibaba.sr | 5 + .../prelude/semantics/test/rulebases/anchor.sr | 9 + .../semantics/test/rulebases/automatanot.sr | 3 + .../semantics/test/rulebases/automatarules.sr | 17 + .../semantics/test/rulebases/backtrackingrules.sr | 28 + .../prelude/semantics/test/rulebases/blending.sr | 8 + .../prelude/semantics/test/rulebases/catchoose.fsa | Bin 0 -> 1847 bytes .../prelude/semantics/test/rulebases/catchoose.txt | 4 + .../prelude/semantics/test/rulebases/cjk-rules.cfg | 6 + .../yahoo/prelude/semantics/test/rulebases/cjk.sr | 19 + .../prelude/semantics/test/rulebases/comparison.sr | 14 + .../semantics/test/rulebases/comparisons.sr | 3 + .../semantics/test/rulebases/duplicaterules.sr | 8 + .../prelude/semantics/test/rulebases/ellipsis.sr | 36 + .../prelude/semantics/test/rulebases/ellipsis2.sr | 2 + .../prelude/semantics/test/rulebases/empty.sr | 0 .../prelude/semantics/test/rulebases/exactmatch.sr | 6 + .../semantics/test/rulebases/exactmatchtrick.sr | 6 + .../test/rulebases/inheritingrules/child1.sr | 7 + .../test/rulebases/inheritingrules/child2.sr | 8 + .../test/rulebases/inheritingrules/cjk.sr | 3 + .../test/rulebases/inheritingrules/grandchild.sr | 5 + .../test/rulebases/inheritingrules/grandfather.sr | 4 + .../test/rulebases/inheritingrules/grandmother.sr | 2 + .../test/rulebases/inheritingrules/parent.sr | 8 + .../semantics/test/rulebases/labelmatching.sr | 11 + .../test/rulebases/match-only-if-not-only-term.sr | 4 + .../prelude/semantics/test/rulebases/matchall.sr | 2 + .../prelude/semantics/test/rulebases/nostemming.sr | 4 + .../yahoo/prelude/semantics/test/rulebases/not.sr | 2 + .../prelude/semantics/test/rulebases/numbers.sr | 12 + .../semantics/test/rulebases/numericterms.sr | 5 + .../prelude/semantics/test/rulebases/orphrase.sr | 9 + .../prelude/semantics/test/rulebases/parameter.sr | 17 + .../prelude/semantics/test/rulebases/parameter2.sr | 2 + .../prelude/semantics/test/rulebases/parameter3.sr | 0 .../semantics/test/rulebases/phrasematch.sr | 5 + .../prelude/semantics/test/rulebases/rules.sr | 71 + .../semantics/test/rulebases/semantic-rules.cfg | 15 + .../prelude/semantics/test/rulebases/semantics.fsa | Bin 0 -> 2703 bytes .../prelude/semantics/test/rulebases/semantics.txt | 5 + .../prelude/semantics/test/rulebases/stemming.sr | 5 + .../prelude/semantics/test/rulebases/stopwords.sr | 8 + .../semantics/test/rulebases/substitution.sr | 2 + .../yahoo/prelude/semantics/test/rulebases/url.sr | 3 + .../prelude/semantics/test/rulebases/weighting.sr | 6 + .../yahoo/prelude/templates/test/BoomTemplate.java | 51 + .../templates/test/GroupedResultTestCase.java | 71 + .../prelude/templates/test/HitContextTestCase.java | 26 + .../prelude/templates/test/TemplateTestCase.java | 52 + .../yahoo/prelude/templates/test/TestTemplate.java | 53 + .../prelude/templates/test/TilingTestCase.java | 307 +++ .../yahoo/prelude/templates/test/qr-templates.cfg | 104 + .../templates/test/templates/asearch/error.templ | 1 + .../templates/test/templates/asearch/footer.templ | 1 + .../templates/test/templates/asearch/header.templ | 1 + .../templates/test/templates/asearch/hit.templ | 1 + .../templates/test/templates/asearch/nohits.templ | 1 + .../test/templates/cgi-bin/asearch/error.templ | 1 + .../test/templates/cgi-bin/asearch/footer.templ | 1 + .../test/templates/cgi-bin/asearch/header.templ | 1 + .../test/templates/cgi-bin/asearch/hit.templ | 1 + .../test/templates/cgi-bin/asearch/nohits.templ | 1 + .../prelude/templates/test/templates/templaterc | 5 + .../templates/test/templates/xsearch/error.templ | 1 + .../templates/test/templates/xsearch/footer.templ | 1 + .../templates/test/templates/xsearch/header.templ | 2 + .../templates/test/templates/xsearch/hit.templ | 5 + .../templates/test/templates/xsearch/nohits.templ | 1 + .../yahoo/prelude/templates/test/tilingexample.xml | 65 + .../prelude/templates/test/tilingexample2.xml | 23 + .../java/com/yahoo/prelude/test/DummySearcher.java | 30 + .../com/yahoo/prelude/test/GetRawWordTestCase.java | 41 + .../com/yahoo/prelude/test/IndexFactsTestCase.java | 277 +++ .../yahoo/prelude/test/IntegrationTestCase.java | 176 ++ .../com/yahoo/prelude/test/LocationTestCase.java | 30 + .../yahoo/prelude/test/NullSetMemberTestCase.java | 23 + .../java/com/yahoo/prelude/test/QueryTestCase.java | 398 ++++ .../prelude/test/RankFeatureDumpTestCase.java | 69 + .../com/yahoo/prelude/test/ResultTestCase.java | 104 + .../yahoo/prelude/test/fieldtypes/field-info.cfg | 21 + .../java/com/yahoo/prelude/test/index-info.cfg | 14 + .../com/yahoo/prelude/test/indexfactstesting.cfg | 29 + .../prelude/test/integration/FirstSearcher.java | 15 + .../prelude/test/integration/SecondSearcher.java | 15 + .../prelude/test/integration/ThirdSearcher.java | 15 + .../prelude/test/integration/qr-searchers.cfg | 8 + .../java/com/yahoo/prelude/test/integration/qr.cfg | 4 + .../java/com/yahoo/prelude/test/qr-fileserver.cfg | 1 + .../java/com/yahoo/prelude/test/qr-logging.cfg | 44 + .../java/com/yahoo/prelude/test/qr-searchers.cfg | 14 + .../java/com/yahoo/prelude/test/qr-summary.cfg | 69 + .../src/test/java/com/yahoo/prelude/test/qr.cfg | 1 + .../java/com/yahoo/prelude/test/specialtokens.cfg | 14 + .../java/com/yahoo/prelude/test/statistics.cfg | 0 .../search/StupidSingleThreadedHttpServer.java | 166 ++ .../cluster/test/ClusterSearcherTestCase.java | 169 ++ .../cluster/test/ClusteredConnectionTestCase.java | 198 ++ .../SearchChainTextRepresentationTestCase.java | 51 + .../com/yahoo/search/dispatch/FillTestCase.java | 91 + .../java/com/yahoo/search/dispatch/MockClient.java | 121 + .../yahoo/search/federation/FutureWaiterTest.java | 109 + .../http/GzipDecompressingEntityTestCase.java | 212 ++ .../search/federation/http/HttpParametersTest.java | 238 ++ .../search/federation/http/HttpPostTestCase.java | 99 + .../yahoo/search/federation/http/HttpTestCase.java | 117 + .../yahoo/search/federation/http/PingTestCase.java | 278 +++ .../federation/http/QueryParametersTestCase.java | 65 + .../com/yahoo/search/federation/image/.gitignore | 0 .../test/SearchChainResolverTestCase.java | 152 ++ .../sourceref/test/SourceRefResolverTestCase.java | 114 + .../test/AddHitsWithRelevanceSearcher.java | 37 + .../search/federation/test/BlockingSearcher.java | 22 + .../federation/test/FederationSearcherTest.java | 306 +++ .../test/FederationSearcherTestCase.java | 411 ++++ .../search/federation/test/FederationTester.java | 75 + .../search/federation/test/HitCountTestCase.java | 135 ++ .../federation/test/SetHitCountsSearcher.java | 39 + .../vespa/test/QueryMarshallerTestCase.java | 160 ++ .../vespa/test/QueryParametersTestCase.java | 40 + .../vespa/test/ResultBuilderTestCase.java | 91 + .../vespa/test/VespaIntegrationTestCase.java | 25 + .../vespa/test/VespaSearcherTestCase.java | 229 ++ .../yahoo/search/federation/vespa/test/idhits.xml | 23 + .../search/federation/vespa/test/nestedhits.xml | 318 +++ .../com/yahoo/search/federation/ysm/.gitignore | 0 .../search/grouping/ContinuationTestCase.java | 22 + .../grouping/GroupingQueryParserTestCase.java | 110 + .../search/grouping/GroupingRequestTestCase.java | 136 ++ .../search/grouping/GroupingValidatorTestCase.java | 73 + .../grouping/UniqueGroupingSearcherTestCase.java | 219 ++ .../grouping/request/BucketResolverTestCase.java | 212 ++ .../request/ExpressionVisitorTestCase.java | 82 + .../request/GroupingOperationTestCase.java | 148 ++ .../grouping/request/MathFunctionsTestCase.java | 67 + .../grouping/request/MathResolverTestCase.java | 133 ++ .../search/grouping/request/RawBufferTestCase.java | 56 + .../search/grouping/request/RequestTestCase.java | 229 ++ .../parser/GroupingParserBenchmarkTest.java | 270 +++ .../request/parser/GroupingParserTestCase.java | 619 +++++ .../search/grouping/result/GroupIdTestCase.java | 53 + .../search/grouping/result/GroupListTestCase.java | 35 + .../search/grouping/result/GroupTestCase.java | 31 + .../search/grouping/result/HitListTestCase.java | 35 + .../grouping/result/HitRendererTestCase.java | 174 ++ .../vespa/CompositeContinuationTestCase.java | 116 + .../grouping/vespa/GroupingExecutorTestCase.java | 765 ++++++ .../grouping/vespa/GroupingTransformTestCase.java | 227 ++ .../grouping/vespa/HitConverterTestCase.java | 138 ++ .../grouping/vespa/IntegerDecoderTestCase.java | 53 + .../grouping/vespa/IntegerEncoderTestCase.java | 35 + .../grouping/vespa/OffsetContinuationTestCase.java | 92 + .../grouping/vespa/RequestBuilderTestCase.java | 885 +++++++ .../grouping/vespa/ResultBuilderTestCase.java | 1108 +++++++++ .../search/grouping/vespa/ResultIdTestCase.java | 71 + .../search/handler/test/SearchHandlerTestCase.java | 516 ++++ .../yahoo/search/handler/test/config/.gitignore | 0 .../yahoo/search/handler/test/config/chains.cfg | 14 + .../config/config_invalid_param/query-profiles.cfg | 5 + .../handler/test/config/config_yql/chains.cfg | 6 + .../yahoo/search/handler/test/config/handlers.cfg | 8 + .../handler/test/config/handlers2/chains.cfg | 10 + .../test/config/handlersInvalid/handlers.cfg | 3 + .../search/handler/test/config/index-info.cfg | 0 .../yahoo/search/handler/test/config/qr-search.cfg | 0 .../search/handler/test/config/qr-searchers.cfg | 4 + .../search/handler/test/config/specialtokens.cfg | 1 + .../yahoo/search/match/test/DocumentDbTest.java | 51 + .../test/MapPageTemplateXMLReadingTestCase.java | 48 + .../test/PageTemplateXMLReadingTestCase.java | 279 +++ .../config/test/examples/choiceFooter.xml | 6 + .../config/test/examples/choiceHeader.xml | 10 + .../pagetemplates/config/test/examples/footer.xml | 5 + .../pagetemplates/config/test/examples/generic.xml | 5 + .../pagetemplates/config/test/examples/header.xml | 7 + .../config/test/examples/includer.xml | 36 + .../test/examples/invalidfilename/invalid.xml | 4 + .../config/test/examples/mapexamples/map1.xml | 21 + .../config/test/examples/richSerp.xml | 17 + .../config/test/examples/richerSerp.xml | 45 + .../pagetemplates/config/test/examples/serp.xml | 5 + .../config/test/examples/slottingSerp.xml | 5 + .../search/pagetemplates/engine/test/AnySource.xml | 4 + .../pagetemplates/engine/test/AnySourceResult.xml | 41 + .../engine/test/AnySourceTestCase.java | 59 + .../engine/test/ChoiceOfRenderers.xml | 17 + .../engine/test/ChoiceOfRenderersResult.xml | 36 + .../engine/test/ChoiceOfRenderersTestCase.java | 51 + .../engine/test/ChoiceOfSubsections.xml | 20 + .../engine/test/ChoiceOfSubsectionsResult.xml | 29 + .../engine/test/ChoiceOfSubsectionsTestCase.java | 68 + .../engine/test/ChoiceOfTwoSources.xml | 7 + .../engine/test/ChoiceOfTwoSourcesResult.xml | 17 + .../engine/test/ChoiceOfTwoSourcesTestCase.java | 51 + .../search/pagetemplates/engine/test/Choices.xml | 45 + .../pagetemplates/engine/test/ChoicesResult.xml | 55 + .../pagetemplates/engine/test/ChoicesTestCase.java | 50 + .../engine/test/ExecutionAbstractTestCase.java | 74 + .../engine/test/MapSectionsToSections.xml | 28 + .../engine/test/MapSectionsToSectionsResult.xml | 96 + .../engine/test/MapSectionsToSectionsTestCase.java | 90 + .../engine/test/MapSourcesToSections.xml | 22 + .../engine/test/MapSourcesToSectionsResult.xml | 73 + .../engine/test/MapSourcesToSectionsTestCase.java | 88 + .../search/pagetemplates/engine/test/Page.xml | 31 + .../pagetemplates/engine/test/PageResult.xml | 43 + .../pagetemplates/engine/test/PageTestCase.java | 44 + .../pagetemplates/engine/test/PageWithBlending.xml | 37 + .../engine/test/PageWithBlendingResult.xml | 45 + .../engine/test/PageWithBlendingTestCase.java | 44 + .../engine/test/PageWithSourceRenderer.xml | 36 + .../engine/test/PageWithSourceRendererResult.xml | 43 + .../test/PageWithSourceRendererTestCase.java | 44 + .../pagetemplates/engine/test/SourceChoice.xml | 7 + .../engine/test/SourceChoiceResult.xml | 17 + .../engine/test/SourceChoiceTestCase.java | 43 + .../engine/test/TwoSectionsFourSources.xml | 5 + .../engine/test/TwoSectionsFourSourcesResult.xml | 65 + .../test/TwoSectionsFourSourcesTestCase.java | 140 ++ .../test/PageTemplateSearcherTestCase.java | 220 ++ .../search/pagetemplates/test/SourceParameters.xml | 16 + .../test/SourceParametersTestCase.java | 54 + .../com/yahoo/search/query/SortingTestCase.java | 88 + .../context/test/ConcurrentTraceTestCase.java | 56 + .../search/query/context/test/LoggingTestCase.java | 59 + .../query/context/test/PropertiesTestCase.java | 43 + .../search/query/context/test/TraceTestCase.java | 101 + .../profile/config/test/MultiProfileTestCase.java | 62 + .../test/QueryProfileConfigurationTestCase.java | 164 ++ .../test/QueryProfileIntegrationTestCase.java | 171 ++ .../test/TypedProfilesConfigurationTestCase.java | 58 + .../profile/config/test/XmlReadingTestCase.java | 421 ++++ .../query/profile/config/test/bug3197426.cfg | 45 + .../config/test/invalidxml1/illegalSetting.xml | 4 + .../config/test/invalidxml2/unparseable.xml | 2 + .../profile/config/test/invalidxml3/default.xml | 4 + .../query/profile/config/test/klee/production.xml | 6 + .../config/test/klee/twitter_dd-us-0.2.4.xml | 14 + .../profile/config/test/klee/twitter_dd-us.xml | 4 + .../query/profile/config/test/klee/twitter_dd.xml | 14 + .../config/test/multiprofile/multiprofile1.xml | 30 + .../test/multiprofile/multiprofileDimensions.xml | 7 + .../profile/config/test/multiprofile/parent1.xml | 4 + .../profile/config/test/multiprofile/parent2.xml | 4 + .../query/profile/config/test/news/default.xml | 8 + .../query/profile/config/test/news/yahoo_uk.xml | 4 + .../profile/config/test/news/yahoo_uk_test.xml | 4 + .../profile/config/test/newscase1/default.xml | 15 + .../query/profile/config/test/newscase1/parent.xml | 5 + .../profile/config/test/newscase2/default.xml | 15 + .../query/profile/config/test/newscase2/parent.xml | 5 + .../profile/config/test/newscase3/default.xml | 15 + .../query/profile/config/test/newscase3/parent.xml | 5 + .../profile/config/test/newscase4/default.xml | 15 + .../query/profile/config/test/newscase4/parent.xml | 5 + .../profile/config/test/newsfe/backend_news.xml | 9 + .../query/profile/config/test/newsfe/default.xml | 4 + .../config/test/newsfe2/backend.news.provider.xml | 8 + .../query/profile/config/test/newsfe2/default.xml | 5 + .../test/query-profile-variants-configuration.cfg | 95 + .../config/test/query-profile-variants2.cfg | 63 + .../config/test/query-profiles-configuration.cfg | 52 + .../config/test/queryprofilevariants2/default.xml | 10 + .../test/queryprofilevariants2/mandatory.xml | 3 + .../queryprofilevariants2/mandatorySpecified.xml | 5 + .../config/test/queryprofilevariants2/multi.xml | 22 + .../test/queryprofilevariants2/multiDimensions.xml | 4 + .../test/queryprofilevariants2/querybest.xml | 5 + .../test/queryprofilevariants2/querylove.xml | 5 + .../queryprofilevariants2/referingQuerybest.xml | 4 + .../root.unoverridableIndex.xml | 5 + .../config/test/queryprofilevariants2/root.xml | 7 + .../test/queryprofilevariants2/rootChild.xml | 3 + .../test/queryprofilevariants2/rootStrict.xml | 3 + .../test/queryprofilevariants2/rootWithFilter.xml | 4 + .../config/test/queryprofilevariants2/test.xml | 5 + .../queryprofilevariants2/types/forbidding.xml | 7 + .../test/queryprofilevariants2/types/mandatory.xml | 5 + .../test/queryprofilevariants2/types/root.xml | 5 + .../queryprofilevariants2/types/rootStrict.xml | 4 + .../profile/config/test/refoverride/MyProfile1.xml | 5 + .../profile/config/test/refoverride/MyProfile2.xml | 5 + .../profile/config/test/refoverride/default.xml | 5 + .../config/test/refoverridetyped/MyProfile1.xml | 5 + .../config/test/refoverridetyped/MyProfile2.xml | 5 + .../config/test/refoverridetyped/default.xml | 5 + .../config/test/refoverridetyped/types/default.xml | 4 + .../profile/config/test/sourceprovider/common.xml | 5 + .../config/test/sourceprovider/myprofile.xml | 6 + .../config/test/sourceprovider/provider.xml | 9 + .../profile/config/test/sourceprovider/source.xml | 4 + .../profile/config/test/systemtest/default.xml | 8 + .../query/profile/config/test/typed-profiles.cfg | 42 + .../query/profile/config/test/typed/.gitignore | 10 + .../query/profile/config/test/typed/chains.cfg | 16 + .../query/profile/config/test/typed/components.cfg | 11 + .../query/profile/config/test/typed/handlers.cfg | 2 + .../query/profile/config/test/typed/index-info.cfg | 0 .../query/profile/config/test/typed/qr-search.cfg | 0 .../profile/config/test/typed/qr-searchers.cfg | 4 + .../profile/config/test/typed/query-profiles.cfg | 49 + .../profile/config/test/typed/specialtokens.cfg | 1 + .../query/profile/config/test/untyped/.gitignore | 10 + .../query/profile/config/test/untyped/chains.cfg | 16 + .../profile/config/test/untyped/components.cfg | 11 + .../query/profile/config/test/untyped/handlers.cfg | 2 + .../profile/config/test/untyped/index-info.cfg | 0 .../profile/config/test/untyped/qr-search.cfg | 0 .../profile/config/test/untyped/qr-searchers.cfg | 4 + .../profile/config/test/untyped/query-profiles.cfg | 29 + .../profile/config/test/untyped/specialtokens.cfg | 1 + .../query/profile/config/test/validxml/default.xml | 10 + .../profile/config/test/validxml/modelSettings.xml | 4 + .../test/validxml/referencingModelSettings.xml | 7 + .../query/profile/config/test/validxml/root.xml | 9 + .../profile/config/test/validxml/someUser.xml | 12 + .../config/test/validxml/types/rootType.xml | 12 + .../profile/config/test/validxml/types/user.xml | 10 + .../config/test/versionrefs/MyProfile-1.0.0.xml | 4 + .../config/test/versionrefs/MyProfile-1.0.2.a.xml | 4 + .../config/test/versionrefs/MyProfile-1.0.2.xml | 4 + .../profile/config/test/versionrefs/default.xml | 5 + .../config/test/versions/testprofile-1.20.100.xml | 3 + .../profile/config/test/versions/testprofile.xml | 3 + .../search/query/profile/test/CloningTestCase.java | 226 ++ .../profile/test/DimensionBindingTestCase.java | 74 + .../query/profile/test/DumpToolTestCase.java | 35 + .../profile/test/QueryFromProfileTestCase.java | 81 + .../test/QueryProfileCloneMicroBenchmark.java | 81 + ...ProfileGetInComplexStructureMicroBenchmark.java | 120 + .../test/QueryProfileGetMicroBenchmark.java | 88 + .../QueryProfileListPropertiesMicroBenchmark.java | 105 + .../test/QueryProfileSubstitutionTestCase.java | 130 + .../query/profile/test/QueryProfileTestCase.java | 562 +++++ .../test/QueryProfileVariantsCloneTestCase.java | 58 + .../profile/test/QueryProfileVariantsTestCase.java | 1052 ++++++++ .../profile/types/test/FieldTypeTestCase.java | 23 + .../profile/types/test/MandatoryTestCase.java | 201 ++ .../query/profile/types/test/NameTestCase.java | 104 + .../types/test/NativePropertiesTestCase.java | 48 + .../query/profile/types/test/OverrideTestCase.java | 179 ++ .../profile/types/test/PatchMatchingTestCase.java | 186 ++ .../test/QueryProfileTypeInheritanceTestCase.java | 121 + .../types/test/QueryProfileTypeTestCase.java | 595 +++++ .../query/properties/test/PropertyMapTestCase.java | 61 + .../test/RequestContextPropertiesTestCase.java | 30 + .../properties/test/SubPropertiesTestCase.java | 38 + .../query/rewrite/RewriterFeaturesTestCase.java | 45 + .../test/GenericExpansionRewriterTestCase.java | 202 ++ .../rewrite/test/MisspellRewriterTestCase.java | 136 ++ .../query/rewrite/test/NameRewriterTestCase.java | 179 ++ .../rewrite/test/QueryRewriteSearcherTestCase.java | 133 ++ .../test/QueryRewriteSearcherTestUtils.java | 125 + .../SearchChainDispatcherSearcherTestCase.java | 179 ++ .../query/rewrite/test/generic_expansion.fsa | Bin 0 -> 2007 bytes .../query/rewrite/test/name_rewriter_entity.fsa | Bin 0 -> 2117 bytes .../test/test_generic_expansion_rewriter.cfg | 3 + .../query/rewrite/test/test_name_rewriter.cfg | 3 + .../query/rewrite/test/test_rewriter_fake_fsa.cfg | 3 + .../com/yahoo/search/query/test/ModelTestCase.java | 112 + .../search/query/test/ParametersTestCase.java | 65 + .../search/query/test/PresentationTestCase.java | 34 + .../query/test/QueryCloneMicroBenchmark.java | 42 + .../yahoo/search/query/test/RankingTestCase.java | 91 + .../textserialize/item/test/ParseItemTestCase.java | 175 ++ .../serializer/test/SerializeItemTestCase.java | 159 ++ .../querytransform/BooleanAttributeParserTest.java | 101 + .../querytransform/BooleanSearcherTestCase.java | 121 + .../querytransform/LegacyCombinatorTestCase.java | 245 ++ .../search/querytransform/LowercasingTestCase.java | 217 ++ .../com/yahoo/search/querytransform/TestUtils.java | 12 + .../querytransform/WandSearcherTestCase.java | 232 ++ .../querytransform/test/NGramSearcherTestCase.java | 369 +++ .../test/QueryCombinatorTestCase.java | 165 ++ .../test/RangeQueryOptimizerTestCase.java | 224 ++ .../test/SortingDegraderTestCase.java | 173 ++ .../rendering/AsyncGroupPopulationTestCase.java | 144 ++ .../search/rendering/JsonRendererTestCase.java | 1111 +++++++++ .../rendering/SyncDefaultRendererTestCase.java | 103 + .../search/rendering/XMLRendererTestCase.java | 123 + .../search/result/DefaultErrorHitTestCase.java | 124 + .../com/yahoo/search/result/NanNumberTestCase.java | 41 + .../yahoo/search/result/TemplatingTestCase.java | 174 ++ .../search/result/test/ArrayOutputTestCase.java | 31 + .../yahoo/search/result/test/CoverageTestCase.java | 61 + .../result/test/DeepHitIteratorTestCase.java | 172 ++ .../result/test/DefaultErrorHitTestCase.java | 38 + .../yahoo/search/result/test/FillingTestCase.java | 59 + .../yahoo/search/result/test/HitGroupTestCase.java | 189 ++ .../config/test/DependencyConfigTestCase.java | 79 + .../config/test/SearchChainConfigurerTestCase.java | 302 +++ .../search/searchchain/config/test/chains.cfg | 56 + .../config/test/chainsConfigUpdate_1.cfg | 8 + .../config/test/chainsConfigUpdate_2.cfg | 10 + .../config/test/dependencyConfig/chains.cfg | 20 + .../config/test/dependencyConfig/handlers.cfg | 2 + .../config/test/dependencyConfig/index-info.cfg | 0 .../config/test/dependencyConfig/qr-search.cfg | 0 .../config/test/dependencyConfig/qr-searchers.cfg | 4 + .../config/test/dependencyConfig/specialtokens.cfg | 1 + .../search/searchchain/config/test/handlers.cfg | 2 + .../config/test/implicitDependencies.cfg | 14 + .../search/searchchain/config/test/index-info.cfg | 0 .../yahoo/search/searchchain/config/test/int.cfg | 1 + .../search/searchchain/config/test/qr-logging.cfg | 1 + .../search/searchchain/config/test/qr-search.cfg | 0 .../searchchain/config/test/qr-searchers.cfg | 4 + .../config/test/searcher1-2.1/Manifest.MF | 12 + .../config/test/searcher1-2.1/Searcher1.java.text | 22 + .../config/test/searcher1-2.2/Manifest.MF | 12 + .../config/test/searcher1-2.2/Searcher1.java.text | 22 + .../config/test/searcher1-2/Manifest.MF | 12 + .../config/test/searcher1-2/Searcher1.java.text | 22 + .../searchchain/config/test/searcher1/Manifest.MF | 12 + .../config/test/searcher1/Searcher1.java | 25 + .../searchchain/config/test/searcher2/Manifest.MF | 12 + .../config/test/searcher2/Searcher2.java | 18 + .../searchchain/config/test/specialtokens.cfg | 1 + .../search/searchchain/config/test/string.cfg | 1 + .../config/test/testInstances/chains.cfg | 27 + .../config/test/testInstances/components.cfg | 24 + .../config/test/testInstances/handlers.cfg | 2 + .../config/test/testInstances/qr-searchers.cfg | 4 + .../config/test/testInstances/specialtokens.cfg | 1 + .../config/test/testInstances/updatesearcher.cfg | 6 + .../config/test/testInstances/updatesearcher2.cfg | 7 + .../searchchain/config/test/three-searchers.cfg | 10 + .../config/test/twosearchers/Manifest.MF | 11 + .../config/test/twosearchers/MultiSearcher1.java | 18 + .../config/test/twosearchers/MultiSearcher2.java | 18 + .../config/test/updatesearcher-2/Manifest.MF | 12 + .../test/updatesearcher-2/UpdateSearcher.java.text | 24 + .../config/test/updatesearcher/Manifest.MF | 12 + .../config/test/updatesearcher/UpdateSearcher.java | 23 + .../yahoo/search/searchchain/model/test/chains.cfg | 33 + .../test/AsyncExecutionOfOneChainTestCase.java | 97 + .../searchchain/test/AsyncExecutionTestCase.java | 156 ++ .../search/searchchain/test/ExecutionTestCase.java | 297 +++ .../searchchain/test/FutureDataTestCase.java | 150 ++ .../searchchain/test/SearchChainTestCase.java | 101 + .../search/searchchain/test/SimpleSearchChain.java | 110 + .../search/searchchain/test/TraceTestCase.java | 221 ++ .../searchchain/test/VespaAsyncSearcherTest.java | 58 + .../test/CacheControlSearcherTestCase.java | 130 + .../test/ConnectionControlSearcherTestCase.java | 97 + .../test/InputCheckingSearcherTestCase.java | 106 + .../yahoo/search/searchers/test/MockMetric.java | 78 + .../searchers/test/RateLimitingBenchmark.java | 208 ++ .../test/RateLimitingSearcherTestCase.java | 130 + .../test/ValidateMatchPhaseSearcherTestCase.java | 120 + .../search/statistics/ElapsedTimeTestCase.java | 433 ++++ .../yahoo/search/statistics/PeakQpsTestCase.java | 164 ++ .../search/statistics/TimingSearcherTestCase.java | 83 + .../com/yahoo/search/statistics/test/.gitignore | 0 .../java/com/yahoo/search/test/QueryBenchmark.java | 59 + .../java/com/yahoo/search/test/QueryTestCase.java | 671 ++++++ .../test/RequestParameterPreservationTestCase.java | 21 + .../com/yahoo/search/test/ResultBenchmark.java | 76 + .../com/yahoo/search/yql/FieldFilterTestCase.java | 90 + .../search/yql/MinimalQueryInserterTestCase.java | 297 +++ .../com/yahoo/search/yql/ResegmentingTestCase.java | 147 ++ .../com/yahoo/search/yql/UserInputTestCase.java | 280 +++ .../yahoo/search/yql/VespaSerializerTestCase.java | 404 ++++ .../search/yql/YqlFieldAndSourceTestCase.java | 159 ++ .../com/yahoo/search/yql/YqlParserTestCase.java | 928 ++++++++ .../interpretation/test/AnnotationTestCase.java | 123 + .../streamingvisitors/ListMergerTestCase.java | 87 + .../streamingvisitors/MetricsSearcherTestCase.java | 141 ++ .../VdsStreamingSearcherTestCase.java | 295 +++ .../streamingvisitors/VdsVisitorTestCase.java | 560 +++++ 653 files changed, 56066 insertions(+) create mode 100644 container-search/src/test/java/com/yahoo/component/chain/dependencies/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/component/provider/test/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/component/test/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/core/config/basic/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/core/config/configurable/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/http/fileserver/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/logging/test/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/test/ConstantFile.java create mode 100644 container-search/src/test/java/com/yahoo/container/test/globalpackages create mode 100644 container-search/src/test/java/com/yahoo/container/test/globalpackageservice/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/test/httpheaders/container-config/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/test/logging/container-config/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/test/methods/container-config/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/container/test/minimal/container-config/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/FastHitTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/PacketTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/Calculator.java create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/CalculatorService.java create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/client/Client.java create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/client/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterService.java create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterServiceImpl.java create mode 100644 container-search/src/test/java/com/yahoo/osgi/test/counterservice/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/cache/test/CacheTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/cluster/test/HasherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumFieldTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/FieldsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/JsonFieldTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DispatchThread.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockFDispatch.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketWrapperTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/category.enum create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/updated-qr-summary-dummy.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/grouping/legacy/test/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/prelude/hitfield/XmlRendererTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/hitfield/test/HitFieldTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/hitfield/test/TokenFieldIteratorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/ItemLabelTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/TaggableItemsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/WordAlternativesItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/TestSegmenter.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/UnicodePropertyDumpTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/SubstringTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/WashPhrasesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/DotProductItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/IntItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/PhraseItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerMicroBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/QueryTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/RangeItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/SegmentItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/WandItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/test/WeightedSetItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/basic.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/composite.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/CollapsePhraseSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/NoRankingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/NonPhrasingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhraseMatcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhrasingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java create mode 100755 container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/accent-removal-index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/cjk-index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/container-http.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases-input.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments-input.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words-input.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa-input.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/querytransform/test/testindexinfonoboost.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/CachingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/ErrorHitRenderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/MultipleResultsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-summary.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserfieldinfo.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserquoting.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testfieldinfo.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testhit.xml create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testindexinfo.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testlazymapping.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testphysicalmapping.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/testquoting.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/searcher/test/validate_sorting.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/compatibility/test/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/rules.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/semantics.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/AlibabaTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/AnchorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataNotTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/BlendingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/CJKTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/Ellipsis2TestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/EllipsisTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/LabelMatchingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchAllTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/NotTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/NumbersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/NumericTermsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/Parameter2TestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/StopwordTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/UrlTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/alibaba.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/anchor.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatanot.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatarules.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/backtrackingrules.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/blending.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk-rules.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparison.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparisons.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/duplicaterules.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis2.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/empty.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatch.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatchtrick.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child1.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child2.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/cjk.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandchild.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandfather.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandmother.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/parent.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/labelmatching.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/match-only-if-not-only-term.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/matchall.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/nostemming.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/not.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numbers.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numericterms.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/orphrase.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter2.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter3.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/rules.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantic-rules.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.fsa create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.txt create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stopwords.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/substitution.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/url.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/weighting.sr create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/qr-templates.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/templaterc create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml create mode 100644 container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/DummySearcher.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/GetRawWordTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/LocationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/ResultTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/fieldtypes/field-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/integration/FirstSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/integration/SecondSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/integration/ThirdSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/integration/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/integration/qr.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/qr-fileserver.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/qr-logging.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/qr-summary.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/qr.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/prelude/test/statistics.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java create mode 100644 container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/image/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml create mode 100644 container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/qr-search.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml create mode 100644 container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml3/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml create mode 100755 container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml create mode 100755 container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml create mode 100755 container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml create mode 100755 container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-search.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-search.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg create mode 100755 container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg create mode 100755 container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-search.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/index-info.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-search.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java create mode 100755 container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/statistics/test/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/vespa/streamingvisitors/ListMergerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java (limited to 'container-search/src/test/java/com/yahoo') diff --git a/container-search/src/test/java/com/yahoo/component/chain/dependencies/.gitignore b/container-search/src/test/java/com/yahoo/component/chain/dependencies/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/component/provider/test/.gitignore b/container-search/src/test/java/com/yahoo/component/provider/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/component/test/.gitignore b/container-search/src/test/java/com/yahoo/component/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/core/config/basic/.gitignore b/container-search/src/test/java/com/yahoo/container/core/config/basic/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/core/config/configurable/.gitignore b/container-search/src/test/java/com/yahoo/container/core/config/configurable/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/http/fileserver/.gitignore b/container-search/src/test/java/com/yahoo/container/http/fileserver/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/logging/test/.gitignore b/container-search/src/test/java/com/yahoo/container/logging/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/test/ConstantFile.java b/container-search/src/test/java/com/yahoo/container/test/ConstantFile.java new file mode 100644 index 00000000000..b6e3fbbac5d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/container/test/ConstantFile.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.test; + +public class ConstantFile { + public static String TEST_STRING = "test_string"; +} diff --git a/container-search/src/test/java/com/yahoo/container/test/globalpackages b/container-search/src/test/java/com/yahoo/container/test/globalpackages new file mode 100644 index 00000000000..c93e063d190 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/container/test/globalpackages @@ -0,0 +1,3 @@ +com.yahoo.container.test +com.yahoo.container +#com.yahoo.test diff --git a/container-search/src/test/java/com/yahoo/container/test/globalpackageservice/.gitignore b/container-search/src/test/java/com/yahoo/container/test/globalpackageservice/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/test/httpheaders/container-config/.gitignore b/container-search/src/test/java/com/yahoo/container/test/httpheaders/container-config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/test/logging/container-config/.gitignore b/container-search/src/test/java/com/yahoo/container/test/logging/container-config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/test/methods/container-config/.gitignore b/container-search/src/test/java/com/yahoo/container/test/methods/container-config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/container/test/minimal/container-config/.gitignore b/container-search/src/test/java/com/yahoo/container/test/minimal/container-config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java b/container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java new file mode 100644 index 00000000000..d89e37b2e23 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.StringWriter; +import java.nio.ByteBuffer; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.fs4.mplex.FS4Channel; +import com.yahoo.fs4.mplex.InvalidChannelException; +import com.yahoo.search.Query; + +/** + * Ensure hex dumping of packets seems to work. + * + * @author Steinar Knutsen + */ +public class PacketQueryTracerTestCase { + FS4Channel channel; + BasicPacket packet; + PacketListener tracer; + + static class MockChannel extends FS4Channel { + + @Override + public void setQuery(Query query) { + super.setQuery(query); + } + + @Override + public Query getQuery() { + return super.getQuery(); + } + + @Override + public Integer getChannelId() { + return 1; + } + + @Override + public void close() { + } + + @Override + public boolean sendPacket(BasicPacket packet) + throws InvalidChannelException, IOException { + return true; + } + + @Override + public BasicPacket[] receivePackets(long timeout, int packetCount) + throws InvalidChannelException, ChannelTimeoutException { + return null; + } + + @Override + public BasicPacket nextPacket(long timeout) + throws InterruptedException, InvalidChannelException { + return null; + } + + @Override + protected void addPacket(BasicPacket packet) + throws InterruptedException, InvalidChannelException { + } + + @Override + public boolean isValid() { + return true; + } + + @Override + public String toString() { + return "MockChannel"; + } + } + + @Before + public void setUp() throws Exception { + channel = new MockChannel(); + channel.setQuery(new Query("/?query=a&tracelevel=11")); + packet = new PingPacket(); + tracer = new PacketQueryTracer(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testPacketSent() throws IOException { + byte[] simulatedPacket = new byte[] { 1, 2, 3 }; + tracer.packetReceived(channel, packet, ByteBuffer.wrap(simulatedPacket)); + StringWriter w = new StringWriter(); + channel.getQuery().getContext(false).render(w); + assertTrue(w.getBuffer().toString().indexOf("PingPacket: 010203") != -1); + } + + @Test + public final void testPacketReceived() throws IOException { + byte[] simulatedPacket = new byte[] { 1, 2, 3 }; + tracer.packetReceived(channel, packet, ByteBuffer.wrap(simulatedPacket)); + StringWriter w = new StringWriter(); + channel.getQuery().getContext(false).render(w); + assertTrue(w.getBuffer().toString().indexOf("PingPacket: 010203") != -1); + } + +} diff --git a/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java b/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java new file mode 100644 index 00000000000..8b9f7a345b1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.mplex; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.logging.Logger; + +import com.yahoo.container.search.Fs4Config; +import com.yahoo.prelude.fastsearch.FS4ResourcePool; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.ChannelTimeoutException; +import com.yahoo.fs4.PacketListener; +import com.yahoo.fs4.PingPacket; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.fs4.mplex.Backend.BackendStatistics; +import com.yahoo.search.Query; + +/** + * Test networking code for talking to dispatch. + * + * @author Steinar Knutsen + */ +public class BackendTestCase { + + public static class MockDispatch implements Runnable { + public ServerSocket socket; + public volatile Socket connection; + volatile int channelId; + + public byte[] packetData = new byte[] { 0, 0, 0, 76, 0, 0, 0, 202 - 256, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0x40, + 0x39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0x40, 0x37, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 0x40, 0x35, 0, 0, 0, 0, 0, 0 }; + + public MockDispatch(final ServerSocket socket) { + this.socket = socket; + } + + @Override + public void run() { + try { + connection = socket.accept(); + } catch (final IOException e) { + e.printStackTrace(); + return; + } + requestRespond(); + } + + void requestRespond() { + final byte[] length = new byte[4]; + try { + connection.getInputStream().read(length); + } catch (final IOException e) { + e.printStackTrace(); + return; + } + final int actual = ByteBuffer.wrap(length).getInt(); + + int read = 0; + int i = 0; + while (read != -1 && i < actual) { + try { + read = connection.getInputStream().read(); + ++i; + } catch (final IOException e) { + e.printStackTrace(); + return; + } + } + final ByteBuffer reply = ByteBuffer.wrap(packetData); + if (channelId != -1) { + reply.putInt(8, channelId); + } + try { + connection.getOutputStream().write(packetData); + } catch (final IOException e) { + e.printStackTrace(); + } + } + + } + + public static class MockPacketListener implements PacketListener { + + @Override + public void packetSent(final FS4Channel channel, + final BasicPacket packet, final ByteBuffer serializedForm) { + + } + + @Override + public void packetReceived(final FS4Channel channel, + final BasicPacket packet, final ByteBuffer serializedForm) { + + } + + } + + public static class MockServer { + public InetSocketAddress host; + public Thread worker; + public MockDispatch dispatch; + + public MockServer() throws IOException { + final ServerSocket socket = new ServerSocket(0); + host = (InetSocketAddress) socket.getLocalSocketAddress(); + dispatch = new MockDispatch(socket); + worker = new Thread(dispatch); + worker.start(); + } + + } + + Backend backend; + MockServer server; + private Logger logger; + private boolean initUseParent; + FS4ResourcePool listeners; + + public static final byte[] PONG = new byte[] { 0, 0, 0, 28, 0, 0, 0, 210 - 256, + 0, 0, 0, 42, 0, 0, 0, 127, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 1, 0, + 0, 0, 1 }; + + @Before + public void setUp() throws Exception { + logger = Logger.getLogger(Backend.class.getName()); + initUseParent = logger.getUseParentHandlers(); + logger.setUseParentHandlers(false); + listeners = new FS4ResourcePool(new Fs4Config()); + + server = new MockServer(); + backend = listeners.getBackend(server.host.getHostString(), server.host.getPort()); + } + + @After + public void tearDown() throws Exception { + listeners.deconstruct(); + if (server.dispatch.socket != null) server.dispatch.socket.close(); + if (server.dispatch.connection !=null) server.dispatch.connection.close(); + if (server.worker!=null) server.worker.join(); + if (logger !=null) logger.setUseParentHandlers(initUseParent); + } + + @Test + public final void testBackend() throws IOException, InvalidChannelException { + try { + final FS4Channel channel = backend.openChannel(); + final Query q = new Query("/?query=a"); + BasicPacket[] b = null; + final int channelId = channel.getChannelId(); + server.dispatch.channelId = channelId; + + assertTrue(backend.sendPacket(QueryPacket.create(q), channelId)); + try { + b = channel.receivePackets(1000, 1); + } catch (final ChannelTimeoutException e) { + fail("Could not get packets from simulated backend."); + } + assertEquals(1, b.length); + assertEquals(202, b[0].getCode()); + channel.close(); + } + catch (java.net.UnknownHostException e) { + // We are on vpn, or have no network + } + } + + @Test + public final void testPinging() throws IOException, InvalidChannelException { + try { + final FS4Channel channel = backend.openPingChannel(); + BasicPacket[] b = null; + server.dispatch.channelId = -1; + server.dispatch.packetData = PONG; + + assertTrue(channel.sendPacket(new PingPacket())); + try { + b = channel.receivePackets(1000, 1); + } catch (final ChannelTimeoutException e) { + fail("Could not get packets from simulated backend."); + } + assertEquals(1, b.length); + assertEquals(210, b[0].getCode()); + channel.close(); + } + catch (java.net.UnknownHostException e) { + // We are on vpn, or have no network + } + } + + @Test + public final void requireStatistics() throws IOException, InvalidChannelException { + try { + final FS4Channel channel = backend.openPingChannel(); + server.dispatch.channelId = -1; + server.dispatch.packetData = PONG; + + assertTrue(channel.sendPacket(new PingPacket())); + try { + channel.receivePackets(1000, 1); + } catch (final ChannelTimeoutException e) { + fail("Could not get packets from simulated backend."); + } + final BackendStatistics stats = backend.getStatistics(); + assertEquals(1, stats.totalConnections()); + } + catch (java.net.UnknownHostException e) { + // We are on vpn, or have no network + } + } +} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/FastHitTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/FastHitTestCase.java new file mode 100644 index 00000000000..60e024b00af --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/test/FastHitTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.test; + +import com.yahoo.prelude.fastsearch.FastHit; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Simon Thoresen Hult + */ +public class FastHitTestCase { + + @Test + public void requireThatIgnoreRowBitsIsFalseByDefault() { + FastHit hit = new FastHit(); + assertFalse(hit.shouldIgnoreRowBits()); + } + + @Test + public void requireThatIgnoreRowBitsCanBeSet() { + FastHit hit = new FastHit(); + hit.setIgnoreRowBits(true); + assertTrue(hit.shouldIgnoreRowBits()); + } +} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java new file mode 100644 index 00000000000..df504aff852 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.test; + +import com.yahoo.fs4.BufferTooSmallException; +import com.yahoo.fs4.GetDocSumsPacket; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tests the GetDocsumsPacket + * + * @author Bjorn Borud + */ +public class GetDocSumsPacketTestCase { + + private static final byte IGNORE = 69; + + @Test + public void testDefaultDocsumClass() { + Query query = new Query("/?query=chain"); + assertNull(query.getPresentation().getSummary()); + } + + @Test + public void testEncodingWithQuery() throws BufferTooSmallException { + FastHit hit = new FastHit(); + hit.setIgnoreRowBits(true); + assertPacket(true, hit, new byte[] { 0, 0, 0, 57, 0, 0, 0, -37, 0, 0, 40, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, -128, 0, 0, 0, 0, 7, + 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 3 }); + + hit = new FastHit(); + hit.setIgnoreRowBits(false); + assertPacket(true, hit, new byte[] {0, 0, 0, 57, 0, 0, 0, -37, 0, 0, 40, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, -128, 0, 0, 0, 0, 7, + 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 2}); + } + + @Test + public void testEncodingWithoutQuery() throws BufferTooSmallException { + FastHit hit = new FastHit(); + hit.setIgnoreRowBits(true); + assertPacket(false, hit, new byte[] { 0, 0, 0, 43, 0, 0, 0, -37, 0, 0, 40, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, -128, 0, 0, 0, 0, 7, + 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 3 + }); + + hit = new FastHit(); + hit.setIgnoreRowBits(false); + assertPacket(false, hit, new byte[] { 0, 0, 0, 43, 0, 0, 0, -37, 0, 0, 40, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, + IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, -128, 0, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 2 + }); + } + + @Test + public void requireThatSessionIdIsEncodedAsPropertyWhenUsingSearchSession() throws BufferTooSmallException { + Result result = new Result(new Query("?query=foo")); + result.getQuery().getSessionId(true); // create session id. + result.getQuery().getRanking().setQueryCache(true); + FastHit hit = new FastHit(); + result.hits().add(hit); + assertPacket(false, result, new byte[] { 0, 0, 0, -123, 0, 0, 0, -37, 0, 0, 56, 17, 0, 0, 0, 0, + // query timeout + IGNORE, IGNORE, IGNORE, IGNORE, + // "default" - rank profile + 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, -128, 0, + // "default" - summaryclass + 0, 0, 0, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', + // 2 property entries + 0, 0, 0, 2, + // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 + 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd', 0, 0, 0, 26, 'q', 'r', 's', 'e', 'r', 'v', 'e', 'r', '.', + IGNORE, '.', IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, '.', IGNORE, + // caches: features => true + 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e', + // flags + 0, 0, 0, 2 + }); + } + + private static void assertPacket(boolean sendQuery, Hit hit, byte[] expected) throws BufferTooSmallException { + Result result = new Result(new Query("?query=foo")); + result.hits().add(hit); + assertPacket(sendQuery, result, expected); + } + + private static void assertPacket(boolean sendQuery, Result result, byte[] expected) throws BufferTooSmallException { + GetDocSumsPacket packet = GetDocSumsPacket.create(result, "default", sendQuery); + ByteBuffer buf = ByteBuffer.allocate(1024); + packet.encode(buf); + buf.flip(); + + byte[] actual = new byte[buf.remaining()]; + buf.get(actual); + // assertEquals(Arrays.toString(expected), Arrays.toString(actual)); + + assertEquals("Equal length", expected.length, actual.length); + for (int i = 0; i < expected.length; ++i) { + if (expected[i] == IGNORE) { + actual[i] = IGNORE; + } + } + + assertArrayEquals(expected, actual); + } +} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java new file mode 100644 index 00000000000..7675acc88fc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.test; + +import java.nio.ByteBuffer; + +import junit.framework.TestCase; + +import com.yahoo.fs4.HexByteIterator; + +/** + * Test of HexByteIterator + * @author tonytv + */ +public class HexByteIteratorTestCase extends TestCase { + public void testHexByteIterator() { + int[] numbers = { 0x00, 0x01, 0xDE, 0xAD, 0xBE, 0xEF, 0xFF }; + + HexByteIterator i = new HexByteIterator( + ByteBuffer.wrap(toBytes(numbers))); + + assertEquals("00", i.next()); + assertEquals("01", i.next()); + assertEquals("DE", i.next()); + assertEquals("AD", i.next()); + assertEquals("BE", i.next()); + assertEquals("EF", i.next()); + assertEquals("FF", i.next()); + assertTrue(!i.hasNext()); + } + + private byte[] toBytes(int[] ints) { + byte[] bytes = new byte[ints.length]; + for (int i=0; iBjorn Borud + */ +public class PacketDecoderTestCase { + static byte[] queryResultPacketData + = new byte[] {0,0,0,104, + 0,0,0,214-256, + 0,0,0,1, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 0,0,0,97, + 0,0,0,3, 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,23, 0,0,0,7, 0,0,0,36, + 0,0,0,4, 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,21, 0,0,0,8, 0,0,0,37}; + static int len = queryResultPacketData.length; + + /** + * In this testcase we have exactly one packet which fills the + * entire buffer + */ + @Test + public void testOnePacket () throws BufferTooSmallException { + ByteBuffer data = ByteBuffer.allocate(len); + data.put(queryResultPacketData); + data.flip(); + + + // not really necessary for testing, but these help visualize + // the state the buffer should be in so a reader of this test + // will not have to + assertEquals(0, data.position()); + assertEquals(len, data.limit()); + assertEquals(len, data.capacity()); + assertEquals(data.limit(), data.capacity()); + + PacketDecoder.DecodedPacket p = PacketDecoder.extractPacket(data); + assertTrue(p.packet instanceof QueryResultPacket); + + // now the buffer should have position == capacity == limit + assertEquals(len, data.position()); + assertEquals(len, data.limit()); + assertEquals(len, data.capacity()); + + // next call to decode on same bufer should result + // in null and buffer should be reset for writing. + p = PacketDecoder.extractPacket(data); + assertTrue(p == null); + + // make sure the buffer is now ready for reading + assertEquals(0, data.position()); + assertEquals(len, data.limit()); + assertEquals(len, data.capacity()); + } + + /** + * In this testcase we only have 3 bytes so we can't + * even determine the size of the packet. + */ + @Test + public void testThreeBytesPacket () throws BufferTooSmallException { + ByteBuffer data = ByteBuffer.allocate(len); + data.put(queryResultPacketData, 0, 3); + data.flip(); + + // packetLength() should return -1 since we don't even have + // the size of the packet + assertEquals(-1, PacketDecoder.packetLength(data)); + + // since we can't determine the size we don't get a packet. + // the buffer should now be at offset 3 so we can read more + // data and limit should be set to capacity + PacketDecoder.DecodedPacket p = PacketDecoder.extractPacket(data); + assertTrue(p == null); + assertEquals(3, data.position()); + assertEquals(len, data.limit()); + assertEquals(len, data.capacity()); + } + + /** + * In this testcase we have a partial packet and room for + * more data + */ + @Test + public void testPartialWithMoreRoom () throws BufferTooSmallException { + ByteBuffer data = ByteBuffer.allocate(len); + data.put(queryResultPacketData, 0, 10); + data.flip(); + + PacketDecoder.DecodedPacket p = PacketDecoder.extractPacket(data); + assertTrue(p == null); + + } + + /** + * In this testcase we have one and a half packet + */ + @Test + public void testOneAndAHalfPackets () throws BufferTooSmallException { + int half = len / 2; + ByteBuffer data = ByteBuffer.allocate(len + half); + data.put(queryResultPacketData); + data.put(queryResultPacketData, 0, half); + assertEquals((len + half), data.position()); + data.flip(); + + // the first packet we should be able to extract just fine + BasicPacket p1 = PacketDecoder.extractPacket(data).packet; + assertTrue(p1 instanceof QueryResultPacket); + + PacketDecoder.DecodedPacket p2 = PacketDecoder.extractPacket(data); + assertTrue(p2 == null); + + // at this point the buffer should be ready for more + // reading so position should be at the end and limit + // should be at capacity + assertEquals(half, data.position()); + assertEquals(data.capacity(), data.limit()); + } + + /** + * Test the case where the buffer is too small for the + * packet + */ + @Test + public void testTooSmallBufferForPacket () { + ByteBuffer data = ByteBuffer.allocate(10); + data.put(queryResultPacketData, 0, 10); + data.flip(); + + try { + PacketDecoder.extractPacket(data); + fail(); + } + catch (BufferTooSmallException e) { + + } + } + + @Test + public void testErrorPacket() throws BufferTooSmallException { + ByteBuffer b = ByteBuffer.allocate(100); + b.putInt(0); + b.putInt(203); + b.putInt(1); + b.putInt(37); + b.putInt(5); + b.put(new byte[] { (byte) 'n', (byte) 'a', (byte) 'l', (byte) 'l', (byte) 'e' }); + b.putInt(0, b.position() - 4); + b.flip(); + DecodedPacket p = PacketDecoder.extractPacket(b); + ErrorPacket e = (ErrorPacket) p.packet; + assertEquals("nalle (37)", e.toString()); + assertEquals(203, e.getCode()); + assertEquals(37, e.getErrorCode()); + b = ByteBuffer.allocate(100); + // warn if encoding support is added untested + e.encode(b); + b.position(0); + assertEquals(4, b.getInt()); + assertEquals(203, b.getInt()); + assertFalse(b.hasRemaining()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/PacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/PacketTestCase.java new file mode 100644 index 00000000000..151d192c25b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/test/PacketTestCase.java @@ -0,0 +1,225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import com.yahoo.fs4.*; +import com.yahoo.search.Query; +import org.junit.Ignore; +import org.junit.Test; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.TestCase.*; +import static org.junit.Assume.assumeTrue; + +/** + * Tests the Packet class. Specifically made this unit test suite + * for checking that queries that are too large for the buffer + * are handled gracefully. + * + * @author Bjorn Borud + */ +public class PacketTestCase { + + /** + * Make sure we don't get false negatives for reasonably sized + * buffers + */ + @Test + public void testSmallQueryOK () { + Query query = new Query("/?query=foo"); + assertNotNull(query); + + QueryPacket queryPacket = QueryPacket.create(query); + assertNotNull(queryPacket); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + int position = buffer.position(); + + try { + queryPacket.encode(buffer, 0); + } + catch (BufferTooSmallException e) { + fail(); + } + + // make sure state of buffer HAS changed and is according + // to contract + assertTrue(position != buffer.position()); + assertTrue(buffer.position() == buffer.limit()); + } + + /** + * Make a query that is too large and then try to encode it + * into a small ByteBuffer + */ + @Test + public void testLargeQueryFail () { + StringBuilder queryBuffer = new StringBuilder(4008); + queryBuffer.append("/?query="); + for (int i=0; i < 1000; i++) { + queryBuffer.append("the%20"); + } + Query query = new Query(queryBuffer.toString()); + assertNotNull(query); + + QueryPacket queryPacket = QueryPacket.create(query); + assertNotNull(queryPacket); + + ByteBuffer buffer = ByteBuffer.allocate(100); + int position = buffer.position(); + int limit = buffer.limit(); + try { + queryPacket.encode(buffer, 0); + fail(); + } + catch (BufferTooSmallException e) { + // success if exception is thrown + } + + // make sure state of buffer is unchanged + assertEquals(position, buffer.position()); + assertEquals(limit, buffer.limit()); + } + + @Test + public void requireThatPacketsCanTurnOnCompression() throws BufferTooSmallException { + QueryPacket queryPacket = QueryPacket.create(new Query("/?query=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int channel = 32; + + queryPacket.encode(buffer, channel); + buffer.flip(); + assertEquals(86, buffer.getInt()); // size + assertEquals(0xda, buffer.getInt()); // code + assertEquals(channel, buffer.getInt()); + + queryPacket.setCompressionLimit(88); + buffer.clear(); + queryPacket.encode(buffer, channel); + buffer.flip(); + assertEquals(86, buffer.getInt()); // size + assertEquals(0xda, buffer.getInt()); // code + + queryPacket.setCompressionLimit(84); + buffer.clear(); + queryPacket.encode(buffer, channel); + buffer.flip(); + assertEquals(57, buffer.getInt()); // size + assertEquals(0x060000da, buffer.getInt()); // code + assertEquals(channel, buffer.getInt()); + } + + @Test + public void requireThatUncompressablePacketsArentCompressed() throws BufferTooSmallException { + QueryPacket queryPacket = QueryPacket.create(new Query("/?query=aaaaaaaaaaaaaaa")); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int channel = 32; + + queryPacket.setCompressionLimit(10); + buffer.clear(); + queryPacket.encode(buffer, channel); + buffer.flip(); + assertEquals(56, buffer.getInt()); // size + assertEquals(0xda, buffer.getInt()); // code + assertEquals(channel, buffer.getInt()); + } + + class MyPacket extends Packet { + private String bodyString = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + private int myCode = 1234; + + @Override + public int getCode() { + return myCode; + } + + @Override + protected void encodeBody(ByteBuffer buffer) { + buffer.put(bodyString.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public void codeDecodedHook(int code) { + assertEquals(myCode, code); + } + + @Override + public void decodeBody(ByteBuffer buffer) { + byte[] bytes = new byte[bodyString.length()]; + buffer.get(bytes); + assertEquals(bodyString, new String(bytes)); + } + } + + @Test + public void requireThatCompressedPacketsCanBeDecompressed() throws BufferTooSmallException { + + MyPacket packet = new MyPacket(); + packet.setCompressionLimit(10); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int channel = 32; + packet.encode(buffer, channel); + + buffer.flip(); + new MyPacket().decode(buffer); + } + + @Test + public void requireThatCompressedByteBufferMayContainExtraData() throws BufferTooSmallException { + + MyPacket packet = new MyPacket(); + packet.setCompressionLimit(10); + ByteBuffer buffer = ByteBuffer.allocate(1024); + buffer.putLong(0xdeadbeefL); + int channel = 32; + packet.encode(buffer, channel); + buffer.limit(buffer.limit() + 8); + buffer.putLong(0xdeadbeefL); + + buffer.flip(); + assertEquals(0xdeadbeefL, buffer.getLong()); // read initial content. + new MyPacket().decode(buffer); + assertEquals(0xdeadbeefL, buffer.getLong()); // read final content. + } + + class MyBasicPacket extends BasicPacket { + private String bodyString = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + private int myCode = 1234; + + @Override + public int getCode() { + return myCode; + } + + @Override + protected void encodeBody(ByteBuffer buffer) { + buffer.put(bodyString.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public void codeDecodedHook(int code) { + assertEquals(myCode, code); + } + + @Override + public void decodeBody(ByteBuffer buffer) { + byte[] bytes = new byte[bodyString.length()]; + buffer.get(bytes); + assertEquals(bodyString, new String(bytes)); + } + } + + @Test + public void requireThatCompressedBasicPacketsCanBeDecompressed() throws BufferTooSmallException { + + MyBasicPacket packet = new MyBasicPacket(); + packet.setCompressionLimit(10); + ByteBuffer buffer = ByteBuffer.allocate(1024); + packet.encode(buffer); + + buffer.flip(); + new MyBasicPacket().decode(buffer); + } +} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java new file mode 100644 index 00000000000..b60e94a7641 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.test; + +import com.yahoo.document.GlobalId; +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.DocumentInfo; +import com.yahoo.fs4.PacketDecoder; +import com.yahoo.fs4.QueryResultPacket; + +import java.nio.ByteBuffer; + +/** + * Tests encoding of query packages + * + * @author bratseth + */ +public class QueryResultTestCase extends junit.framework.TestCase { + + public QueryResultTestCase(String name) { + super(name); + } + + private static GlobalId gid1 = new GlobalId(new byte[] {1,1,1,1,1,1,1,1,1,1,1,1}); + private static GlobalId gid2 = new GlobalId(new byte[] {2,2,2,2,2,2,2,2,2,2,2,2}); + + public void testDecodeQueryResult() { + byte[] packetData=new byte[] {0,0,0,76, + 0,0,0,202-256, + 0,0,0,1, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0}; // 4 + 40 + 2*12 + ByteBuffer buffer=ByteBuffer.allocate(100); + buffer.put(packetData); + buffer.flip(); + BasicPacket packet=PacketDecoder.decode(buffer); + assertTrue(packet instanceof QueryResultPacket); + QueryResultPacket result=(QueryResultPacket)packet; + + assertTrue( ! result.getMldFeature()); + assertTrue( ! result.getDatasetFeature()); + + assertEquals(25,result.getMaxRank()); + assertEquals(111,result.getDocstamp()); + + assertEquals(2,result.getDocuments().size()); + DocumentInfo document1= result.getDocuments().get(0); + assertEquals(gid1, document1.getGlobalId()); + assertEquals(23.0,document1.getMetric()); + DocumentInfo document2= result.getDocuments().get(1); + assertEquals(gid2, document2.getGlobalId()); + assertEquals(21.0,document2.getMetric()); + } + + public void testDecodeQueryResultMld() { + byte[] packetData=new byte[] {0,0,0,92, + 0,0,0,208-256, + 0,0,0,1, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + ByteBuffer buffer=ByteBuffer.allocate(100); + buffer.put(packetData); + buffer.flip(); + BasicPacket packet=PacketDecoder.decode(buffer); + assertTrue(packet instanceof QueryResultPacket); + QueryResultPacket result=(QueryResultPacket)packet; + + assertTrue( result.getMldFeature()); + assertTrue( ! result.getDatasetFeature()); + + assertEquals(25,result.getMaxRank()); + assertEquals(111,result.getDocstamp()); + + assertEquals(2,result.getDocuments().size()); + DocumentInfo document1= result.getDocuments().get(0); + assertEquals(gid1,document1.getGlobalId()); + assertEquals(23.0,document1.getMetric()); + assertEquals(7,document1.getPartId()); + assertEquals(36,document1.getDistributionKey()); + DocumentInfo document2= result.getDocuments().get(1); + assertEquals(gid2,document2.getGlobalId()); + assertEquals(21.0,document2.getMetric()); + assertEquals(8,document2.getPartId()); + assertEquals(37,document2.getDistributionKey()); + } + + public void testDecodeQueryResultMLD2() { + byte[] packetData=new byte[] {0,0,0,96, + 0,0,0,214-256, + 0,0,0,1, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 0,0,0,97, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + ByteBuffer buffer=ByteBuffer.allocate(100); + buffer.put(packetData); + buffer.flip(); + BasicPacket packet=PacketDecoder.decode(buffer); + assertTrue(packet instanceof QueryResultPacket); + QueryResultPacket result=(QueryResultPacket)packet; + + assertTrue( result.getMldFeature()); + assertTrue( result.getDatasetFeature()); + + assertEquals(25,result.getMaxRank()); + assertEquals(111,result.getDocstamp()); + assertEquals(97,result.getDataset()); + + assertEquals(2,result.getDocuments().size()); + DocumentInfo document1= result.getDocuments().get(0); + assertEquals(gid1,document1.getGlobalId()); + assertEquals(23.0,document1.getMetric()); + assertEquals(7,document1.getPartId()); + assertEquals(36,document1.getDistributionKey()); + DocumentInfo document2= result.getDocuments().get(1); + assertEquals(gid2,document2.getGlobalId()); + assertEquals(21.0,document2.getMetric()); + assertEquals(8,document2.getPartId()); + assertEquals(37,document2.getDistributionKey()); + } + + public void testDecodeQueryResultX() { + byte[] packetData=new byte[] {0,0,0,100, + 0,0,0,217-256, + 0,0,0,1, + 0,0,0,15, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 0,0,0,97, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + ByteBuffer buffer=ByteBuffer.allocate(200); + buffer.put(packetData); + buffer.flip(); + BasicPacket packet=PacketDecoder.decode(buffer); + assertTrue(packet instanceof QueryResultPacket); + QueryResultPacket result=(QueryResultPacket)packet; + + assertTrue(result.getMldFeature()); + assertTrue(result.getDatasetFeature()); + + assertEquals(5,result.getTotalDocumentCount()); + assertEquals(25,result.getMaxRank()); + assertEquals(111,result.getDocstamp()); + assertEquals(97,result.getDataset()); + + assertEquals(2,result.getDocuments().size()); + DocumentInfo document1= result.getDocuments().get(0); + assertEquals(gid1,document1.getGlobalId()); + assertEquals(23.0,document1.getMetric()); + assertEquals(7,document1.getPartId()); + assertEquals(36,document1.getDistributionKey()); + DocumentInfo document2= result.getDocuments().get(1); + assertEquals(gid2,document2.getGlobalId()); + assertEquals(21.0,document2.getMetric()); + assertEquals(8,document2.getPartId()); + assertEquals(37,document2.getDistributionKey()); + } + + public void testDecodeQueryResultMoreHits() { + byte[] packetData=new byte[] {0,0,0,100, + 0,0,0,217-256, + 0,0,0,1, + 0,0,0,15, + 0,0,0,0, + 0,0,0,2, + 0,0,0,0,0,0,0,5, + 0x40,0x39,0,0,0,0,0,0, + 0,0,0,111, + 0,0,0,97, + 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + ByteBuffer buffer=ByteBuffer.allocate(200); + buffer.put(packetData); + buffer.flip(); + BasicPacket packet=PacketDecoder.decode(buffer); + assertTrue(packet instanceof QueryResultPacket); + QueryResultPacket result=(QueryResultPacket)packet; + + assertEquals(2,result.getDocuments().size()); + DocumentInfo document1= result.getDocuments().get(0); + assertEquals(gid1,document1.getGlobalId()); + DocumentInfo document2= result.getDocuments().get(1); + assertEquals(gid2,document2.getGlobalId()); + } +} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java new file mode 100644 index 00000000000..462685425eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java @@ -0,0 +1,294 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.fs4.test; + +import com.yahoo.fs4.BufferTooSmallException; +import com.yahoo.fs4.Packet; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.prelude.Freshness; +import com.yahoo.prelude.query.*; +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.search.Query; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * Tests encoding of query x packages + * + * @author bratseth + */ +public class QueryTestCase extends junit.framework.TestCase { + + public QueryTestCase(String name) { + super(name); + } + + public void testEncodePacket() throws BufferTooSmallException { + Query query=new Query("/?query=chain&timeout=0"); + query.setWindow(2, 8); + QueryPacket packet=QueryPacket.create(query); + assertEquals(2,packet.getOffset()); + assertEquals(8,packet.getHits()); + + byte[] encoded = packetToBytes(packet); + byte[] correctBuffer=new byte[] {0,0,0,46,0,0,0,-38,0,0,0,0, // Header + 0,0,0,6, // Features + 2, + 8, + 0,0,0,50, // querytimeout + 0,0,-64,4, // qflags + 7, + 'd', 'e', 'f', 'a', 'u', 'l', 't', + 0,0,0,1,0,0,0,8,4, + 0,5, + 99,104,97,105,110}; + assertEqualArrays(correctBuffer,encoded); + } + + public void testEncodeQueryPacketWithSomeAdditionalFeatures() throws BufferTooSmallException { + Query query=new Query("/?query=chain&dataset=10&type=phrase&timeout=0"); + // Because the rank mapping now needs config and a searcher, + // we do the sledgehammer dance: + query.getRanking().setProfile("two"); + query.setWindow(2, 8); + QueryPacket packet=QueryPacket.create(query); + byte[] encoded = packetToBytes(packet); + byte[] correctBuffer=new byte[] {0,0,0,42,0,0,0,-38,0,0,0,0, // Header + 0,0,0,6, // Features + 2, + 8, + 0,0,0,50, // querytimeout + 0,0,-64,4, // QFlags + 3, + 't','w','o', // Ranking + 0,0,0,1,0,0,0,8,4, + 0,5, + 99,104,97,105,110}; + assertEqualArrays(correctBuffer, encoded); + } + + /** This test will tell you if you have screwed up the binary encoding, but it won't tell you how */ + public void testEncodeQueryPacketWithManyFeatures() throws BufferTooSmallException { + Query query=new Query("/?query=chain" + + "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + + "&model.searchPath=7/3"); + query.getRanking().setFreshness(new Freshness("123456")); + query.getRanking().setSorting("+field1 -field2"); + query.getRanking().setProfile("two"); + Highlight highlight = new Highlight(); + highlight.addHighlightTerm("field1","term1"); + highlight.addHighlightTerm("field1","term2"); + query.getPresentation().setHighlight(highlight); + + query.prepare(); + + QueryPacket packet=QueryPacket.create(query); + byte[] encoded = packetToBytes(packet); + // System.out.println(Arrays.toString(encoded)); + byte[] correctBuffer=new byte[] { + 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, -64, 4, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' + }; + assertEqualArrays(correctBuffer,encoded); + } + + /** This test will tell you if you have screwed up the binary encoding, but it won't tell you how */ + public void testEncodeQueryPacketWithManyFeaturesFresnhessAsString() throws BufferTooSmallException { + Query query=new Query("/?query=chain" + + "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + + "&model.searchPath=7/3"); + query.getRanking().setFreshness("123456"); + query.getRanking().setSorting("+field1 -field2"); + query.getRanking().setProfile("two"); + Highlight highlight = new Highlight(); + highlight.addHighlightTerm("field1","term1"); + highlight.addHighlightTerm("field1","term2"); + query.getPresentation().setHighlight(highlight); + + query.prepare(); + + QueryPacket packet=QueryPacket.create(query); + byte[] encoded = packetToBytes(packet); + // System.out.println(Arrays.toString(encoded)); + byte[] correctBuffer=new byte[] { + 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, -64, 4, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' + }; + assertEqualArrays(correctBuffer,encoded); + } + + public void testEncodeQueryPacketWithLabelsConnectivityAndSignificance() throws BufferTooSmallException { + Query query=new Query(); + AndItem and = new AndItem(); + WeightedSetItem taggable1 = new WeightedSetItem("field1"); + taggable1.setLabel("foo"); + WeightedSetItem taggable2 = new WeightedSetItem("field2"); + taggable1.setLabel("bar"); + and.addItem(taggable1); + and.addItem(taggable2); + WordItem word1 = new WordItem("word1", "field3"); + word1.setSignificance(0.37); + WordItem word2 = new WordItem("word1", "field3"); + word2.setSignificance(0.81); + word2.setConnectivity(word1, 0.15); + and.addItem(word1); + and.addItem(word2); + + query.getModel().getQueryTree().setRoot(and); + + query.prepare(); + + QueryPacket packet=QueryPacket.create(query); + byte[] encoded = packetToBytes(packet); + byte[] correctBuffer=new byte[] { + 0, 0, 1, 16, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, ignored, ignored, ignored, ignored, 0, 0, -64, 4, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 1, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 18, 'v', 'e', 's', 'p', 'a', 46, 'l', 'a', 'b', 'e', 'l', 46, 'b', 'a', 'r', 46, 'i', 'd', 0, 0, 0, 1, '1', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 1, '3', 0, 0, 0, 22, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 'c', 'o', 'n', 'n', 'e', 'x', 'i', 't', 'y', 0, 0, 0, 4, '0', 46, '1', '5', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '3', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '3', '7', 0, 0, 0, 25, 'v', 'e', 's', 'p', 'a', 46, 't', 'e', 'r', 'm', 46, '4', 46, 's', 'i', 'g', 'n', 'i', 'f', 'i', 'c', 'a', 'n', 'c', 'e', 0, 0, 0, 4, '0', 46, '8', '1', 0, 0, 0, 5, 0, 0, 0, '4', 1, 4, 79, 1, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 79, 2, 0, 6, 'f', 'i', 'e', 'l', 'd', '2', 68, 3, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', '1', 68, 4, 6, 'f', 'i', 'e', 'l', 'd', '3', 5, 'w', 'o', 'r', 'd', 49 + }; + assertEqualArrays(correctBuffer,encoded); + } + + public void testEncodeSortSpec() throws BufferTooSmallException { + Query query=new Query("/?query=chain&sortspec=%2Ba+-b&timeout=0"); + query.setWindow(2, 8); + QueryPacket packet=QueryPacket.create(query); + ByteBuffer buffer=ByteBuffer.allocate(500); + buffer.limit(0); + packet.encode(buffer, 0); + byte[] encoded=new byte[buffer.position()]; + buffer.rewind(); + buffer.get(encoded); + byte[] correctBuffer=new byte[] {0,0,0,55,0,0,0,-38,0,0,0,0, // Header + 0,0,0,-122, // Features + 2, // offset + 8, // maxhits + 0,0,0,50, // querytimeout + 0,0,-64,4, // qflags + 7, + 'd', 'e', 'f', 'a', 'u', 'l', 't', + 0,0,0,5, // sortspec length + 43,97,32,45,98, // sortspec + 0,0,0,1, // num stackitems + 0,0,0,8,4, + 0,5, + 99,104,97,105,110}; + assertEqualArrays(correctBuffer,encoded); + + // Encode again to test grantEncodingBuffer + buffer = packet.grantEncodingBuffer(0); + encoded = new byte[buffer.limit()]; + buffer.get(encoded); + assertEqualArrays(correctBuffer,encoded); + } + + public void testPhraseEqualsPhraseWithPhraseSegment() throws BufferTooSmallException { + Query query = new Query(); + PhraseItem p = new PhraseItem(); + PhraseSegmentItem ps = new PhraseSegmentItem("a b", false, false); + ps.addItem(new WordItem("a")); + ps.addItem(new WordItem("b")); + p.addItem(ps); + query.getModel().getQueryTree().setRoot(p); + + query.setTimeout(0); + QueryPacket queryPacket = QueryPacket.create(query); + + ByteBuffer buffer1 = ByteBuffer.allocate(1024); + + queryPacket.encode(buffer1, 0); + + query = new Query(); + p = new PhraseItem(); + p.addItem(new WordItem("a")); + p.addItem(new WordItem("b")); + query.getModel().getQueryTree().setRoot(p); + + query.setTimeout(0); + queryPacket = QueryPacket.create(query); + assertNotNull(queryPacket); + + ByteBuffer buffer2 = ByteBuffer.allocate(1024); + + queryPacket.encode(buffer2, 0); + + byte[] encoded1 = new byte[buffer1.position()]; + buffer1.rewind(); + buffer1.get(encoded1); + byte[] encoded2 = new byte[buffer2.position()]; + buffer2.rewind(); + buffer2.get(encoded2); + assertEqualArrays(encoded2, encoded1); + } + + public void testPatchInChannelId() throws BufferTooSmallException { + Query query=new Query("/?query=chain&timeout=0"); + query.setWindow(2, 8); + QueryPacket packet=QueryPacket.create(query); + assertEquals(2,packet.getOffset()); + assertEquals(8, packet.getHits()); + + ByteBuffer buffer = packet.grantEncodingBuffer(0x07070707); + + byte[] correctBuffer=new byte[] {0,0,0,46,0,0,0,-38,7,7,7,7, // Header + 0,0,0,6, // Features + 2, + 8, + 0,0,0,50, // querytimeout + 0,0,-64,4, // qflags + 7, + 'd', 'e', 'f', 'a', 'u', 'l', 't', + 0,0,0,1,0,0,0,8,4, + 0,5, + 99,104,97,105,110}; + + byte[] encoded=new byte[buffer.limit()]; + buffer.get(encoded); + + assertEqualArrays(correctBuffer,encoded); + + packet.allocateAndEncode(0x07070707); + buffer = packet.grantEncodingBuffer(0x09090909); + correctBuffer=new byte[] {0,0,0,46,0,0,0,-38,9,9,9,9, // Header + 0,0,0,6, // Features + 2, + 8, + 0,0,0,50, // querytimeout + 0,0,-64,4, // qflags + 7, + 'd', 'e', 'f', 'a', 'u', 'l', 't', + 0,0,0,1,0,0,0,8,4, + 0,5, + 99,104,97,105,110}; + + encoded=new byte[buffer.limit()]; + buffer.get(encoded); + + assertEqualArrays(correctBuffer,encoded); + } + + public static byte[] packetToBytes(Packet packet) { + try { + ByteBuffer buffer=ByteBuffer.allocate(500); + buffer.limit(0); + packet.encode(buffer,0); + byte[] encoded=new byte[buffer.position()]; + buffer.rewind(); + buffer.get(encoded); + return encoded; + } + catch (BufferTooSmallException e) { + throw new RuntimeException(e); + } + } + + public static void assertEqualArrays(byte[] correct, byte[] test) { + assertEquals("Incorrect length,", correct.length, test.length); + for (int i=0; iGeir Storli + */ +public class RankFeaturesTestCase { + + @Test + public void requireThatRankPropertiesTakesBothStringAndObject() { + RankProperties p = new RankProperties(); + p.put("string", "b"); + p.put("object", new Integer(7)); + assertEquals("7", p.get("object").get(0)); + assertEquals("b", p.get("string").get(0)); + } + + @Test + public void requireThatSingleTensorIsBinaryEncoded() { + Tensor tensor = MapTensor.from("{ {x:a, y:b}:2.0, {z:c}:3.0 }"); + assertTensorEncodingAndDecoding("query(my_tensor)", "my_tensor", tensor); + assertTensorEncodingAndDecoding("$my_tensor", "my_tensor", tensor); + } + + @Test + public void requireThatMultipleTensorsAreBinaryEncoded() { + Tensor tensor1 = MapTensor.from("{ {x:a, y:b}:2.0, {z:c}:3.0 }"); + Tensor tensor2 = MapTensor.from("{ {x:a, y:b, z:c}:5.0 }"); + assertTensorEncodingAndDecoding(Arrays.asList( + new Entry("query(tensor1)", "tensor1", tensor1), + new Entry("$tensor2", "tensor2", tensor2))); + } + + private static class Entry { + final String key; + final String normalizedKey; + final Tensor tensor; + Entry(String key, String normalizedKey, Tensor tensor) { + this.key = key; + this.normalizedKey = normalizedKey; + this.tensor = tensor; + } + } + + private static void assertTensorEncodingAndDecoding(List entries) { + RankProperties properties = createRankPropertiesWithTensors(entries); + assertEquals(entries.size(), properties.asMap().size()); + + Map decodedProperties = decode(encode(properties)); + assertEquals(entries.size() * 2, properties.asMap().size()); // tensor type info has been added + assertEquals(entries.size() * 2, decodedProperties.size()); + for (Entry entry : entries) { + assertEquals(entry.tensor, (Tensor) decodedProperties.get(entry.normalizedKey)); + assertEquals("tensor", (String) decodedProperties.get(entry.normalizedKey + ".type")); + } + } + + private static void assertTensorEncodingAndDecoding(String key, String normalizedKey, Tensor tensor) { + assertTensorEncodingAndDecoding(Arrays.asList(new Entry(key, normalizedKey, tensor))); + } + + private static RankProperties createRankPropertiesWithTensors(List entries) { + RankFeatures features = new RankFeatures(); + for (Entry entry : entries) { + features.put(entry.key, entry.tensor); + } + RankProperties properties = new RankProperties(); + features.prepare(properties); + return properties; + } + + private static byte[] encode(RankProperties properties) { + ByteBuffer buffer = ByteBuffer.allocate(512); + properties.encode(buffer, true); + byte[] result = new byte[buffer.position()]; + buffer.rewind(); + buffer.get(result); + return result; + } + + private static Map decode(byte[] encodedProperties) { + GrowableByteBuffer buffer = GrowableByteBuffer.wrap(encodedProperties); + byte[] mapNameBytes = new byte[buffer.getInt()]; + buffer.get(mapNameBytes); + int numEntries = buffer.getInt(); + Map result = new HashMap<>(); + for (int i = 0; i < numEntries; ++i) { + byte[] keyBytes = new byte[buffer.getInt()]; + buffer.get(keyBytes); + String key = Utf8.toString(keyBytes); + byte[] value = new byte[buffer.getInt()]; + buffer.get(value); + if (key.contains(".type")) { + result.put(key, Utf8.toString(value)); + } else { + result.put(key, TypedBinaryFormat.decode(value)); + } + } + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/osgi/test/Calculator.java b/container-search/src/test/java/com/yahoo/osgi/test/Calculator.java new file mode 100644 index 00000000000..e53b423d2b7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/Calculator.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi.test; + +/** + * A public interface which can be implemented by a bundle + * + * @author bratseth + */ +public interface Calculator { + + int add(int a,int b); + +} diff --git a/container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/CalculatorService.java b/container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/CalculatorService.java new file mode 100644 index 00000000000..d33b74030ab --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/CalculatorService.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi.test.calculatorservice; + +import org.osgi.framework.BundleActivator; +import org.osgi.framework.BundleContext; +import org.osgi.framework.ServiceRegistration; + +import com.yahoo.osgi.test.Calculator; + +/** + * An implementation of an interface which is not part of the bundle + * + * @author bratseth + */ +public class CalculatorService implements Calculator, BundleActivator { + + private ServiceRegistration calculatorServiceRegistration; + + public int add(int a,int b) { + return a+b; + } + + public void start(BundleContext context) { + calculatorServiceRegistration=context.registerService(Calculator.class.getName(), this, null); + } + + public void stop(BundleContext context) { + calculatorServiceRegistration.unregister(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/Manifest.MF b/container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/Manifest.MF new file mode 100644 index 00000000000..cfb2f945761 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/calculatorservice/Manifest.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: CalculatorService +Bundle-SymbolicName: com.yahoo.osgi.test.calculatorservice.CalculatorService +Bundle-Version: 1.0.0 +Bundle-Activator: com.yahoo.osgi.test.calculatorservice.CalculatorService +Bundle-Vendor: Yahoo! +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search +Export-Package: com.yahoo.osgi.test.calculatorservice diff --git a/container-search/src/test/java/com/yahoo/osgi/test/client/Client.java b/container-search/src/test/java/com/yahoo/osgi/test/client/Client.java new file mode 100644 index 00000000000..47d5ca1f16a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/client/Client.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi.test.client; + +import org.osgi.framework.BundleActivator; +import org.osgi.framework.BundleContext; +import org.osgi.framework.ServiceReference; + +import com.yahoo.osgi.test.counterservice.CounterService; + +/** + * A bundle which is a client calling to another bundle. + * + * @author bratseth + */ +public class Client implements BundleActivator { + + @SuppressWarnings("unchecked") + public void start(BundleContext context) { + ServiceReference counterServiceReference=context.getServiceReference(CounterService.class.getName()); + CounterService counterService =(CounterService)context.getService(counterServiceReference); + assertEquals(0, counterService.getCounter()); + counterService.incrementCounter(); + assertEquals(1, counterService.getCounter()); + } + + public void stop(BundleContext context) {} + + private void assertEquals(Object correct,Object test) { + if (!correct.equals(test)) + throw new RuntimeException("Expected " + correct + ", got " + test); + } + +} diff --git a/container-search/src/test/java/com/yahoo/osgi/test/client/Manifest.MF b/container-search/src/test/java/com/yahoo/osgi/test/client/Manifest.MF new file mode 100644 index 00000000000..e50de302fc5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/client/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Client +Bundle-SymbolicName: com.yahoo.osgi.test.client.Client +Bundle-Version: 1.0.0 +Bundle-Activator: com.yahoo.osgi.test.client.Client +Bundle-Vendor: Yahoo! +Import-Package: com.yahoo.osgi.test.counterservice,org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterService.java b/container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterService.java new file mode 100644 index 00000000000..ddb6445a91c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterService.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi.test.counterservice; + +/** + * Interface to the test bundle service + * + * @author bratseth + */ +public interface CounterService { + + public int getCounter(); + + public void incrementCounter(); + +} diff --git a/container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterServiceImpl.java b/container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterServiceImpl.java new file mode 100644 index 00000000000..595afdb2525 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/counterservice/CounterServiceImpl.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.osgi.test.counterservice; + +import org.osgi.framework.BundleActivator; +import org.osgi.framework.BundleContext; +import org.osgi.framework.ServiceRegistration; + +/** + * A service which must be imported by another bundle to be used + * + * @author bratseth + */ +public class CounterServiceImpl implements CounterService, BundleActivator { + + private int counter=0; + + private ServiceRegistration counterServiceRegistration; + + public void start(BundleContext context) { + counterServiceRegistration=context.registerService(CounterService.class.getName(), this, null); + } + + public void stop(BundleContext context) { + counterServiceRegistration.unregister(); + } + + public int getCounter() { + return counter; + } + + public void incrementCounter() { + counter++; + } + +} diff --git a/container-search/src/test/java/com/yahoo/osgi/test/counterservice/Manifest.MF b/container-search/src/test/java/com/yahoo/osgi/test/counterservice/Manifest.MF new file mode 100644 index 00000000000..a7644eea4cf --- /dev/null +++ b/container-search/src/test/java/com/yahoo/osgi/test/counterservice/Manifest.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: CounterService +Bundle-SymbolicName: com.yahoo.osgi.test.counterservice.CounterService +Bundle-Version: 1.0.0 +Bundle-Activator: com.yahoo.osgi.test.counterservice.CounterServiceImpl +Bundle-Vendor: Yahoo! +Import-Package: org.osgi.framework;version="1.3.0",com.yahoo.osgi.test.calculatorservice, + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search +Export-Package: com.yahoo.osgi.test.counterservice \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java b/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java new file mode 100644 index 00000000000..7684e55eabc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/IndexFactsFactory.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.ConfigInstance; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.container.QrSearchersConfig; + +/** + * @author Simon Thoresen + */ +public abstract class IndexFactsFactory { + + public static IndexFacts newInstance(String configId) { + return new IndexFacts(new IndexModel(resolveConfig(IndexInfoConfig.class, configId), + resolveConfig(QrSearchersConfig.class, configId))); + + } + + public static IndexFacts newInstance(String indexInfoConfigId, String qrSearchersConfigId) { + return new IndexFacts(new IndexModel(resolveConfig(IndexInfoConfig.class, indexInfoConfigId), + resolveConfig(QrSearchersConfig.class, qrSearchersConfigId))); + + } + + private static T resolveConfig(Class configClass, String configId) { + if (configId == null) return null; + return ConfigGetter.getConfig(configClass, configId); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/cache/test/CacheTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cache/test/CacheTestCase.java new file mode 100644 index 00000000000..668603fce29 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/cache/test/CacheTestCase.java @@ -0,0 +1,171 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.cache.test; + +import junit.framework.TestCase; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.statistics.Statistics; +import com.yahoo.prelude.cache.Cache; +import com.yahoo.prelude.cache.QueryCacheKey; + +@SuppressWarnings({"rawtypes", "unchecked"}) +public class CacheTestCase extends TestCase { + + private Result getSomeResult(Query q, String id) { + Result r = new Result(q); + r.hits().add(new Hit(id, 10)); + return r; + } + + public void testBasicGet() { + Cache cache=new Cache<>(100*1024,3600, 100000, Statistics.nullImplementation); + Query q = new Query("/std_xmls_a00?hits=5&offset=5&query=flowers+shop&tracelevel=4&objid=ffffffffffffffff"); + Query q2 = new Query("/std_xmls_a00?hits=5&offset=5&query=flowers+shop&tracelevel=4&objid=ffffffffffffffff"); + QueryCacheKey qk = new QueryCacheKey(q); + QueryCacheKey qk2 = new QueryCacheKey(q2); + Result r = getSomeResult(q, "foo"); + Result r2 = getSomeResult(q, "bar"); + assertNull(cache.get(qk)); + cache.put(qk, r); + assertNotNull(cache.get(qk)); + assertEquals(cache.get(qk), r); + cache.put(qk2, r); + assertEquals(cache.get(qk2), r); + cache.put(qk, r2); + assertEquals(cache.get(qk), r2); + } + + public void testPutTooLarge() { + byte[] tenKB = new byte[10*1024]; + for (int i = 0 ; i <10*1024 ; i++) { + tenKB[i]=127; + } + byte[] sevenKB = new byte[7*1024]; + for (int i = 0 ; i <7*1024 ; i++) { + sevenKB[i]=127; + } + Cache cache=new Cache(9*1024,3600, 100*1024, Statistics.nullImplementation); // 9 KB + assertFalse(cache.put("foo", tenKB)); + assertTrue(cache.put("foo", sevenKB)); + assertEquals(cache.get("foo"), sevenKB); + } + + public void testInvalidate() { + byte[] tenKB = new byte[10*1024]; + for (int i = 0 ; i <10*1024 ; i++) { + tenKB[i]=127; + } + byte[] sevenKB = new byte[7*1024]; + for (int i = 0 ; i <7*1024 ; i++) { + sevenKB[i]=127; + } + Cache cache=new Cache(11*1024,3600, 100*1024, Statistics.nullImplementation); // 11 KB + assertTrue(cache.put("foo", sevenKB)); + assertTrue(cache.put("bar", tenKB)); + assertNull(cache.get("foo")); + assertEquals(cache.get("bar"), tenKB); + } + + public void testInvalidateLRU() { + Cache cache=new Cache(10*1024,3600, 100*1024, Statistics.nullImplementation); // 10 MB + byte[] fiveKB = new byte[5*1024]; + for (int i = 0 ; i <5*1024 ; i++) { + fiveKB[i]=127; + } + + byte[] twoKB = new byte[2*1024]; + for (int i = 0 ; i <2*1024 ; i++) { + twoKB[i]=127; + } + + byte[] fourKB = new byte[4*1024]; + for (int i = 0 ; i <4*1024 ; i++) { + fourKB[i]=127; + } + assertTrue(cache.put("five", fiveKB)); + assertTrue(cache.put("two", twoKB)); + Object dummy = cache.get("five"); // Makes two LRU + assertEquals(dummy, fiveKB); + assertTrue(cache.put("four", fourKB)); + assertNull(cache.get("two")); + assertEquals(cache.get("five"), fiveKB); + assertEquals(cache.get("four"), fourKB); + + // Same, without the access, just to check + cache=new Cache(10*1024,3600, 100*1024, Statistics.nullImplementation); // 10 KB + assertTrue(cache.put("five", fiveKB)); + assertTrue(cache.put("two", twoKB)); + assertTrue(cache.put("four", fourKB)); + assertEquals(cache.get("two"), twoKB); + assertNull(cache.get("five")); + assertEquals(cache.get("four"), fourKB); + } + + public void testPutSameKey() { + Cache cache=new Cache(10*1024,3600, 100*1024, Statistics.nullImplementation); // 10 MB + byte[] fiveKB = new byte[5*1024]; + for (int i = 0 ; i <5*1024 ; i++) { + fiveKB[i]=127; + } + + byte[] twoKB = new byte[2*1024]; + for (int i = 0 ; i <2*1024 ; i++) { + twoKB[i]=127; + } + + byte[] fourKB = new byte[4*1024]; + for (int i = 0 ; i <4*1024 ; i++) { + fourKB[i]=127; + } + assertTrue(cache.put("five", fiveKB)); + assertTrue(cache.put("two", twoKB)); + assertEquals(cache.get("two"), twoKB); + assertEquals(cache.get("five"), fiveKB); + assertTrue(cache.put("five", twoKB)); + assertEquals(cache.get("five"), twoKB); + assertEquals(cache.get("two"), twoKB); + } + + public void testExpire() throws InterruptedException { + Cache cache=new Cache(10*1024,50, 10000, Statistics.nullImplementation); // 10 KB, 50ms expire + cache.put("foo", "bar"); + cache.put("hey", "ho"); + assertEquals(cache.get("foo"), "bar"); + assertEquals(cache.get("hey"), "ho"); + Thread.sleep(100); + assertNull(cache.get("foo")); + assertNull(cache.get("hey")); + } + + public void testInsertSame() { + Cache cache=new Cache(100*1024,500, 100000, Statistics.nullImplementation); // 100 KB, .5 sec expire + Query q = new Query("/std_xmls_a00?hits=5&offset=5&query=flowers+shop&tracelevel=4&objid=ffffffffffffffff"); + Result r = getSomeResult(q, "foo"); + QueryCacheKey k = new QueryCacheKey(q); + cache.put(k, r); + assertEquals(1, cache.size()); + q = new Query("/std_xmls_a00?hits=5&offset=5&query=flowers+shop&tracelevel=4&objid=ffffffffffffffff"); + k = new QueryCacheKey(q); + cache.put(k, r); + assertEquals(1, cache.size()); + } + + public void testMaxSize() { + Cache cache=new Cache(20*1024,500, 3*1024, Statistics.nullImplementation); + byte[] fourKB = new byte[4*1024]; + for (int i = 0 ; i <4*1024 ; i++) { + fourKB[i]=127; + } + byte[] twoKB = new byte[2*1024]; + for (int i = 0 ; i <2*1024 ; i++) { + twoKB[i]=127; + } + assertFalse(cache.put("four", fourKB)); + assertTrue(cache.put("two", twoKB)); + assertNull(cache.get("four")); + assertNotNull(cache.get("two")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java new file mode 100644 index 00000000000..7ff2b5ffb45 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -0,0 +1,593 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.cluster; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.search.Fs4Config; +import com.yahoo.container.search.LegacyEmulationConfig; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.prelude.*; +import com.yahoo.prelude.fastsearch.*; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.container.handler.VipStatus; +import com.yahoo.container.protect.Error; +import com.yahoo.statistics.Statistics; +import com.yahoo.vespa.config.search.DispatchConfig; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; + +/** + * Tests cluster monitoring + * + * @author bratseth + */ +public class ClusterSearcherTestCase extends junit.framework.TestCase { + + public ClusterSearcherTestCase(String name) { + super(name); + } + + public void testNoBackends() { + ClusterSearcher cluster = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("dummy"))); + try { + cluster.getMonitor().getConfiguration().setRequestTimeout(100); + Execution execution = new Execution(cluster, Execution.Context.createContextStub()); + Query query = new Query("query=hello"); + query.setHits(10); + com.yahoo.search.Result result = execution.search(query); + assertTrue(result.hits().getError() != null); + assertEquals("No backends in service. Try later", result.hits() + .getError().getMessage()); + } finally { + cluster.deconstruct(); + } + } + + + private IndexFacts createIndexFacts() { + Map> clusters = new LinkedHashMap<>(); + clusters.put("cluster1", Arrays.asList("type1", "type2", "type3")); + clusters.put("cluster2", Arrays.asList("type4", "type5")); + clusters.put("type1", Arrays.asList("type6")); + Map searchDefs = new LinkedHashMap<>(); + searchDefs.put("type1", new SearchDefinition("type1")); + searchDefs.put("type2", new SearchDefinition("type2")); + searchDefs.put("type3", new SearchDefinition("type3")); + searchDefs.put("type4", new SearchDefinition("type4")); + searchDefs.put("type5", new SearchDefinition("type5")); + searchDefs.put("type6", new SearchDefinition("type6")); + SearchDefinition union = new SearchDefinition("union"); + return new IndexFacts(new IndexModel(clusters, searchDefs, union)); + } + + private Set resolve(ClusterSearcher searcher, String query) { + return searcher.resolveDocumentTypes(new Query("?query=hello" + query), createIndexFacts()); + } + + public void testThatDocumentTypesAreResolved() { + ClusterSearcher cluster1 = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("type1", "type2", "type3"))); + try { + ClusterSearcher type1 = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("type6"))); + try { + assertEquals(new LinkedHashSet<>(Arrays.asList("type1", "type2", "type3")), resolve(cluster1, "")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type6")), resolve(type1, "")); + { // specify restrict + assertEquals(new LinkedHashSet<>(Arrays.asList("type1")), resolve(cluster1, "&restrict=type1")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2")), resolve(cluster1, "&restrict=type2")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2", "type3")), resolve(cluster1, "&restrict=type2,type3")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2")), resolve(cluster1, "&restrict=type2,type4")); + assertEquals(new LinkedHashSet<>(Arrays.asList()), resolve(cluster1, "&restrict=type4")); + } + { // specify sources + assertEquals(new LinkedHashSet<>(Arrays.asList("type1", "type2", "type3")), resolve(cluster1, "&sources=cluster1")); + assertEquals(new LinkedHashSet<>(Arrays.asList()), resolve(cluster1, "&sources=cluster2")); + assertEquals(new LinkedHashSet<>(Arrays.asList()), resolve(cluster1, "&sources=type1")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type6")), resolve(type1, "&sources=type1")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2")), resolve(cluster1, "&sources=type2")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2", "type3")), resolve(cluster1, "&sources=type2,type3")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2")), resolve(cluster1, "&sources=type2,type4")); + assertEquals(new LinkedHashSet<>(Arrays.asList()), resolve(cluster1, "&sources=type4")); + } + { // specify both + assertEquals(new LinkedHashSet<>(Arrays.asList("type1")), resolve(cluster1, "&sources=cluster1&restrict=type1")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2")), resolve(cluster1, "&sources=cluster1&restrict=type2")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2", "type3")), resolve(cluster1, "&sources=cluster1&restrict=type2,type3")); + assertEquals(new LinkedHashSet<>(Arrays.asList("type2")), resolve(cluster1, "&sources=cluster2&restrict=type2")); + } + } finally { + type1.deconstruct(); + } + } finally { + cluster1.deconstruct(); + } + } + + public void testThatDocumentTypesAreResolvedTODO_REMOVE() { + ClusterSearcher cluster1 = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("type1", "type2", "type3"))); + try { + ClusterSearcher type1 = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("type6"))); + try { + assertEquals(new LinkedHashSet<>(Arrays.asList()), resolve(cluster1, "&sources=cluster2")); + } finally { + type1.deconstruct(); + } + } finally { + cluster1.deconstruct(); + } + } + + private static class MyMockSearcher extends VespaBackEndSearcher { + + private final String type1 = "type1"; + private final String type2 = "type2"; + private final String type3 = "type3"; + private final Map> results = new LinkedHashMap<>(); + private final boolean expectAttributePrefetch; + public static final String ATTRIBUTE_PREFETCH = "attributeprefetch"; + + private String getId(String type, int i) { + return "doc:" + type + ":" + i; + } + + private Hit createHit(String id, double relevancy) { + return createHit(null, id, relevancy); + } + + private Hit createHit(Query query, String id, double relevancy) { + Hit hit = new FastHit(); + hit.setId(id); + hit.setRelevance(relevancy); + hit.setQuery(query); + hit.setFillable(); + return hit; + } + + private Hit createHit(Query query, Hit hit) { + Hit retval = new FastHit(); + retval.setId(hit.getId()); + retval.setRelevance(hit.getRelevance()); + retval.setQuery(query); + retval.setFillable(); + return retval; + } + + private List getHits(Query query) { + Set restrict = query.getModel().getRestrict(); + if (restrict.size() == 1) { + return results.get(restrict.iterator().next()); + } + return null; + } + + private void init() { + results.put(type1, Arrays.asList(createHit(getId(type1, 0), 9), + createHit(getId(type1, 1), 6), + createHit(getId(type1, 2), 3))); + + results.put(type2, Arrays.asList(createHit(getId(type2, 0), 10), + createHit(getId(type2, 1), 7), + createHit(getId(type2, 2), 4))); + + results.put(type3, Arrays.asList(createHit(getId(type3, 0), 11), + createHit(getId(type3, 1), 8), + createHit(getId(type3, 2), 5))); + } + + public MyMockSearcher(boolean expectAttributePrefetch) { + this.expectAttributePrefetch = expectAttributePrefetch; + init(); + } + + @Override + protected com.yahoo.search.Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) { + return null; // search() is overriden, this should never be called + } + + @Override + public com.yahoo.search.Result search(Query query, Execution execution) { + com.yahoo.search.Result result = new com.yahoo.search.Result(query); + List hits = getHits(query); + if (hits != null) { + if (result.getHitOrderer() == null) { // order by relevancy + for (int i = query.getOffset(); i < Math.min(hits.size(), query.getOffset() + query.getHits()); ++i) { + result.hits().add(createHit(query, hits.get(i))); + } + } else { // order by ascending relevancy + for (int i = hits.size() - 1 + query.getOffset(); i >= 0; --i) { + result.hits().add(createHit(query, hits.get(i))); + } + } + result.setTotalHitCount(hits.size()); + } else if (query.getModel().getRestrict().isEmpty()) { + result.hits().add(createHit(query, getId(type1, 3), 2)); + result.setTotalHitCount(1); + } + return result; + } + + @Override + protected void doPartialFill(com.yahoo.search.Result result, String summaryClass) { + if (summaryClass.equals(ATTRIBUTE_PREFETCH) && !expectAttributePrefetch) { + throw new IllegalArgumentException("Got summary class '" + ATTRIBUTE_PREFETCH + "' when not expected"); + } + Set restrictSet = new LinkedHashSet<>(); + for (Iterator hits = result.hits().unorderedDeepIterator(); hits.hasNext(); ) { + Hit hit = hits.next(); + restrictSet.addAll(hit.getQuery().getModel().getRestrict()); + } + if (restrictSet.size() != 1) { + throw new IllegalArgumentException("Expected 1 doctype, got " + restrictSet.size() + ": " + Arrays.toString(restrictSet.toArray())); + } + // Generate summary content + for (Iterator hits = result.hits().unorderedDeepIterator(); hits.hasNext(); ) { + Hit hit = hits.next(); + if (summaryClass.equals(ATTRIBUTE_PREFETCH)) { + hit.setField("asc-score", hit.getRelevance().getScore()); + } else { + hit.setField("score", "score: " + hit.getRelevance().getScore()); + } + hit.setFilled(summaryClass); + } + } + } + + private Execution createExecution() { + return createExecution(Arrays.asList("type1", "type2", "type3"), false); + } + + private Execution createExecution(boolean expectAttributePrefetch) { + return createExecution(Arrays.asList("type1", "type2", "type3"), expectAttributePrefetch); + } + + private Execution createExecution(List docTypesList, boolean expectAttributePrefetch) { + Set documentTypes = new LinkedHashSet<>(); + documentTypes.addAll(docTypesList); + ClusterSearcher cluster = new ClusterSearcher(documentTypes); + try { + cluster.addBackendSearcher(new MyMockSearcher( + expectAttributePrefetch)); + cluster.setValidRankProfile("default", documentTypes); + cluster.addValidRankProfile("testprofile", "type1"); + return new Execution(cluster, Execution.Context.createContextStub()); + } finally { + cluster.deconstruct(); + } + } + + public void testThatSingleDocumentTypeCanBeSearched() { + { // Explicit 1 type in restrict set + Execution execution = createExecution(); + Query query = new Query("?query=hello&restrict=type1"); + com.yahoo.search.Result result = execution.search(query); + assertEquals(3, result.getTotalHitCount()); + List hits = result.hits().asList(); + assertEquals(3, hits.size()); + assertEquals(9.0, hits.get(0).getRelevance().getScore()); + assertEquals(6.0, hits.get(1).getRelevance().getScore()); + assertEquals(3.0, hits.get(2).getRelevance().getScore()); + } + { // Only 1 registered type in cluster searcher, empty restrict set + // NB ! Empty restrict sets does not exist below the cluster searcher. + // restrict set is set by cluster searcher to tell which documentdb is used. + // Modify test to mirror that change. + Execution execution = createExecution(Arrays.asList("type1"), false); + Query query = new Query("?query=hello"); + com.yahoo.search.Result result = execution.search(query); + assertEquals(3, result.getTotalHitCount()); + List hits = result.hits().asList(); + assertEquals(3, hits.size()); + assertEquals(9.0, hits.get(0).getRelevance().getScore()); + } + } + + public void testThatSubsetOfDocumentTypesCanBeSearched() { + Execution execution = createExecution(); + Query query = new Query("?query=hello&restrict=type1,type3"); + + com.yahoo.search.Result result = execution.search(query); + assertEquals(6, result.getTotalHitCount()); + List hits = result.hits().asList(); + assertEquals(6, hits.size()); + assertEquals(11.0, hits.get(0).getRelevance().getScore()); + assertEquals(9.0, hits.get(1).getRelevance().getScore()); + assertEquals(8.0, hits.get(2).getRelevance().getScore()); + assertEquals(6.0, hits.get(3).getRelevance().getScore()); + assertEquals(5.0, hits.get(4).getRelevance().getScore()); + assertEquals(3.0, hits.get(5).getRelevance().getScore()); + } + + public void testThatMultipleDocumentTypesCanBeSearchedAndFilled() { + Execution execution = createExecution(); + Query query = new Query("?query=hello"); + + com.yahoo.search.Result result = execution.search(query); + assertEquals(9, result.getTotalHitCount()); + List hits = result.hits().asList(); + assertEquals(9, hits.size()); + assertEquals(11.0, hits.get(0).getRelevance().getScore()); + assertEquals(10.0, hits.get(1).getRelevance().getScore()); + assertEquals(9.0, hits.get(2).getRelevance().getScore()); + assertEquals(8.0, hits.get(3).getRelevance().getScore()); + assertEquals(7.0, hits.get(4).getRelevance().getScore()); + assertEquals(6.0, hits.get(5).getRelevance().getScore()); + assertEquals(5.0, hits.get(6).getRelevance().getScore()); + assertEquals(4.0, hits.get(7).getRelevance().getScore()); + assertEquals(3.0, hits.get(8).getRelevance().getScore()); + for (int i = 0; i < 9; ++i) { + assertNull(hits.get(i).getField("score")); + } + + execution.fill(result, "summary"); + + hits = result.hits().asList(); + assertEquals("score: 11.0", hits.get(0).getField("score")); + assertEquals("score: 10.0", hits.get(1).getField("score")); + assertEquals("score: 9.0", hits.get(2).getField("score")); + assertEquals("score: 8.0", hits.get(3).getField("score")); + assertEquals("score: 7.0", hits.get(4).getField("score")); + assertEquals("score: 6.0", hits.get(5).getField("score")); + assertEquals("score: 5.0", hits.get(6).getField("score")); + assertEquals("score: 4.0", hits.get(7).getField("score")); + assertEquals("score: 3.0", hits.get(8).getField("score")); + } + + private com.yahoo.search.Result getResult(int offset, int hits, Execution execution) { + return getResult(offset, hits, null, execution); + } + + private com.yahoo.search.Result getResult(int offset, int hits, String extra, Execution execution) { + Query query = new Query("?query=hello" + (extra != null ? (extra) : "")); + query.setOffset(offset); + query.setHits(hits); + return execution.search(query); + } + + private void assertResult(int totalHitCount, List expHits, com.yahoo.search.Result result) { + assertEquals(totalHitCount, result.getTotalHitCount()); + List hits = result.hits().asList(); + assertEquals(expHits.size(), hits.size()); + for (int i = 0; i < expHits.size(); ++i) { + assertEquals(expHits.get(i), hits.get(i).getRelevance().getScore()); + } + } + + public void testThatWeCanSpecifyNumHitsAndHitOffset() { + Execution ex = createExecution(); + + // all types + assertResult(9, Arrays.asList(11.0, 10.0), getResult(0, 2, ex)); + assertResult(9, Arrays.asList(10.0, 9.0), getResult(1, 2, ex)); + assertResult(9, Arrays.asList(9.0, 8.0), getResult(2, 2, ex)); + assertResult(9, Arrays.asList(8.0, 7.0), getResult(3, 2, ex)); + assertResult(9, Arrays.asList(7.0, 6.0), getResult(4, 2, ex)); + assertResult(9, Arrays.asList(6.0, 5.0), getResult(5, 2, ex)); + assertResult(9, Arrays.asList(5.0, 4.0), getResult(6, 2, ex)); + assertResult(9, Arrays.asList(4.0, 3.0), getResult(7, 2, ex)); + assertResult(9, Arrays.asList(3.0), getResult(8, 2, ex)); + assertResult(9, new ArrayList(), getResult(9, 2, ex)); + assertResult(9, Arrays.asList(11.0, 10.0, 9.0, 8.0, 7.0), getResult(0, 5, ex)); + assertResult(9, Arrays.asList(6.0, 5.0, 4.0, 3.0), getResult(5, 5, ex)); + + // restrict=type1 + assertResult(3, Arrays.asList(9.0, 6.0), getResult(0, 2, "&restrict=type1", ex)); + assertResult(3, Arrays.asList(6.0, 3.0), getResult(1, 2, "&restrict=type1", ex)); + assertResult(3, Arrays.asList(3.0), getResult(2, 2, "&restrict=type1", ex)); + assertResult(3, new ArrayList<>(), getResult(3, 2, "&restrict=type1", ex)); + } + + public void testThatWeCanSpecifyNumHitsAndHitOffsetWhenSorting() { + Execution ex = createExecution(true); + + String extra = "&restrict=type1,type2&sorting=%2Basc-score"; + com.yahoo.search.Result result = getResult(0, 2, extra, ex); + assertEquals(3.0, result.hits().asList().get(0).getField("asc-score")); + assertEquals(4.0, result.hits().asList().get(1).getField("asc-score")); + assertResult(6, Arrays.asList(3.0, 4.0), getResult(0, 2, extra, ex)); + assertResult(6, Arrays.asList(4.0, 6.0), getResult(1, 2, extra, ex)); + assertResult(6, Arrays.asList(6.0, 7.0), getResult(2, 2, extra, ex)); + assertResult(6, Arrays.asList(7.0, 9.0), getResult(3, 2, extra, ex)); + assertResult(6, Arrays.asList(9.0, 10.0), getResult(4, 2, extra, ex)); + assertResult(6, Arrays.asList(10.0), getResult(5, 2, extra, ex)); + assertResult(6, new ArrayList<>(), getResult(6, 2, extra, ex)); + } + + public void testLocalConnect() throws UnknownHostException { + ClusterSearcher cluster = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("dummy"))); + boolean canGetLocalName; + boolean canFindYahoo; + final String yahoo = "www.yahoo.com"; + + try { + if (null != InetAddress.getByName(yahoo)) { + canFindYahoo = true; + } else { + canFindYahoo = false; + } + } catch (Exception e) { + canFindYahoo = false; + } + + try { + InetAddress.getLocalHost().getCanonicalHostName(); + canGetLocalName = true; + } catch (Exception e) { + canGetLocalName = false; + } + + assertFalse(cluster.isRemote("127.0.0.1")); + assertFalse(cluster.isRemote("localhost")); + + if (canGetLocalName) { + assertFalse(cluster.isRemote(InetAddress.getLocalHost() + .getCanonicalHostName())); + } + + if (canFindYahoo) { + assertTrue(cluster.isRemote(yahoo)); + } + } + + public void testRequireThatSearchFailsForUndefinedRankProfileWithOneDocType() { + Execution execution = createExecution(Arrays.asList("type1"), false); + + // "default" rank profile + Query query = new Query("?query=hello"); + com.yahoo.search.Result result = execution.search(query); + assertEquals(3, result.getTotalHitCount()); + + // specified "default" rank profile + query = new Query("?query=hello&ranking.profile=default"); + result = execution.search(query); + assertEquals(3, result.getTotalHitCount()); + + // empty rank profile, should fail + query = new Query("?query=hello&ranking.profile="); + result = execution.search(query); + assertEquals(0, result.getTotalHitCount()); + assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); + + // invalid rank profile + query = new Query("?query=hello&ranking.profile=undefined"); + result = execution.search(query); + assertEquals(0, result.getTotalHitCount()); + assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); + + // valid rank profile for type1 + query = new Query("?query=hello&ranking.profile=testprofile"); + result = execution.search(query); + assertEquals(3, result.getTotalHitCount()); + } + + public void testRequireThatSearchFailsForUndefinedRankProfileWithMultipleDocTypes() { + Execution execution = createExecution(Arrays.asList("type1", "type2", "type3"), false); + + // "default" rank profile + Query query = new Query("?query=hello"); + com.yahoo.search.Result result = execution.search(query); + assertEquals(9, result.getTotalHitCount()); + + // specified "default" rank profile + query = new Query("?query=hello&ranking.profile=default"); + result = execution.search(query); + assertEquals(9, result.getTotalHitCount()); + + // empty rank profile, should fail + query = new Query("?query=hello&ranking.profile="); + result = execution.search(query); + assertEquals(0, result.getTotalHitCount()); + assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); + + // invalid rank profile + query = new Query("?query=hello&ranking.profile=undefined"); + result = execution.search(query); + assertEquals(0, result.getTotalHitCount()); + assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); + + // testprofile is only defined for type1, but should pass as it exists in at least one document type + query = new Query("?query=hello&ranking.profile=testprofile"); + result = execution.search(query); + assertEquals(9, result.getTotalHitCount()); + + // testprofile is only defined for type1, but should fail when restricting doc types + query = new Query("?query=hello&ranking.profile=testprofile&restrict=type1,type3"); + result = execution.search(query); + assertEquals(0, result.getTotalHitCount()); + assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); + + // testprofile is only defined for type1, ok if restricted to type1 + query = new Query("?query=hello&ranking.profile=testprofile&restrict=type1"); + result = execution.search(query); + assertEquals(3, result.getTotalHitCount()); + } + + private static ClusterSearcher createSearcher(Double maxQueryTimeout, + Double maxQueryCacheTimeout) { + ComponentId id = new ComponentId("test-id"); + QrSearchersConfig qrsCfg = new QrSearchersConfig(new QrSearchersConfig.Builder(). + searchcluster(new QrSearchersConfig.Searchcluster.Builder().name("test-cluster"))); + ClusterConfig.Builder clusterCfgBld = new ClusterConfig.Builder().clusterName("test-cluster"); + if (maxQueryTimeout != null) { + clusterCfgBld.maxQueryTimeout(maxQueryTimeout); + } + if (maxQueryCacheTimeout != null) { + clusterCfgBld.maxQueryCacheTimeout(maxQueryCacheTimeout); + } + ClusterConfig clusterCfg = new ClusterConfig(clusterCfgBld); + DocumentdbInfoConfig documentDbCfg = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder(). + documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1"))); + LegacyEmulationConfig emulationCfg = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder()); + QrMonitorConfig monitorCfg = new QrMonitorConfig(new QrMonitorConfig.Builder()); + Statistics statMgr = Statistics.nullImplementation; + Fs4Config fs4Cfg = new Fs4Config(new Fs4Config.Builder()); + FS4ResourcePool listeners = new FS4ResourcePool(fs4Cfg); + ClusterSearcher searcher = new ClusterSearcher(id, + qrsCfg, clusterCfg, documentDbCfg, emulationCfg, monitorCfg, new DispatchConfig(new DispatchConfig.Builder()), statMgr, listeners, new VipStatus()); + return searcher; + } + + private static class QueryTimeoutFixture { + ClusterSearcher searcher; + Execution exec; + Query query; + QueryTimeoutFixture(Double maxQueryTimeout, Double maxQueryCacheTimeout) { + searcher = createSearcher(maxQueryTimeout, maxQueryCacheTimeout); + exec = new Execution(searcher, Execution.Context.createContextStub()); + query = new Query("?query=hello&restrict=type1"); + } + void search() { + searcher.search(query, exec); + } + } + + public void testThatQueryTimeoutIsCappedWithDefaultMax() { + QueryTimeoutFixture f = new QueryTimeoutFixture(null, null); + f.query.setTimeout(600001); + f.search(); + assertEquals(600000, f.query.getTimeout()); + } + + public void testThatQueryTimeoutIsNotCapped() { + QueryTimeoutFixture f = new QueryTimeoutFixture(null, null); + f.query.setTimeout(599999); + f.search(); + assertEquals(599999, f.query.getTimeout()); + } + + public void testThatQueryTimeoutIsCappedWithSpecifiedMax() { + QueryTimeoutFixture f = new QueryTimeoutFixture(new Double(70), null); + f.query.setTimeout(70001); + f.search(); + assertEquals(70000, f.query.getTimeout()); + } + + public void testThatQueryCacheIsDisabledIfTimeoutIsLargerThanMax() { + QueryTimeoutFixture f = new QueryTimeoutFixture(null, null); + f.query.setTimeout(10001); + f.query.getRanking().setQueryCache(true); + f.search(); + assertFalse(f.query.getRanking().getQueryCache()); + } + + public void testThatQueryCacheIsNotDisabledIfTimeoutIsOk() { + QueryTimeoutFixture f = new QueryTimeoutFixture(null, null); + f.query.setTimeout(10000); + f.query.getRanking().setQueryCache(true); + f.search(); + assertTrue(f.query.getRanking().getQueryCache()); + } + + public void testThatQueryCacheIsDisabledIfTimeoutIsLargerThanConfiguredMax() { + QueryTimeoutFixture f = new QueryTimeoutFixture(null, new Double(5)); + f.query.setTimeout(5001); + f.query.getRanking().setQueryCache(true); + f.search(); + assertFalse(f.query.getRanking().getQueryCache()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/test/HasherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/test/HasherTestCase.java new file mode 100644 index 00000000000..527425eff4d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/test/HasherTestCase.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.cluster.test; + +import com.yahoo.container.handler.VipStatus; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.prelude.cluster.Hasher; +import com.yahoo.prelude.fastsearch.CacheKey; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests the Hashing/failover/whatever functionality. + * + * @author bratseth + * @author Steinar Knutsen + */ +public class HasherTestCase extends junit.framework.TestCase { + + public HasherTestCase (String name) { + super(name); + } + + public void testEmptyHasher() { + Hasher hasher=new Hasher(); + assertNull(hasher.select(0)); + } + + private static class MockBackend extends VespaBackEndSearcher { + + @Override + protected Result doSearch2(Query query, QueryPacket queryPacket, + CacheKey cacheKey, Execution execution) { + return null; + } + + @Override + protected void doPartialFill(Result result, String summaryClass) { + } + } + + public void testOneHasher() { + Hasher hasher = new Hasher(); + VespaBackEndSearcher o1 = new MockBackend(); + hasher.add(o1); + assertSame(o1, hasher.select(0)); + assertSame(o1, hasher.select(1)); + + hasher.remove(o1); + assertNull(hasher.select(0)); + } + + public void testAddAndRemove() { + Hasher hasher = new Hasher(); + VespaBackEndSearcher v0 = new MockBackend(); + VespaBackEndSearcher v1 = new MockBackend(); + VespaBackEndSearcher v2 = new MockBackend(); + VespaBackEndSearcher v3 = new MockBackend(); + v1.setLocalDispatching(false); + v3.setLocalDispatching(false); + hasher.add(v1); + hasher.add(v0); + assertSame(v0, hasher.select(0)); + hasher.add(v2); + VespaBackEndSearcher tmp1 = hasher.select(0); + assertTrue(v0 == tmp1 || v2 == tmp1); + if (tmp1 == v0) { + assertSame(v2, hasher.select(0)); + assertSame(v0, hasher.select(0)); + } else { + assertSame(v0, hasher.select(0)); + assertSame(v2, hasher.select(0)); + } + hasher.remove(v2); + hasher.remove(v2); + assertEquals(2, hasher.getNodeCount()); + assertSame(v0, hasher.select(0)); + hasher.remove(v0); + assertEquals(1, hasher.getNodeCount()); + assertSame(v1, hasher.select(0)); + hasher.add(v3); + hasher.add(v0); + assertSame(v0, hasher.select(0)); + } + + public void testPreferLocal() { + Hasher hasher = new Hasher(); + VespaBackEndSearcher v0 = new MockBackend(); + VespaBackEndSearcher v1 = new MockBackend(); + VespaBackEndSearcher v2 = new MockBackend(); + v1.setLocalDispatching(false); + v2.setLocalDispatching(false); + + hasher.add(v1); + hasher.add(v2); + hasher.add(v0); + assertTrue(hasher.select(0).isLocalDispatching()); + + hasher = new Hasher(); + hasher.add(v1); + hasher.add(v0); + hasher.add(v2); + assertTrue(hasher.select(0).isLocalDispatching()); + + + hasher = new Hasher(); + hasher.add(v0); + hasher.add(v1); + hasher.add(v2); + assertTrue(hasher.select(0).isLocalDispatching()); + + hasher = new Hasher(); + hasher.add(v0); + hasher.add(v1); + hasher.add(v2); + hasher.remove(v1); + assertTrue(hasher.select(0).isLocalDispatching()); + + hasher = new Hasher(); + hasher.add(v0); + hasher.add(v1); + assertTrue(hasher.select(0).isLocalDispatching()); + hasher.add(v2); + assertTrue(hasher.select(0).isLocalDispatching()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumFieldTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumFieldTestCase.java new file mode 100644 index 00000000000..a28f47fa3ca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/DocsumFieldTestCase.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import com.yahoo.prelude.fastsearch.DocsumField; +import com.yahoo.prelude.fastsearch.FastHit; + + +/** + * Tests DocsumField class functionality + * + * @author Bjørn Borud + */ +public class DocsumFieldTestCase extends junit.framework.TestCase { + + public DocsumFieldTestCase(String name) { + super(name); + } + + public void testConstructors() { + DocsumField.create("test", "string"); + DocsumField.create("test", "integer"); + DocsumField.create("test", "byte"); + DocsumField.create("test", "int64"); + } + + public void testByte() { + FastHit hit = new FastHit(); + DocsumField c = DocsumField.create("test", "byte"); + byte[] byteData = { 10, 20, 30, 40}; + ByteBuffer buffer = ByteBuffer.wrap(byteData); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + c.decode(buffer, hit); + assertEquals(1, buffer.position()); + assertEquals("10", hit.getField("test").toString()); + + c.decode(buffer, hit); + assertEquals(2, buffer.position()); + assertEquals("20", hit.getField("test").toString()); + + c.decode(buffer, hit); + assertEquals(3, buffer.position()); + assertEquals("30", hit.getField("test").toString()); + } + + public void testLongString() { + FastHit hit = new FastHit(); + DocsumField c = DocsumField.create("test", "longstring"); + byte[] byteData = { + 4, 0, 0, 0, 'c', 'a', 'f', 'e', 4, 0, 0, 0, 'B', 'A', + 'B', 'E'}; + ByteBuffer buffer = ByteBuffer.wrap(byteData); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + c.decode(buffer, hit); + assertEquals(8, buffer.position()); + assertEquals("cafe", hit.getField("test")); + + c.decode(buffer, hit); + assertEquals(16, buffer.position()); + assertEquals("BABE", hit.getField("test")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/FieldsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/FieldsTestCase.java new file mode 100644 index 00000000000..266025edde2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/FieldsTestCase.java @@ -0,0 +1,322 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.zip.Deflater; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.search.result.NanNumber; +import com.yahoo.text.Utf8; + +public class FieldsTestCase { + + ByteBuffer scratchSpace; + FastHit contains; + String fieldName = "field"; + + @Before + public void setUp() throws Exception { + scratchSpace = ByteBuffer.allocate(10000); + contains = new FastHit(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testByte() { + int s = scratchSpace.position(); + final byte value = (byte) 5; + scratchSpace.put(value); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new ByteField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new ByteField(fieldName).decode(scratchSpace, contains); + assertEquals(Byte.valueOf(value), contains.getField(fieldName)); + } + + @Test + public final void testData() { + String value = "nalle"; + int s = scratchSpace.position(); + scratchSpace.putShort((short) value.length()); + scratchSpace.put(Utf8.toBytes(value)); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new DataField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new DataField(fieldName).decode(scratchSpace, contains); + assertEquals(value, contains.getField(fieldName).toString()); + } + + @Test + public final void testDouble() { + int s = scratchSpace.position(); + final double value = 5.0d; + scratchSpace.putDouble(value); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new DoubleField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new DoubleField(fieldName).decode(scratchSpace, contains); + // slightly evil, but value is a exactly expressible as a double + assertEquals(Double.valueOf(value), contains.getField(fieldName)); + } + + @Test + public final void testFloat() { + int s = scratchSpace.position(); + final float value = 5.0f; + scratchSpace.putFloat(value); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new FloatField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new FloatField(fieldName).decode(scratchSpace, contains); + // slightly evil, but value is a exactly expressible as a float + assertEquals(Float.valueOf(value), contains.getField(fieldName)); + } + + @Test + public final void testInt64() { + int s = scratchSpace.position(); + final long value = 5; + scratchSpace.putLong(value); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new Int64Field(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new Int64Field(fieldName).decode(scratchSpace, contains); + assertEquals(Long.valueOf(value), contains.getField(fieldName)); + } + + @Test + public final void testInteger() { + int s = scratchSpace.position(); + final int value = 5; + scratchSpace.putInt(value); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new IntegerField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new IntegerField(fieldName).decode(scratchSpace, contains); + assertEquals(Integer.valueOf(value), contains.getField(fieldName)); + } + + @Test + public final void testNanExpressions() { + byte b = ByteField.EMPTY_VALUE; + short s = ShortField.EMPTY_VALUE; + int i = IntegerField.EMPTY_VALUE; + long l = Int64Field.EMPTY_VALUE; + assertFalse(((short) b) == s); + assertFalse(((int) s) == i); + assertFalse(((long) i) == l); + scratchSpace.put(b); + scratchSpace.putShort(s); + scratchSpace.putInt(i); + scratchSpace.putLong(l); + scratchSpace.putFloat(Float.NaN); + scratchSpace.putDouble(Double.NaN); + scratchSpace.flip(); + final String bytename = fieldName + "_b"; + new ByteField(bytename).decode(scratchSpace, contains); + final String shortname = fieldName + "_s"; + new ShortField(shortname).decode(scratchSpace, contains); + final String intname = fieldName + "_i"; + new IntegerField(intname).decode(scratchSpace, contains); + final String longname = fieldName + "_l"; + new Int64Field(longname).decode(scratchSpace, contains); + final String floatname = fieldName + "_f"; + new FloatField(floatname).decode(scratchSpace, contains); + final String doublename = fieldName + "_d"; + new DoubleField(doublename).decode(scratchSpace, contains); + assertSame(NanNumber.NaN, contains.getField(bytename)); + assertSame(NanNumber.NaN, contains.getField(shortname)); + assertSame(NanNumber.NaN, contains.getField(intname)); + assertSame(NanNumber.NaN, contains.getField(longname)); + assertSame(NanNumber.NaN, contains.getField(floatname)); + assertSame(NanNumber.NaN, contains.getField(doublename)); + } + + @Test + public final void testJSON() { + String value = "{1: 2}"; + int s = scratchSpace.position(); + scratchSpace.putInt(value.length()); + scratchSpace.put(Utf8.toBytes(value)); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new JSONField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new JSONField(fieldName).decode(scratchSpace, contains); + assertEquals(value, ((JSONString) contains.getField(fieldName)).getContent()); + } + + @Test + public final void testLongdata() { + String value = "nalle"; + int s = scratchSpace.position(); + scratchSpace.putInt(value.length()); + scratchSpace.put(Utf8.toBytes(value)); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new LongdataField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new LongdataField(fieldName).decode(scratchSpace, contains); + assertEquals(value, contains.getField(fieldName).toString()); + } + + @Test + public final void testLongstring() { + String value = "nalle"; + int s = scratchSpace.position(); + scratchSpace.putInt(value.length()); + scratchSpace.put(Utf8.toBytes(value)); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new LongstringField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new LongstringField(fieldName).decode(scratchSpace, contains); + assertEquals(value, contains.getField(fieldName)); + } + + @Test + public final void testShort() { + int s = scratchSpace.position(); + final short value = 5; + scratchSpace.putShort(value); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new ShortField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new ShortField(fieldName).decode(scratchSpace, contains); + assertEquals(Short.valueOf(value), contains.getField(fieldName)); + } + + @Test + public final void testString() { + String value = "nalle"; + int s = scratchSpace.position(); + scratchSpace.putShort((short) value.length()); + scratchSpace.put(Utf8.toBytes(value)); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new StringField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new StringField(fieldName).decode(scratchSpace, contains); + assertEquals(value, contains.getField(fieldName)); + } + + @Test + public final void testXML() { + String value = "nalle"; + int s = scratchSpace.position(); + scratchSpace.putInt(value.length()); + scratchSpace.put(Utf8.toBytes(value)); + int l = scratchSpace.position(); + scratchSpace.flip(); + assertEquals(l, new XMLField(fieldName).getLength(scratchSpace)); + scratchSpace.position(s); + new XMLField(fieldName).decode(scratchSpace, contains); + assertTrue(contains.getField(fieldName).getClass() == XMLString.class); + assertEquals(value, contains.getField(fieldName).toString()); + } + + @Test + public final void testCompressionLongdata() { + String value = "000000000000000000000000000000000000000000000000000000000000000"; + byte[] raw = Utf8.toBytesStd(value); + byte[] output = new byte[raw.length * 2]; + Deflater compresser = new Deflater(); + compresser.setInput(raw); + compresser.finish(); + int compressedDataLength = compresser.deflate(output); + compresser.end(); + scratchSpace.putInt((compressedDataLength + 4) | (1 << 31)); + scratchSpace.putInt(raw.length); + scratchSpace.put(output, 0, compressedDataLength); + scratchSpace.flip(); + assertTrue(new LongdataField(fieldName).isCompressed(scratchSpace)); + new LongdataField(fieldName).decode(scratchSpace, contains); + assertEquals(value, contains.getField(fieldName).toString()); + } + + @Test + public final void testCompressionJson() { + String value = "{0:000000000000000000000000000000000000000000000000000000000000000}"; + byte[] raw = Utf8.toBytesStd(value); + byte[] output = new byte[raw.length * 2]; + Deflater compresser = new Deflater(); + compresser.setInput(raw); + compresser.finish(); + int compressedDataLength = compresser.deflate(output); + compresser.end(); + scratchSpace.putInt((compressedDataLength + 4) | (1 << 31)); + scratchSpace.putInt(raw.length); + scratchSpace.put(output, 0, compressedDataLength); + scratchSpace.flip(); + assertTrue(new JSONField(fieldName).isCompressed(scratchSpace)); + new JSONField(fieldName).decode(scratchSpace, contains); + assertEquals(value, ((JSONString) contains.getField(fieldName)).getContent()); + } + + @Test + public final void testCompressionLongstring() { + String value = "000000000000000000000000000000000000000000000000000000000000000"; + byte[] raw = Utf8.toBytesStd(value); + byte[] output = new byte[raw.length * 2]; + Deflater compresser = new Deflater(); + compresser.setInput(raw); + compresser.finish(); + int compressedDataLength = compresser.deflate(output); + compresser.end(); + scratchSpace.putInt((compressedDataLength + 4) | (1 << 31)); + scratchSpace.putInt(raw.length); + scratchSpace.put(output, 0, compressedDataLength); + scratchSpace.flip(); + assertTrue(new LongstringField(fieldName).isCompressed(scratchSpace)); + new LongstringField(fieldName).decode(scratchSpace, contains); + assertEquals(value, contains.getField(fieldName)); + } + + @Test + public final void testCompressionXml() { + String value = "000000000000000000000000000000000000000000000000000000000000000"; + byte[] raw = Utf8.toBytesStd(value); + byte[] output = new byte[raw.length * 2]; + Deflater compresser = new Deflater(); + compresser.setInput(raw); + compresser.finish(); + int compressedDataLength = compresser.deflate(output); + compresser.end(); + scratchSpace.putInt((compressedDataLength + 4) | (1 << 31)); + scratchSpace.putInt(raw.length); + scratchSpace.put(output, 0, compressedDataLength); + scratchSpace.flip(); + assertTrue(new XMLField(fieldName).isCompressed(scratchSpace)); + new XMLField(fieldName).decode(scratchSpace, contains); + assertTrue(contains.getField(fieldName).getClass() == XMLString.class); + assertEquals(value, contains.getField(fieldName).toString()); + + } + + @Test + public final void checkLengthFieldLengths() { + assertEquals(2, new DataField(fieldName).sizeOfLength()); + assertEquals(4, new JSONField(fieldName).sizeOfLength()); + assertEquals(4, new LongdataField(fieldName).sizeOfLength()); + assertEquals(4, new LongstringField(fieldName).sizeOfLength()); + assertEquals(2, new StringField(fieldName).sizeOfLength()); + assertEquals(4, new XMLField(fieldName).sizeOfLength()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/JsonFieldTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/JsonFieldTestCase.java new file mode 100644 index 00000000000..8f63beb747b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/JsonFieldTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + +import static org.junit.Assert.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.data.access.*; +import com.yahoo.data.access.simple.*; + +public class JsonFieldTestCase { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void requireThatWeightedSetsItemsAreConvertedToStrings() { + Value.ArrayValue topArr = new Value.ArrayValue(); + topArr.add(new Value.ArrayValue() + .add(new Value.DoubleValue(17.5)) + .add(new Value.LongValue(10))); + topArr.add(new Value.ArrayValue() + .add(new Value.DoubleValue(0.25)) + .add(new Value.DoubleValue(20))); + + Inspector c = JSONField.convertTop(topArr); + + assertEquals(Type.STRING, c.entry(0).entry(0).type()); + assertEquals(Type.LONG, c.entry(0).entry(1).type()); + assertEquals(Type.STRING, c.entry(1).entry(0).type()); + assertEquals(Type.DOUBLE, c.entry(1).entry(1).type()); + + assertEquals("17.5", c.entry(0).entry(0).asString()); + assertEquals(10, c.entry(0).entry(1).asLong()); + assertEquals("0.25", c.entry(1).entry(0).asString()); + assertEquals(20.0, c.entry(1).entry(1).asDouble(), 0.01); + } + + @Test + public final void requireThatNewWeightedSetsAreConvertedToOldFormat() { + Value.ArrayValue topArr = new Value.ArrayValue(); + topArr.add(new Value.ObjectValue() + .put("item", new Value.DoubleValue(17.5)) + .put("weight", new Value.LongValue(10))); + topArr.add(new Value.ObjectValue() + .put("item", new Value.DoubleValue(0.25)) + .put("weight", new Value.DoubleValue(20))); + topArr.add(new Value.ObjectValue() + .put("item", new Value.StringValue("foob")) + .put("weight", new Value.DoubleValue(30))); + + Inspector c = JSONField.convertTop(topArr); + + assertEquals(Type.STRING, c.entry(0).entry(0).type()); + assertEquals(Type.LONG, c.entry(0).entry(1).type()); + assertEquals(Type.STRING, c.entry(1).entry(0).type()); + assertEquals(Type.DOUBLE, c.entry(1).entry(1).type()); + assertEquals(Type.STRING, c.entry(2).entry(0).type()); + assertEquals(Type.DOUBLE, c.entry(2).entry(1).type()); + + assertEquals("17.5", c.entry(0).entry(0).asString()); + assertEquals(10, c.entry(0).entry(1).asLong()); + assertEquals("0.25", c.entry(1).entry(0).asString()); + assertEquals(20.0, c.entry(1).entry(1).asDouble(), 0.01); + assertEquals("foob", c.entry(2).entry(0).asString()); + assertEquals(30.0, c.entry(2).entry(1).asDouble(), 0.01); + } + + @Test + public final void requireThatArrayValuesAreConvertedToStrings() { + Value.ArrayValue topArr = new Value.ArrayValue(); + topArr.add(new Value.DoubleValue(17.5)); + topArr.add(new Value.DoubleValue(0.25)); + topArr.add(new Value.LongValue(10)); + topArr.add(new Value.DoubleValue(20)); + + Inspector c = JSONField.convertTop(topArr); + + assertEquals(Type.STRING, c.entry(0).type()); + assertEquals(Type.STRING, c.entry(1).type()); + assertEquals(Type.STRING, c.entry(2).type()); + assertEquals(Type.STRING, c.entry(3).type()); + + assertEquals("17.5", c.entry(0).asString()); + assertEquals("0.25", c.entry(1).asString()); + assertEquals("10", c.entry(2).asString()); + assertEquals("20.0", c.entry(3).asString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java new file mode 100644 index 00000000000..47a3003371e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch; + + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.search.LegacyEmulationConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.fastsearch.Docsum; +import com.yahoo.prelude.fastsearch.DocsumDefinition; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.RawData; +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.result.NanNumber; +import com.yahoo.search.result.StructuredData; +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.slime.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import java.nio.charset.StandardCharsets; + +import org.junit.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.*; + + +public class SlimeSummaryTestCase { + + public static DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); + } + + public static DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); + } + + public byte[] makeEmptyDocsum() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + + public byte[] makeDocsum() { + Slime slime = new Slime(); + Cursor docsum = slime.setObject(); + docsum.setLong("integer_field", 4); + docsum.setLong("short_field", 2); + docsum.setLong("byte_field", 1); + docsum.setDouble("float_field", 4.5); + docsum.setDouble("double_field", 8.75); + docsum.setLong("int64_field", 8); + docsum.setString("string_field", "string_value"); + docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8)); + docsum.setString("longstring_field", "longstring_value"); + docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8)); + docsum.setString("xmlstring_field", "xmlstring_value"); + { + Cursor field = docsum.setObject("jsonstring_field"); + field.setLong("foo", 1); + field.setLong("bar", 2); + } + byte[] tmp = BinaryFormat.encode(slime); + ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID); + buf.order(ByteOrder.BIG_ENDIAN); + buf.put(tmp); + return buf.array(); + } + + @Test + public void testDecodingEmpty() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + LegacyEmulationConfig emul = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, emul); + byte[] docsum = makeEmptyDocsum(); + FastHit hit = new FastHit(); + set.lazyDecode("default", docsum, hit); + assertThat(hit.getField("integer_field"), equalTo((Object) NanNumber.NaN)); + assertThat(hit.getField("short_field"), equalTo((Object) NanNumber.NaN)); + assertThat(hit.getField("byte_field"), equalTo((Object) NanNumber.NaN)); + assertThat(hit.getField("float_field"), equalTo((Object) NanNumber.NaN)); + assertThat(hit.getField("double_field"), equalTo((Object) NanNumber.NaN)); + assertThat(hit.getField("int64_field"), equalTo((Object) NanNumber.NaN)); + assertThat(hit.getField("string_field"), equalTo((Object)"")); + assertThat(hit.getField("data_field"), instanceOf(RawData.class)); + assertThat(hit.getField("data_field").toString(), equalTo("")); + assertThat(hit.getField("longstring_field"), equalTo((Object)"")); + assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); + assertThat(hit.getField("longdata_field").toString(), equalTo("")); + assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); + assertThat(hit.getField("xmlstring_field").toString(), equalTo("")); + // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); + assertThat(hit.getField("jsonstring_field").toString(), equalTo("")); + } + + @Test + public void testDecodingEmptyWithoutForcedFill() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(false))); + byte[] docsum = makeEmptyDocsum(); + FastHit hit = new FastHit(); + set.lazyDecode("default", docsum, hit); + assertThat(hit.getField("integer_field"), equalTo(null)); + assertThat(hit.getField("short_field"), equalTo(null)); + assertThat(hit.getField("byte_field"), equalTo(null)); + assertThat(hit.getField("float_field"), equalTo(null)); + assertThat(hit.getField("double_field"), equalTo(null)); + assertThat(hit.getField("int64_field"), equalTo(null)); + assertThat(hit.getField("string_field"), equalTo(null)); + assertThat(hit.getField("data_field"), equalTo(null)); + assertThat(hit.getField("data_field"), equalTo(null)); + assertThat(hit.getField("longstring_field"), equalTo(null)); + assertThat(hit.getField("longdata_field"), equalTo(null)); + assertThat(hit.getField("longdata_field"), equalTo(null)); + assertThat(hit.getField("xmlstring_field"), equalTo(null)); + assertThat(hit.getField("xmlstring_field"), equalTo(null)); + assertThat(hit.getField("jsonstring_field"), equalTo(null)); + } + + @Test + public void testDecoding() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + byte[] docsum = makeDocsum(); + FastHit hit = new FastHit(); + set.lazyDecode("default", docsum, hit); + assertThat(hit.getField("integer_field"), equalTo((Object)new Integer(4))); + assertThat(hit.getField("short_field"), equalTo((Object)new Short((short)2))); + assertThat(hit.getField("byte_field"), equalTo((Object)new Byte((byte)1))); + assertThat(hit.getField("float_field"), equalTo((Object)new Float(4.5f))); + assertThat(hit.getField("double_field"), equalTo((Object)new Double(8.75))); + assertThat(hit.getField("int64_field"), equalTo((Object)new Long(8L))); + assertThat(hit.getField("string_field"), equalTo((Object)"string_value")); + assertThat(hit.getField("data_field"), instanceOf(RawData.class)); + assertThat(hit.getField("data_field").toString(), equalTo("data_value")); + assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value")); + assertThat(hit.getField("longdata_field"), instanceOf(RawData.class)); + assertThat(hit.getField("longdata_field").toString(), equalTo("longdata_value")); + assertThat(hit.getField("xmlstring_field"), instanceOf(XMLString.class)); + assertThat(hit.getField("xmlstring_field").toString(), equalTo("xmlstring_value")); + if (hit.getField("jsonstring_field") instanceof JSONString) { + JSONString jstr = (JSONString) hit.getField("jsonstring_field"); + assertThat(jstr.getContent(), equalTo("{\"foo\":1,\"bar\":2}")); + assertThat(jstr.getParsedJSON(), notNullValue()); + + com.yahoo.data.access.Inspectable obj = jstr; + com.yahoo.data.access.Inspector value = obj.inspect(); + assertThat(value.field("foo").asLong(), equalTo(1L)); + assertThat(value.field("bar").asLong(), equalTo(2L)); + } else { + StructuredData sdata = (StructuredData) hit.getField("jsonstring_field"); + assertThat(sdata.toJson(), equalTo("{\"foo\":1,\"bar\":2}")); + + com.yahoo.data.access.Inspectable obj = sdata; + com.yahoo.data.access.Inspector value = obj.inspect(); + assertThat(value.field("foo").asLong(), equalTo(1L)); + assertThat(value.field("bar").asLong(), equalTo(2L)); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg new file mode 100644 index 00000000000..a188754db19 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg @@ -0,0 +1,30 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[1] +documentdb[0].summaryclass[0].name default +documentdb[0].summaryclass[0].id 0 +documentdb[0].summaryclass[0].fields[12] +documentdb[0].summaryclass[0].fields[0].name integer_field +documentdb[0].summaryclass[0].fields[0].type integer +documentdb[0].summaryclass[0].fields[1].name short_field +documentdb[0].summaryclass[0].fields[1].type short +documentdb[0].summaryclass[0].fields[2].name byte_field +documentdb[0].summaryclass[0].fields[2].type byte +documentdb[0].summaryclass[0].fields[3].name float_field +documentdb[0].summaryclass[0].fields[3].type float +documentdb[0].summaryclass[0].fields[4].name double_field +documentdb[0].summaryclass[0].fields[4].type double +documentdb[0].summaryclass[0].fields[5].name int64_field +documentdb[0].summaryclass[0].fields[5].type int64 +documentdb[0].summaryclass[0].fields[6].name string_field +documentdb[0].summaryclass[0].fields[6].type string +documentdb[0].summaryclass[0].fields[7].name data_field +documentdb[0].summaryclass[0].fields[7].type data +documentdb[0].summaryclass[0].fields[8].name longstring_field +documentdb[0].summaryclass[0].fields[8].type longstring +documentdb[0].summaryclass[0].fields[9].name longdata_field +documentdb[0].summaryclass[0].fields[9].type longdata +documentdb[0].summaryclass[0].fields[10].name xmlstring_field +documentdb[0].summaryclass[0].fields[10].type xmlstring +documentdb[0].summaryclass[0].fields[11].name jsonstring_field +documentdb[0].summaryclass[0].fields[11].type jsonstring diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java new file mode 100644 index 00000000000..20f8c33cb7d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/CacheKeyTestCase.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + + +import com.yahoo.fs4.QueryPacket; +import com.yahoo.search.Query; +import com.yahoo.prelude.fastsearch.CacheKey; + + +/** + * @author Steinar Knutsen + */ +public class CacheKeyTestCase extends junit.framework.TestCase { + + public CacheKeyTestCase(String name) { + super(name); + } + + public void testHitsOffsetEquality() { + Query a = new Query("/?query=abcd"); + QueryPacket p1 = QueryPacket.create(a); + a.setWindow(100, 1000); + QueryPacket p2 = QueryPacket.create(a); + CacheKey k1 = new CacheKey(p1); + CacheKey k2 = new CacheKey(p2); + assertEquals(k1, k2); + assertEquals(k1.hashCode(), k2.hashCode()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DispatchThread.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DispatchThread.java new file mode 100644 index 00000000000..d70aa20ac35 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DispatchThread.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// -*- mode: java; folded-file: t; c-basic-offset: 4 -*- +// +// +package com.yahoo.prelude.fastsearch.test; + + +import com.yahoo.prelude.ConfigurationException; + + +/** + * Thread-wrapper for MockFDispatch + * + * @author Bjorn Borud + */ +public class DispatchThread extends Thread { + int listenPort; + long replyDelay; + long byteDelay; + MockFDispatch dispatch; + Object barrier = new Object(); + + /** + * Instantiate MockFDispatch; if the wanted port is taken we + * bump the port number. Note that the delays are not + * accurate: in reality they will be significantly longer for + * low values. + * + * @param listenPort Wanted port number, note that this may be + * bumped if someone is already running something + * on this port, so it is a starting point for + * scanning only + * @param replyDelay how many milliseconds we should delay when + * replying + * @param byteDelay how many milliseconds we delay for each byte + * written + */ + + public DispatchThread(int listenPort, long replyDelay, long byteDelay) { + this.listenPort = listenPort; + this.replyDelay = replyDelay; + this.byteDelay = byteDelay; + dispatch = new MockFDispatch(listenPort, replyDelay, byteDelay); + dispatch.setBarrier(barrier); + } + + /** + * Run the MockFDispatch and anticipate multiple instances of + * same running. + */ + public void run() { + int maxTries = 20; + // the following section is here to make sure that this + // test is somewhat robust, ie. if someone is already + // listening to the port in question, we'd like to NOT + // fail, but keep probing until we find a port we can use. + boolean up = false; + + while ((!up) && (maxTries-- != 0)) { + try { + dispatch.run(); + up = true; + } catch (ConfigurationException e) { + listenPort++; + dispatch.setListenPort(listenPort); + } + } + } + + /** + * Wait until MockFDispatch is ready to accept connections + * or we time out and indicate which of the two outcomes it was. + * + * @return If we time out we return false. Else we + * return true + * + */ + public boolean waitOnBarrier(long timeout) throws InterruptedException { + long start = System.currentTimeMillis(); + + synchronized (barrier) { + barrier.wait(timeout); + } + long diff = System.currentTimeMillis() - start; + + return (diff < timeout); + } + + /** + * Return the port on which the MockFDispatch actually listens. + * use this instead of assuming where it is since, if more than + * one application tries to use the port we've assigned to it + * we might have to up the port number. + * + * @return port number of active MockFDispatch instance + * + */ + public int listenPort() { + return listenPort; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java new file mode 100644 index 00000000000..9ecadc5a479 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DocsumDefinitionTestCase.java @@ -0,0 +1,394 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.fastsearch.ByteField; +import com.yahoo.prelude.fastsearch.DataField; +import com.yahoo.prelude.fastsearch.Docsum; +import com.yahoo.prelude.fastsearch.DocsumDefinition; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.IntegerField; +import com.yahoo.prelude.fastsearch.StringField; +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; + + +/** + * Tests docsum class functionality + * + * @author bratseth + */ +public class DocsumDefinitionTestCase extends junit.framework.TestCase { + + public DocsumDefinitionTestCase(String name) { + super(name); + } + + public void testReading() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + + String[] defs = new String[] { "[0,default]", "[1,version1]", + "[237,withranklog]", "[2,version2]", "[3,version3]", + "[4,version4]", "[5,version5]" }; + String setAsString = set.toString(); + for (String d : defs) { + assertFalse(setAsString.indexOf(d) == -1); + } + assertEquals(7, set.size()); + + DocsumDefinition docsum0 = set.getDocsumDefinition(0); + + assertNotNull(docsum0); + assertEquals("default", docsum0.getName()); + assertEquals(19, docsum0.getFieldCount()); + assertNull(docsum0.getField(19)); + assertEquals("DSHOST", docsum0.getField(7).getName()); + + assertTrue(docsum0.getField(1) instanceof StringField); + assertTrue(docsum0.getField(6) instanceof ByteField); + assertTrue(docsum0.getField(7) instanceof IntegerField); + assertTrue(docsum0.getField(18) instanceof DataField); + } + + public void testDecoding() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + FastHit hit = new FastHit(); + + set.lazyDecode(null, docsum4, hit); + assertEquals("Arts/Celebrities/Madonna", hit.getField("TOPIC")); + assertEquals("1", hit.getField("EXTINFOSOURCE").toString()); + assertEquals("10", hit.getField("LANG1").toString()); + assertEquals("352", hit.getField("WORDS").toString()); + assertEquals("index:0/0/0/" + FastHit.asHexString(hit.getGlobalId()), hit.getId().toString()); + } + + public void testDecodingCompressed() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + FastHit hit = new FastHit(); + + set.lazyDecode(null, docsum5, hit); + assertEquals("Madonna", hit.getField("TITLE")); + assertEquals(561, ((String) hit.getField("DYNTEASER")).length()); + } + + public void testLazyDecoding() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + DocsumDefinition def = set.getDocsumDefinition(4); + Docsum sum = new Docsum(def, docsum4); + FastHit hit = new FastHit(); + hit.addSummary(sum); + + assertEquals("Arts/Celebrities/Madonna", hit.getField("TOPIC").toString()); + assertEquals("1", hit.getField("EXTINFOSOURCE").toString()); + assertEquals("10", hit.getField("LANG1").toString()); + assertEquals("352", hit.getField("WORDS").toString()); + assertEquals("index:0/0/0/" + FastHit.asHexString(hit.getGlobalId()), hit.getId().toString()); + } + + public void testLazyDecodingCompressed() { + String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"; + DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf); + DocsumDefinition def = set.getDocsumDefinition(5); + Docsum sum = new Docsum(def, docsum5); + FastHit hit = new FastHit(); + hit.addSummary(sum); + + assertEquals("Madonna", hit.getField("TITLE")); + assertEquals(561, ((String) hit.getField("DYNTEASER")).length()); + + } + + public static GlobalId createGlobalId(int docId) { + return new GlobalId((new DocumentId("doc:test:" + docId)).getGlobalId()); + } + + public static byte[] docsum4 = { + (byte) 0x04, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x1e, (byte) 0x00, + (byte) 0x68, // 4, 104, 0, 'h' + (byte) 0x74, (byte) 0x74, (byte) 0x70, (byte) 0x3a, (byte) 0x2f, + (byte) 0x2f, (byte) 0x77, (byte) 0x77, (byte) 0x77, (byte) 0x2e, + (byte) 0x73, (byte) 0x74, (byte) 0x75, (byte) 0x64, (byte) 0x79, + (byte) 0x6f, (byte) 0x66, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x2e, + (byte) 0x63, (byte) 0x6f, (byte) 0x6d, (byte) 0x2f, (byte) 0x00, + (byte) 0x00, (byte) 0x4d, (byte) 0x00, (byte) 0x53, (byte) 0x74, + (byte) 0x75, (byte) 0x64, (byte) 0x79, (byte) 0x4f, (byte) 0x66, + (byte) 0x4d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x2e, (byte) 0x63, (byte) 0x6f, + (byte) 0x6d, (byte) 0x20, (byte) 0x2d, (byte) 0x20, (byte) 0x49, + (byte) 0x6e, (byte) 0x74, (byte) 0x65, (byte) 0x72, (byte) 0x76, + (byte) 0x69, (byte) 0x65, (byte) 0x77, (byte) 0x73, (byte) 0x2c, + (byte) 0x20, (byte) 0x41, (byte) 0x72, (byte) 0x74, (byte) 0x69, + (byte) 0x63, (byte) 0x6c, (byte) 0x65, (byte) 0x73, (byte) 0x2c, + (byte) 0x20, (byte) 0x52, (byte) 0x65, (byte) 0x76, (byte) 0x69, + (byte) 0x65, (byte) 0x77, (byte) 0x73, (byte) 0x2c, (byte) 0x20, + (byte) 0x51, (byte) 0x75, (byte) 0x6f, (byte) 0x74, (byte) 0x65, + (byte) 0x73, (byte) 0x2c, (byte) 0x20, (byte) 0x45, (byte) 0x73, + (byte) 0x73, (byte) 0x61, (byte) 0x79, (byte) 0x73, (byte) 0x20, + (byte) 0x61, (byte) 0x6e, (byte) 0x64, (byte) 0x20, (byte) 0x6d, + (byte) 0x6f, (byte) 0x72, (byte) 0x65, (byte) 0x2e, (byte) 0x2e, + (byte) 0xfd, (byte) 0x00, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x61, (byte) 0x2e, (byte) 0x2e, (byte) 0x2e, + (byte) 0x18, (byte) 0x00, (byte) 0x41, (byte) 0x72, (byte) 0x74, + (byte) 0x73, (byte) 0x2f, (byte) 0x43, (byte) 0x65, (byte) 0x6c, + (byte) 0x65, (byte) 0x62, (byte) 0x72, (byte) 0x69, (byte) 0x74, + (byte) 0x69, (byte) 0x65, (byte) 0x73, (byte) 0x2f, (byte) 0x4d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x13, (byte) 0x00, (byte) 0x43, (byte) 0x65, + (byte) 0x6c, (byte) 0x65, (byte) 0x62, (byte) 0x72, (byte) 0x69, + (byte) 0x74, (byte) 0x69, (byte) 0x65, (byte) 0x73, (byte) 0x2f, + (byte) 0x4d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x4f, (byte) 0x00, (byte) 0x55, + (byte) 0x73, (byte) 0x65, (byte) 0x73, (byte) 0x20, (byte) 0x69, + (byte) 0x6e, (byte) 0x74, (byte) 0x65, (byte) 0x72, (byte) 0x76, + (byte) 0x69, (byte) 0x65, (byte) 0x77, (byte) 0x73, (byte) 0x2c, + (byte) 0x20, (byte) 0x61, (byte) 0x72, (byte) 0x74, (byte) 0x69, + (byte) 0x63, (byte) 0x6c, (byte) 0x65, (byte) 0x73, (byte) 0x2c, + (byte) 0x20, (byte) 0x61, (byte) 0x6e, (byte) 0x64, (byte) 0x20, + (byte) 0x71, (byte) 0x75, (byte) 0x6f, (byte) 0x74, (byte) 0x65, + (byte) 0x73, (byte) 0x20, (byte) 0x74, (byte) 0x6f, (byte) 0x20, + (byte) 0x73, (byte) 0x74, (byte) 0x75, (byte) 0x64, (byte) 0x79, + (byte) 0x20, (byte) 0x68, (byte) 0x6f, (byte) 0x77, (byte) 0x20, + (byte) 0x4d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x68, (byte) 0x61, + (byte) 0x73, (byte) 0x20, (byte) 0x63, (byte) 0x68, (byte) 0x61, + (byte) 0x6e, (byte) 0x67, (byte) 0x65, (byte) 0x64, (byte) 0x20, + (byte) 0x63, (byte) 0x75, (byte) 0x6c, (byte) 0x74, (byte) 0x75, + (byte) 0x72, (byte) 0x65, (byte) 0x2e, (byte) 0x01, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x76, (byte) 0x16, + (byte) 0x32, (byte) 0x00, (byte) 0xe6, (byte) 0x23, (byte) 0x00, + (byte) 0x00, (byte) 0x60, (byte) 0x01, (byte) 0x00, (byte) 0x00, + (byte) 0x2c, (byte) 0xcb, (byte) 0x70, (byte) 0x3e, (byte) 0x2c, + (byte) 0xcb, (byte) 0x70, (byte) 0x3e, (byte) 0x0a, (byte) 0xff, + (byte) 0xff, (byte) 0xff, (byte) 0x2e, (byte) 0xd3, (byte) 0x3a, + (byte) 0xa1, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0xfd, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x1c, (byte) 0x69, + (byte) 0x6e, (byte) 0x74, (byte) 0x6f, (byte) 0x20, (byte) 0x6f, + (byte) 0x6e, (byte) 0x65, (byte) 0x20, (byte) 0x77, (byte) 0x65, + (byte) 0x62, (byte) 0x73, (byte) 0x69, (byte) 0x74, (byte) 0x65, + (byte) 0x20, (byte) 0x6f, (byte) 0x6e, (byte) 0x20, (byte) 0x68, + (byte) 0x6f, (byte) 0x77, (byte) 0x20, (byte) 0x02, (byte) 0x4d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x68, (byte) 0x61, + (byte) 0x73, (byte) 0x20, (byte) 0x61, (byte) 0x6e, (byte) 0x64, + (byte) 0x20, (byte) 0x63, (byte) 0x6f, (byte) 0x6e, (byte) 0x74, + (byte) 0x69, (byte) 0x6e, (byte) 0x75, (byte) 0x65, (byte) 0x73, + (byte) 0x20, (byte) 0x74, (byte) 0x6f, (byte) 0x20, (byte) 0x73, + (byte) 0x68, (byte) 0x61, (byte) 0x70, (byte) 0x65, (byte) 0x1c, + (byte) 0x74, (byte) 0x68, (byte) 0x65, (byte) 0x20, (byte) 0x73, + (byte) 0x69, (byte) 0x74, (byte) 0x65, (byte) 0x20, (byte) 0x66, + (byte) 0x75, (byte) 0x6c, (byte) 0x6c, (byte) 0x20, (byte) 0x6a, + (byte) 0x75, (byte) 0x73, (byte) 0x74, (byte) 0x69, (byte) 0x63, + (byte) 0x65, (byte) 0x20, (byte) 0x2e, (byte) 0x2e, (byte) 0x20, + (byte) 0x54, (byte) 0x68, (byte) 0x69, (byte) 0x73, (byte) 0x20, + (byte) 0x4f, (byte) 0x72, (byte) 0x69, (byte) 0x67, (byte) 0x69, + (byte) 0x6e, (byte) 0x61, (byte) 0x6c, (byte) 0x20, (byte) 0x02, + (byte) 0x4d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x57, + (byte) 0x65, (byte) 0x62, (byte) 0x72, (byte) 0x69, (byte) 0x6e, + (byte) 0x67, (byte) 0x20, (byte) 0x73, (byte) 0x69, (byte) 0x74, + (byte) 0x65, (byte) 0x20, (byte) 0x6f, (byte) 0x77, (byte) 0x6e, + (byte) 0x65, (byte) 0x64, (byte) 0x20, (byte) 0x62, (byte) 0x79, + (byte) 0x20, (byte) 0x4a, (byte) 0x65, (byte) 0x6e, (byte) 0x6e, + (byte) 0x69, (byte) 0x66, (byte) 0x65, (byte) 0x72, (byte) 0x20, + (byte) 0x57, (byte) 0x61, (byte) 0x6c, (byte) 0x6c, (byte) 0x72, + (byte) 0x61, (byte) 0x66, (byte) 0x66, (byte) 0x1c, (byte) 0x02, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x02, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x02, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x02, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x02, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x02, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x6e, (byte) 0x61, (byte) 0x03, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, + (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, + (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, + (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, + (byte) 0x6d, (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, + (byte) 0x61, (byte) 0x20, (byte) 0x6d, (byte) 0x61, (byte) 0x64, + (byte) 0x6f, (byte) 0x6e, (byte) 0x61, (byte) 0x20, (byte) 0x6d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x61, + (byte) 0x1c}; + + // this has compressed fields + public static byte[] docsum5 = { + (byte) 0x05, (byte) 0x00, (byte) 0x00, (byte) 0x00, // 5, 41, 0, 'h', 't', 't' + (byte) 0x29, (byte) 0x00, (byte) 0x68, (byte) 0x74, (byte) 0x74, + (byte) 0x70, (byte) 0x3a, (byte) 0x2f, (byte) 0x2f, (byte) 0x77, + (byte) 0x77, (byte) 0x77, (byte) 0x2e, (byte) 0x68, (byte) 0x75, + (byte) 0x6e, (byte) 0x67, (byte) 0x61, (byte) 0x72, (byte) 0x69, + (byte) 0x61, (byte) 0x6e, (byte) 0x62, (byte) 0x65, (byte) 0x73, + (byte) 0x74, (byte) 0x2e, (byte) 0x63, (byte) 0x6f, (byte) 0x6d, + (byte) 0x2f, (byte) 0x4a, (byte) 0x6f, (byte) 0x7a, (byte) 0x73, + (byte) 0x61, (byte) 0x2f, (byte) 0x37, (byte) 0x2e, (byte) 0x68, + (byte) 0x74, (byte) 0x6d, (byte) 0x6c, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x07, (byte) 0x00, (byte) 0x4d, + (byte) 0x61, (byte) 0x64, (byte) 0x6f, (byte) 0x6e, (byte) 0x6e, + (byte) 0x61, (byte) 0x79, (byte) 0x00, (byte) 0x41, (byte) 0x6d, + (byte) 0x65, (byte) 0x6e, (byte) 0x6e, (byte) 0x79, (byte) 0x69, + (byte) 0x62, (byte) 0x65, (byte) 0x6e, (byte) 0x20, (byte) 0x6d, + (byte) 0xc3, (byte) 0xa1, (byte) 0x73, (byte) 0x20, (byte) 0x6f, + (byte) 0x72, (byte) 0x73, (byte) 0x7a, (byte) 0xc3, (byte) 0xa1, + (byte) 0x67, (byte) 0x62, (byte) 0x61, (byte) 0x20, (byte) 0x6b, + (byte) 0xc3, (byte) 0xad, (byte) 0x76, (byte) 0xc3, (byte) 0xa1, + (byte) 0x6e, (byte) 0x6a, (byte) 0x61, (byte) 0x20, (byte) 0x20, + (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, + (byte) 0x61, (byte) 0x20, (byte) 0x74, (byte) 0x65, (byte) 0x72, + (byte) 0x6d, (byte) 0xc3, (byte) 0xa9, (byte) 0x6b, (byte) 0x65, + (byte) 0x74, (byte) 0x20, (byte) 0x6b, (byte) 0xc3, (byte) 0xbc, + (byte) 0x6c, (byte) 0x64, (byte) 0x65, (byte) 0x6e, (byte) 0x69, + (byte) 0x2c, (byte) 0x20, (byte) 0x6b, (byte) 0xc3, (byte) 0xa9, + (byte) 0x72, (byte) 0x6a, (byte) 0xc3, (byte) 0xbc, (byte) 0x6b, + (byte) 0x20, (byte) 0x65, (byte) 0x2d, (byte) 0x6d, (byte) 0x61, + (byte) 0x69, (byte) 0x6c, (byte) 0x2d, (byte) 0x62, (byte) 0x65, + (byte) 0x6e, (byte) 0x20, (byte) 0x76, (byte) 0x65, (byte) 0x67, + (byte) 0x79, (byte) 0x65, (byte) 0x20, (byte) 0x66, (byte) 0x65, + (byte) 0x6c, (byte) 0x20, (byte) 0x76, (byte) 0x65, (byte) 0x6c, + (byte) 0xc3, (byte) 0xbc, (byte) 0x6e, (byte) 0x6b, (byte) 0x20, + (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, + (byte) 0x20, (byte) 0x61, (byte) 0x20, (byte) 0x6b, (byte) 0x61, + (byte) 0x70, (byte) 0x63, (byte) 0x73, (byte) 0x6f, (byte) 0x6c, + (byte) 0x61, (byte) 0x74, (byte) 0x6f, (byte) 0x74, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x05, (byte) 0x27, (byte) 0x01, (byte) 0x00, (byte) 0xc2, + (byte) 0x11, (byte) 0x00, (byte) 0x00, (byte) 0x3e, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0xc1, (byte) 0x00, (byte) 0x58, + (byte) 0x3d, (byte) 0xc1, (byte) 0x00, (byte) 0x58, (byte) 0x3d, + (byte) 0x17, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0x2f, + (byte) 0xf0, (byte) 0xe4, (byte) 0xc3, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x42, (byte) 0x01, (byte) 0x00, (byte) 0x80, + (byte) 0x49, (byte) 0x02, (byte) 0x00, (byte) 0x00, (byte) 0x78, + (byte) 0x9c, (byte) 0xad, (byte) 0x91, (byte) 0xb1, (byte) 0x6e, + (byte) 0xc2, (byte) 0x40, (byte) 0x0c, (byte) 0x86, (byte) 0x5f, + (byte) 0xc5, (byte) 0xdd, (byte) 0x09, (byte) 0x4d, (byte) 0x04, + (byte) 0x41, (byte) 0x88, (byte) 0xa9, (byte) 0xd0, (byte) 0x16, + (byte) 0xb5, (byte) 0x48, (byte) 0x08, (byte) 0xa9, (byte) 0xd0, + (byte) 0x85, (byte) 0xcd, (byte) 0x81, (byte) 0x6b, (byte) 0x08, + (byte) 0xb9, (byte) 0xbb, (byte) 0xa0, (byte) 0x9c, (byte) 0x83, + (byte) 0x94, (byte) 0x6c, (byte) 0x7d, (byte) 0x80, (byte) 0x4e, + (byte) 0x7d, (byte) 0x82, (byte) 0x8c, (byte) 0x19, (byte) 0x58, + (byte) 0x3b, (byte) 0xb1, (byte) 0xf9, (byte) 0xc5, (byte) 0x7a, + (byte) 0x51, (byte) 0xa1, (byte) 0x64, (byte) 0x40, (byte) 0xea, + (byte) 0x52, (byte) 0x4f, (byte) 0xff, (byte) 0xd9, (byte) 0xf7, + (byte) 0x7f, (byte) 0xbe, (byte) 0xb3, (byte) 0x27, (byte) 0xfc, + (byte) 0xb9, (byte) 0x9c, (byte) 0x0f, (byte) 0xe1, (byte) 0x12, + (byte) 0x93, (byte) 0xd7, (byte) 0x87, (byte) 0xe7, (byte) 0x45, + (byte) 0xe3, (byte) 0xfc, (byte) 0xe4, (byte) 0x78, (byte) 0x6e, + (byte) 0xdf, (byte) 0x87, (byte) 0x16, (byte) 0x8c, (byte) 0xb2, + (byte) 0x35, (byte) 0xee, (byte) 0x84, (byte) 0xa1, (byte) 0x16, + (byte) 0xcc, (byte) 0x89, (byte) 0xcb, (byte) 0x8d, (byte) 0xcc, + (byte) 0x21, (byte) 0x6b, (byte) 0x83, (byte) 0xe7, (byte) 0xb7, + (byte) 0x2f, (byte) 0x57, (byte) 0x17, (byte) 0x42, (byte) 0x0e, + (byte) 0xdc, (byte) 0x9e, (byte) 0xe3, (byte) 0xdd, (byte) 0x76, + (byte) 0x3a, (byte) 0x7d, (byte) 0xa7, (byte) 0xdb, (byte) 0xeb, + (byte) 0xf6, (byte) 0xe1, (byte) 0x5f, (byte) 0x62, (byte) 0x08, + (byte) 0xf3, (byte) 0xe5, (byte) 0x6c, (byte) 0x34, (byte) 0x7b, + (byte) 0x81, (byte) 0x7b, (byte) 0xfe, (byte) 0x98, (byte) 0x3e, + (byte) 0x0e, (byte) 0x9a, (byte) 0xa5, (byte) 0x29, (byte) 0xae, + (byte) 0x13, (byte) 0xad, (byte) 0xf1, (byte) 0xba, (byte) 0xaf, + (byte) 0xe3, (byte) 0xc2, (byte) 0x4a, (byte) 0x81, (byte) 0xc2, + (byte) 0x10, (byte) 0x4d, (byte) 0x33, (byte) 0xed, (byte) 0xf9, + (byte) 0x75, (byte) 0xda, (byte) 0x14, (byte) 0x5c, (byte) 0x49, + (byte) 0x61, (byte) 0xae, (byte) 0x3b, (byte) 0x4f, (byte) 0x7d, + (byte) 0x0b, (byte) 0xe0, (byte) 0x32, (byte) 0x05, (byte) 0xac, + (byte) 0x11, (byte) 0x39, (byte) 0xa6, (byte) 0x49, (byte) 0x6a, + (byte) 0x3d, (byte) 0x65, (byte) 0x18, (byte) 0x41, (byte) 0x1c, + (byte) 0xd5, (byte) 0x42, (byte) 0x4a, (byte) 0x3e, (byte) 0xd8, + (byte) 0x39, (byte) 0x18, (byte) 0x88, (byte) 0xf9, (byte) 0x4b, + (byte) 0x92, (byte) 0xe1, (byte) 0x2a, (byte) 0xe4, (byte) 0x8a, + (byte) 0x7e, (byte) 0x8c, (byte) 0x5c, (byte) 0x19, (byte) 0x40, + (byte) 0x6b, (byte) 0x7e, (byte) 0x1f, (byte) 0x0f, (byte) 0x1d, + (byte) 0x82, (byte) 0xc8, (byte) 0x00, (byte) 0x61, (byte) 0x4a, + (byte) 0x28, (byte) 0x15, (byte) 0x16, (byte) 0x05, (byte) 0xde, + (byte) 0x9c, (byte) 0xe1, (byte) 0x4a, (byte) 0x68, (byte) 0x9d, + (byte) 0x47, (byte) 0x81, (byte) 0xd0, (byte) 0xa0, (byte) 0x6a, + (byte) 0xca, (byte) 0x89, (byte) 0x1e, (byte) 0xa0, (byte) 0xe5, + (byte) 0x1d, (byte) 0xf6, (byte) 0x5c, (byte) 0xea, (byte) 0xed, + (byte) 0xf9, (byte) 0x57, (byte) 0x08, (byte) 0x24, (byte) 0x52, + (byte) 0xc5, (byte) 0x55, (byte) 0x2c, (byte) 0xc8, (byte) 0xd6, + (byte) 0x8e, (byte) 0x72, (byte) 0x2d, (byte) 0x74, (byte) 0xd4, + (byte) 0xb2, (byte) 0xaa, (byte) 0x4a, (byte) 0xb7, (byte) 0x7c, + (byte) 0x8c, (byte) 0x41, (byte) 0x38, (byte) 0x0a, (byte) 0x23, + (byte) 0xe9, (byte) 0xd4, (byte) 0xa0, (byte) 0xbd, (byte) 0x08, + (byte) 0x73, (byte) 0x01, (byte) 0x6f, (byte) 0x42, (byte) 0x5a, + (byte) 0x25, (byte) 0xf9, (byte) 0xa8, (byte) 0xe3, (byte) 0x5f, + (byte) 0x42, (byte) 0x8c, (byte) 0xbb, (byte) 0x95, (byte) 0x49, + (byte) 0x24, (byte) 0x52, (byte) 0x42, (byte) 0xa7, (byte) 0x07, + (byte) 0x6c, (byte) 0x32, (byte) 0x1d, (byte) 0xd8, (byte) 0x65, + (byte) 0xde, (byte) 0x29, (byte) 0x24, (byte) 0xdc, (byte) 0x6b, + (byte) 0x41, (byte) 0xed, (byte) 0x4d, (byte) 0xf6, (byte) 0xc7, + (byte) 0x12, (byte) 0x4c, (byte) 0x91, (byte) 0x04, (byte) 0x49, + (byte) 0x5a, (byte) 0x8f, (byte) 0x04, (byte) 0x9b, (byte) 0x3b, + (byte) 0xf0, (byte) 0x7b, (byte) 0xae, (byte) 0xeb, (byte) 0xc2, + (byte) 0x98, (byte) 0xbe, (byte) 0x01, (byte) 0xd3, (byte) 0xfa, + (byte) 0xa2, (byte) 0x8f }; + + public static DocsumDefinitionSet createDocsumDefinitionSet(String configID) { + DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); + return new DocsumDefinitionSet(config.documentdb(0)); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java new file mode 100644 index 00000000000..faaf3f5c2b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -0,0 +1,633 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.search.Fs4Config; +import com.yahoo.fs4.mplex.*; +import com.yahoo.fs4.test.QueryTestCase; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.container.protect.Error; +import com.yahoo.document.GlobalId; +import com.yahoo.fs4.*; +import com.yahoo.processing.execution.Execution.Trace; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.prelude.fastsearch.*; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +/** + * Tests the Fast searcher + * + * @author bratseth + */ +@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" }) +public class FastSearcherTestCase { + + private final static DocumentdbInfoConfig documentdbInfoConfig = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder()); + private MockBackend mockBackend; + + @Test + public void testNoNormalizing() { + Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); + FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new MockDispatcher(), + new SummaryParameters(null), + new ClusterParams("testhittype"), + new CacheParams(100, 1e64), + documentdbInfoConfig); + + MockFSChannel.setEmptyDocsums(false); + + + assertEquals(100, fastSearcher.getCacheControl().capacity()); // Default cache = 100Mb + + Result result = doSearch(fastSearcher, new Query("?query=ignored"), 0, 10); + + assertTrue(result.hits().get(0).getRelevance().getScore() > 1000); + } + + @Test + public void testNullQuery() { + Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); + FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new MockDispatcher(), + new SummaryParameters(null), + new ClusterParams("testhittype"), + new CacheParams(100, 1e64), + documentdbInfoConfig); + + String query = "?junkparam=ignored"; + Result result = doSearch(fastSearcher,new Query(query), 0, 10); + com.yahoo.search.result.ErrorMessage message = result.hits().getError(); + + assertNotNull("Got error", message); + assertEquals("Null query", message.getMessage()); + assertEquals(query, message.getDetailedMessage()); + assertEquals(Error.NULL_QUERY.code, message.getCode()); + } + + @Test + public void testQueryWithRestrict() { + mockBackend = new MockBackend(); + DocumentdbInfoConfig documentdbConfigWithOneDb = + new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("testDb"))); + FastSearcher fastSearcher = new FastSearcher(mockBackend, new MockDispatcher(), new SummaryParameters(null), + new ClusterParams("testhittype"), + new CacheParams(100, 1e64), documentdbConfigWithOneDb); + + Query query = new Query("?query=foo&model.restrict=testDb"); + query.prepare(); + Result result = doSearch(fastSearcher, query, 0, 10); + + Packet receivedPacket = mockBackend.getChannel().getLastQueryPacket(); + byte[] encoded = QueryTestCase.packetToBytes(receivedPacket); + System.out.println(Arrays.toString(encoded)); + byte[] correct = new byte[] { + 0, 0, 0, 100, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, + QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, // time left + 0, 0, -64, 4, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 5, 109, 97, 116, 99, 104, 0, 0, 0, 1, 0, 0, 0, 24, 100, 111, 99, 117, 109, 101, 110, 116, 100, 98, 46, 115, 101, 97, 114, 99, 104, 100, 111, 99, 116, 121, 112, 101, 0, 0, 0, 6, 116, 101, 115, 116, 68, 98, 0, 0, 0, 1, 0, 0, 0, 7, 68, 1, 0, 3, 102, 111, 111 + }; + QueryTestCase.assertEqualArrays(correct, encoded); + } + + @Test + public void testSearch() { + FastSearcher fastSearcher = createFastSearcher(); + + assertEquals(100, fastSearcher.getCacheControl().capacity()); // Default cache =100MB + + Result result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10); + + Execution execution = new Execution(chainedAsSearchChain(fastSearcher), Execution.Context.createContextStub()); + assertEquals(2, result.getHitCount()); + execution.fill(result); + assertCorrectHit1((FastHit) result.hits().get(0)); + assertCorrectTypes1((FastHit) result.hits().get(0)); + for (int idx = 0; idx < result.getHitCount(); idx++) { + assertTrue(!result.hits().get(idx).isCached()); + } + + // Repeat the request a couple of times, to verify whether the packet cache works + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10); + assertEquals(2, result.getHitCount()); + execution.fill(result); + assertCorrectHit1((FastHit) result.hits().get(0)); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(result.hits().get(i) + " should be cached", + result.hits().get(i).isCached()); + } + + // outside-range cache hit + result = doSearch(fastSearcher,new Query("?query=ignored"), 6, 3); + // fill should still work (nop) + execution.fill(result); + + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10); + assertEquals(2, result.getHitCount()); + assertCorrectHit1((FastHit) result.hits().get(0)); + assertTrue("All hits are cached and the result knows it", + result.isCached()); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(result.hits().get(i) + " should be cached", + result.hits().get(i).isCached()); + } + + clearCache(fastSearcher); + + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10); + assertEquals(2, result.getHitCount()); + execution.fill(result); + assertCorrectHit1((FastHit) result.hits().get(0)); + assertTrue("All hits are not cached", !result.isCached()); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(!result.hits().get(i).isCached()); + } + + // Test that partial result sets can be retrieved from the cache + clearCache(fastSearcher); + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1); + assertEquals(1, result.getConcreteHitCount()); + execution.fill(result); + + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2); + assertEquals(2, result.getConcreteHitCount()); + execution.fill(result); + // First hit should be cached but not second hit + assertTrue(result.hits().get(0).isCached()); + assertFalse(result.hits().get(1).isCached()); + + // Check that the entire result set is returned from cache now + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2); + assertEquals(2, result.getConcreteHitCount()); + execution.fill(result); + // both first and second should now be cached + assertTrue(result.hits().get(0).isCached()); + assertTrue(result.hits().get(1).isCached()); + + // Tests that the cache _hit_ is not returned if _another_ + // hit is requested + clearCache(fastSearcher); + + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1); + assertEquals(1, result.getConcreteHitCount()); + + result = doSearch(fastSearcher,new Query("?query=ignored"), 1, 1); + assertEquals(1, result.getConcreteHitCount()); + + for (int i = 0; i < result.getHitCount(); i++) { + assertFalse("Hit " + i + " should not be cached.", + result.hits().get(i).isCached()); + } + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(Collections.emptyList()), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private void doFill(Searcher searcher, Result result) { + createExecution(searcher).fill(result); + } + + @Test + public void requireThatPropertiesAreReencoded() throws Exception { + FastSearcher fastSearcher = createFastSearcher(); + + assertEquals(100, fastSearcher.getCacheControl().capacity()); // Default cache =100MB + + Query query = new Query("?query=ignored"); + query.getRanking().setQueryCache(true); + Result result = doSearch(fastSearcher, query, 0, 10); + + Execution execution = new Execution(chainedAsSearchChain(fastSearcher), Execution.Context.createContextStub()); + assertEquals(2, result.getHitCount()); + execution.fill(result); + + Packet receivedPacket = mockBackend.getChannel().getLastReceived(); + ByteBuffer buf = ByteBuffer.allocate(1000); + receivedPacket.encode(buf); + buf.flip(); + byte[] actual = new byte[buf.remaining()]; + buf.get(actual); + + byte IGNORE = 69; + byte[] expected = new byte[] { 0, 0, 0, -85, 0, 0, 0, -37, 0, 0, 48, 17, 0, 0, 0, 0, + // query timeout + IGNORE, IGNORE, IGNORE, IGNORE, + // "default" - rank profile + 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, -128, 0, + // 3 property entries (rank, match, caches) + 0, 0, 0, 3, + // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 + 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd', 0, 0, 0, 26, 'q', 'r', 's', 'e', 'r', 'v', 'e', 'r', '.', + IGNORE, '.', IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, IGNORE, '.', IGNORE, + // match: documentdb.searchdoctype => test + 0, 0, 0, 5, 'm', 'a', 't', 'c', 'h', 0, 0, 0, 1, 0, 0, 0, 24, 'd', 'o', 'c', 'u', 'm', 'e', 'n', 't', 'd', 'b', '.', 's', 'e', 'a', 'r', 'c', 'h', 'd', 'o', 'c', 't', 'y', 'p', 'e', 0, 0, 0, 4, 't', 'e', 's', 't', + // sessionId => qrserver.0.XXXXXXXXXXXXX.0 + 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e', + // flags + 0, 0, 0, 2 + }; + //System.out.println(Arrays.toString(actual)); + assertEquals(expected.length, actual.length); + for (int i = 0; i < expected.length; ++i) { + if (expected[i] == IGNORE) { + actual[i] = IGNORE; + } + } + assertArrayEquals(expected, actual); + } + + private FastSearcher createFastSearcher() { + mockBackend = new MockBackend(); + ConfigGetter getter = new ConfigGetter<>(DocumentdbInfoConfig.class); + DocumentdbInfoConfig config = getter.getConfig("file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"); + + MockFSChannel.resetDocstamp(); + Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); + return new FastSearcher(mockBackend, new MockDispatcher(), new SummaryParameters(null), + new ClusterParams("testhittype"), new CacheParams(100, 1e64), config); + } + + @Ignore + public void testSinglePhaseCachedSupersets() { + Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); + MockFSChannel.resetDocstamp(); + FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new MockDispatcher(), + new SummaryParameters(null), + new ClusterParams("testhittype"), + new CacheParams(100, 1e64), + documentdbInfoConfig); + + CacheControl c = fastSearcher.getCacheControl(); + + Result result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2); + Query q = new Query("?query=ignored"); + ((WordItem) q.getModel().getQueryTree().getRoot()).setUniqueID(1); + QueryPacket queryPacket = QueryPacket.create(q); + CacheKey k = new CacheKey(queryPacket); + PacketWrapper p = c.lookup(k, q); + assertEquals(1, p.getResultPackets().size()); + + result = doSearch(fastSearcher,new Query("?query=ignored"), 1, 1); + p = c.lookup(k, q); + // ensure we don't get redundant QueryResultPacket instances + // in the cache + assertEquals(1, p.getResultPackets().size()); + + assertEquals(1, result.getConcreteHitCount()); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(result.hits().get(i).isCached()); + } + + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1); + p = c.lookup(k, q); + assertEquals(1, p.getResultPackets().size()); + assertEquals(1, result.getConcreteHitCount()); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(result.hits().get(i).isCached()); + } + + } + + @Test + public void testMultiPhaseCachedSupersets() { + Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); + MockFSChannel.resetDocstamp(); + FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new MockDispatcher(), + new SummaryParameters(null), + new ClusterParams("testhittype"), + new CacheParams(100, 1e64), + documentdbInfoConfig); + + Result result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2); + result = doSearch(fastSearcher,new Query("?query=ignored"), 1, 1); + assertEquals(1, result.getConcreteHitCount()); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(result.hits().get(i).isCached()); + if (!result.hits().get(i).isMeta()) { + assertTrue(result.hits().get(i).getFilled().isEmpty()); + } + } + + result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1); + assertEquals(1, result.getConcreteHitCount()); + for (int i = 0; i < result.getHitCount(); i++) { + assertTrue(result.hits().get(i).isCached()); + if (!result.hits().get(i).isMeta()) { + assertTrue(result.hits().get(i).getFilled().isEmpty()); + } + } + + } + + // TODO: Enable this - it fails when on vpn + @Ignore + public void testPing() throws IOException, InterruptedException { + Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); + BackendTestCase.MockServer server = new BackendTestCase.MockServer(); + FS4ResourcePool listeners = new FS4ResourcePool(new Fs4Config()); + Backend backend = listeners.getBackend(server.host.getHostString(),server.host.getPort()); + FastSearcher fastSearcher = new FastSearcher(backend, new MockDispatcher(), + new SummaryParameters(null), + new ClusterParams("testhittype"), + new CacheParams(0, 0.0d), + documentdbInfoConfig); + server.dispatch.packetData = BackendTestCase.PONG; + Chain chain = new Chain(fastSearcher); + Execution e = new Execution(chain, Execution.Context.createContextStub()); + Pong pong = e.ping(new Ping()); + assertEquals(127, pong.getPongPacket(0).getDocstamp()); + backend.shutdown(); + listeners.deconstruct(); + server.dispatch.socket.close(); + server.dispatch.connection.close(); + server.worker.join(); + assertEquals(1, pong.getPongPacketsSize()); + Pong other = new Pong(); + other.setPingInfo(null); + other.addError(ErrorMessage.createServerIsMisconfigured("as usual")); + pong.merge(other); + assertEquals(1, pong.getErrors().size()); + assertEquals(1, pong.getPongPackets().size()); + assertEquals("", other.getPingInfo()); + pong.setPingInfo("blbl"); + assertEquals("Result of pinging using blbl error : Service is misconfigured (as usual)", + pong.toString()); + assertEquals("Result of pinging error : Service is misconfigured (as usual)", + other.toString()); + } + + private void clearCache(FastSearcher fastSearcher) { + fastSearcher.getCacheControl().clear(); + } + + private void assertCorrectTypes1(FastHit hit) { + assertEquals(String.class, hit.getField("TITLE").getClass()); + assertEquals(Integer.class, hit.getField("BYTES").getClass()); + } + + private void assertCorrectHit1(FastHit hit) { + assertEquals( + "StudyOfMadonna.com - Interviews, Articles, Reviews, Quotes, Essays and more..", + hit.getField("TITLE")); + assertEquals("352", hit.getField("WORDS").toString()); + assertEquals(2003., hit.getRelevance().getScore(), 0.01d); + assertEquals("index:0/234/0/" + FastHit.asHexString(hit.getGlobalId()), hit.getId().toString()); + assertEquals("9190", hit.getField("BYTES").toString()); + assertEquals("testhittype", hit.getSource()); + } + + private static class MockBackend extends Backend { + + private MockFSChannel channel; + + public MockBackend() { + channel = new MockFSChannel(this, 1); + } + + public FS4Channel openChannel() { + return channel; + } + + public MockFSChannel getChannel() { return channel; } + + public void shutdown() {} + } + + + /** + * A channel which returns hardcoded packets of the same type as fdispatch + */ + private static class MockFSChannel extends FS4Channel { + + public MockFSChannel(Backend backend, Integer channelId) {} + + private Packet lastReceived = null; + + private QueryPacket lastQueryPacket = null; + + /** Initial value of docstamp */ + private static int docstamp = 1088490666; + + private static boolean emptyDocsums = false; + + public synchronized boolean sendPacket(BasicPacket bPacket) { + Packet packet = (Packet) bPacket; + + try { + packet.encode(ByteBuffer.allocate(65536), 0); + } catch (BufferTooSmallException e) { + throw new RuntimeException("Too small buffer to encode packet in mock backend."); + } + if (packet instanceof QueryPacket) { + lastQueryPacket = (QueryPacket) packet; + } else if (!(packet instanceof GetDocSumsPacket)) { + throw new RuntimeException( + "Mock channel don't know what to reply to " + packet); + } + lastReceived = packet; + return true; + } + + /** Change docstamp to invalidate cache */ + public static void resetDocstamp() { + docstamp = 1088490666; + } + + /** Flip sending (in)valid docsums */ + public static void setEmptyDocsums(boolean d) { + emptyDocsums = d; + } + + /** Returns the last query packet received or null if none */ + public QueryPacket getLastQueryPacket() { + return lastQueryPacket; + } + + public Packet getLastReceived() { + return lastReceived; + } + + public BasicPacket[] receivePackets(long timeout, int packetCount) { + List packets = new java.util.ArrayList(); + + if (lastReceived instanceof QueryPacket) { + lastQueryPacket = (QueryPacket) lastReceived; + QueryResultPacket result = QueryResultPacket.create(); + + result.setDocstamp(docstamp); + result.setChannel(0); + result.setTotalDocumentCount(2); + result.setOffset(lastQueryPacket.getOffset()); + + if (lastQueryPacket.getOffset() == 0 + && lastQueryPacket.getLastOffset() >= 1) { + result.addDocument( + new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(123), + 2003, 234, 1000)); + } + if (lastQueryPacket.getOffset() <= 1 + && lastQueryPacket.getLastOffset() >= 2) { + result.addDocument( + new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(456), + 1855, 234, 1001)); + } + packets.add(result); + } else if (lastReceived instanceof GetDocSumsPacket) { + addDocsums(packets, lastQueryPacket); + } + while (packetCount >= 0 && packets.size() > packetCount) { + packets.remove(packets.size() - 1); + } + + return (Packet[]) packets.toArray(new Packet[packets.size()]); + } + + /** Adds the number of docsums requested in queryPacket.getHits() */ + private void addDocsums(List packets, QueryPacket queryPacket) { + int numHits = queryPacket.getHits(); + + if (lastReceived instanceof GetDocSumsPacket) { + numHits = ((GetDocSumsPacket) lastReceived).getNumDocsums(); + } + for (int i = 0; i < numHits; i++) { + ByteBuffer buffer; + + if (emptyDocsums) { + buffer = createEmptyDocsumPacketData(); + } else { + int[] docids = { + 123, 456, 789, 789, 789, 789, 789, 789, 789, + 789, 789, 789 }; + + buffer = createDocsumPacketData(docids[i], + DocsumDefinitionTestCase.docsum4); + } + buffer.position(0); + packets.add(PacketDecoder.decode(buffer)); + } + packets.add(EolPacket.create()); + } + + private ByteBuffer createEmptyDocsumPacketData() { + ByteBuffer buffer = ByteBuffer.allocate(16); + + buffer.limit(buffer.capacity()); + buffer.position(0); + buffer.putInt(12); // length + buffer.putInt(205); // a code for docsumpacket + buffer.putInt(0); // channel + buffer.putInt(0); // dummy location + return buffer; + } + + private ByteBuffer createDocsumPacketData(int docid, byte[] docsumData) { + ByteBuffer buffer = ByteBuffer.allocate(docsumData.length + 4 + 8 + GlobalId.LENGTH); + + buffer.limit(buffer.capacity()); + buffer.position(0); + buffer.putInt(docsumData.length + 8 + GlobalId.LENGTH); + buffer.putInt(205); // Docsum packet code + buffer.putInt(0); + byte[] rawGid = DocsumDefinitionTestCase.createGlobalId(docid).getRawId(); + buffer.put(rawGid); + buffer.put(docsumData); + return buffer; + } + + public void close() {} + } + + @Test + public void null_summary_is_included_in_trace() { + String summary = null; + assertThat(getTraceString(summary), containsString("summary=[null]")); + } + + @Test + public void non_null_summary_is_included_in_trace() { + String summary = "all"; + System.out.println(getTraceString(summary)); + assertThat(getTraceString(summary), containsString("summary='all'")); + } + + private String getTraceString(String summary) { + FastSearcher fastSearcher = createFastSearcher(); + + Query query = new Query("?query=ignored"); + query.getPresentation().setSummary(summary); + query.setTraceLevel(2); + + Result result = doSearch(fastSearcher, query, 0, 10); + doFill(fastSearcher, result); + + Trace trace = query.getContext(false).getTrace(); + final AtomicReference fillTraceString = new AtomicReference<>(); + + + trace.traceNode().accept(new TraceVisitor() { + @Override + public void visit(TraceNode traceNode) { + if (traceNode.payload() instanceof String && traceNode.payload().toString().contains("fill to dispatch")) + fillTraceString.set((String) traceNode.payload()); + + } + }); + + return fillTraceString.get(); + } + + /** Just a stub for now */ + private static class MockDispatcher extends Dispatcher { + + public MockDispatcher() { + super(new DispatchConfig(new DispatchConfig.Builder())); + } + + public void fill(Result result, String summaryClass) { + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockFDispatch.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockFDispatch.java new file mode 100644 index 00000000000..cc977a48cdd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockFDispatch.java @@ -0,0 +1,211 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.prelude.ConfigurationException; + + +/** + * A server which replies to any query with the same query result after + * a configurable delay, with a configurable slowness (delay between each byte). + * Connections are never timed out. + * + * @author bratseth + */ +public class MockFDispatch { + + private static int connectionCount = 0; + + private static Logger log = Logger.getLogger(MockFDispatch.class.getName()); + + /** The port we accept incoming requests at */ + private int listenPort = 0; + + private long replyDelay; + + private long byteDelay; + + private Object barrier; + + private static byte[] queryResultPacketData = new byte[] { + 0, 0, 0, 64, 0, 0, + 0, 214 - 256, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, + 25, 0, 0, 0, 111, 0, 0, 0, 97, 0, 0, 0, 3, 0, 0, 0, 23, 0, 0, 0, 7, 0, 0, + 0, 36, 0, 0, 0, 4, 0, 0, 0, 21, 0, 0, 0, 8, 0, 0, 0, 37}; + + private static byte[] docsumData = DocsumDefinitionTestCase.docsum4; + + private static byte[] docsumHeadPacketData = new byte[] { + 0, 0, 3, 39, 0, 0, + 0, 205 - 256, 0, 0, 0, 1, 0, 0, 0, 0}; + + private static byte[] eolPacketData = new byte[] { + 0, 0, 0, 8, 0, 0, 0, + 200 - 256, 0, 0, 0, 1 }; + + private Set connectionThreads = new HashSet<>(); + + public MockFDispatch(int listenPort, long replyDelay, long byteDelay) { + this.replyDelay = replyDelay; + this.byteDelay = byteDelay; + this.listenPort = listenPort; + } + + public void setBarrier(Object barrier) { + this.barrier = barrier; + } + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + public void run() { + try { + ServerSocketChannel channel = createServerSocket(listenPort); + + channel.socket().setReuseAddress(true); + while (!Thread.currentThread().isInterrupted()) { + try { + // notify those waiting at the barrier that they + // can now proceed and talk to us + synchronized (barrier) { + if (barrier != null) { + barrier.notify(); + } + } + SocketChannel socketChannel = channel.accept(); + + connectionThreads.add(new ConnectionThread(socketChannel)); + } catch (ClosedByInterruptException e) {// We'll exit + } catch (ClosedChannelException e) { + return; + } catch (Exception e) { + log.log(Level.WARNING, "Unexpected error reading request", e); + } + } + channel.close(); + } catch (IOException e) { + throw new ConfigurationException("Socket channel failure", e); + } + } + + private ServerSocketChannel createServerSocket(int listenPort) + throws IOException { + ServerSocketChannel channel = ServerSocketChannel.open(); + ServerSocket socket = channel.socket(); + + socket.bind( + new InetSocketAddress(InetAddress.getLocalHost(), listenPort)); + String host = socket.getInetAddress().getHostName(); + + log.fine("Accepting dfispatch requests at " + host + ":" + listenPort); + return channel; + } + + public static void main(String[] args) { + log.setLevel(Level.FINE); + MockFDispatch m = new MockFDispatch(7890, Integer.parseInt(args[0]), + Integer.parseInt(args[1])); + + m.run(); + } + + private class ConnectionThread extends Thread { + + private ByteBuffer writeBuffer = ByteBuffer.allocate(2000); + + private ByteBuffer readBuffer = ByteBuffer.allocate(2000); + + private int connectionNr = 0; + + private SocketChannel channel; + + public ConnectionThread(SocketChannel channel) { + this.channel = channel; + fillBuffer(writeBuffer); + start(); + } + + private void fillBuffer(ByteBuffer buffer) { + buffer.clear(); + buffer.put(queryResultPacketData); + buffer.put(docsumHeadPacketData); + buffer.put(docsumData); + buffer.put(docsumHeadPacketData); + buffer.put(docsumData); + buffer.put(eolPacketData); + } + + public void run() { + connectionNr = connectionCount++; + log.fine("Opened connection " + connectionNr); + + try { + long lastRequest = System.currentTimeMillis(); + + while ((System.currentTimeMillis() - lastRequest) <= 5000 + && (!isInterrupted())) { + readBuffer.clear(); + channel.read(readBuffer); + lastRequest = System.currentTimeMillis(); + delay(replyDelay); + + if (byteDelay > 0) { + writeSlow(writeBuffer); + } else { + write(writeBuffer); + } + log.fine( + "Replied in " + + (System.currentTimeMillis() - lastRequest) + + " ms"); + } + + log.fine("Closing timed out connection " + connectionNr); + connectionCount--; + channel.close(); + } catch (IOException e) {} + } + + private void write(ByteBuffer writeBuffer) throws IOException { + writeBuffer.flip(); + channel.write(writeBuffer); + } + + private void writeSlow(ByteBuffer writeBuffer) throws IOException { + writeBuffer.flip(); + int dataSize = writeBuffer.limit(); + + for (int i = 0; i < dataSize; i++) { + writeBuffer.position(i); + writeBuffer.limit(i + 1); + channel.write(writeBuffer); + delay(byteDelay); + } + writeBuffer.limit(dataSize); + } + + private void delay(long delay) { + + try { + Thread.sleep(delay); + } catch (InterruptedException e) {} + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java new file mode 100644 index 00000000000..c3ab826bad8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketCacheTestCase.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + + +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.BufferTooSmallException; +import com.yahoo.fs4.PacketDecoder; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.search.Query; +import com.yahoo.prelude.fastsearch.CacheKey; +import com.yahoo.prelude.fastsearch.PacketCache; +import com.yahoo.prelude.fastsearch.PacketWrapper; + +import java.nio.ByteBuffer; + + +/** + * Tests the packet cache. Also tested in FastSearcherTestCase. + * + * @author Jon S Bratseth + */ +public class PacketCacheTestCase extends junit.framework.TestCase { + + static byte[] queryResultPacketData = new byte[] { + 0, 0, 0, 104, + 0, 0, 0,214 - 256, + 0, 0, 0, 1, + 0, 0, 0, 0, + 0, 0, 0, 2, + 0, 0, 0, 0, + 0, 0, 0, 5, + 0x40,0x39,0,0,0, 0, 0, 25, + 0, 0, 0, 111, + 0, 0, 0, 97, + 0,0,0,3, 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,0, 0,0,0,7, 0,0,0,36, + 0,0,0,4, 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,0, 0,0,0,8, 0,0,0,37}; + static int length = queryResultPacketData.length; // 4 + 68 + 2*12 bytes + + static CacheKey key1 = new CacheKey(QueryPacket.create(new Query("/?query=key1"))); + static CacheKey key2 = new CacheKey(QueryPacket.create(new Query("/?query=key2"))); + static CacheKey key3 = new CacheKey(QueryPacket.create(new Query("/?query=key3"))); + static CacheKey key4 = new CacheKey(QueryPacket.create(new Query("/?query=key4"))); + + public PacketCacheTestCase(String name) { + super(name); + } + + public void testPutAndGet() throws BufferTooSmallException { + PacketCache cache = new PacketCache(0, (length + 30) * 3 - 1, 1e64); + + cache.setMaxCacheItemPercentage(50); + + final int keysz = 30; + + // first control assumptions + assertEquals(keysz, key1.byteSize()); + assertEquals(keysz, key2.byteSize()); + assertEquals(keysz, key3.byteSize()); + + cache.put(key1, createCacheEntry(key1)); + assertNotNull(cache.get(key1)); + assertEquals(keysz + length, cache.totalPacketSize()); + + cache.put(key2, createCacheEntry(key2)); + assertNotNull(cache.get(key1)); + assertNotNull(cache.get(key2)); + assertEquals(keysz*2 + length*2, cache.totalPacketSize()); + + cache.put(key1, createCacheEntry(key1)); + assertNotNull(cache.get(key1)); + assertNotNull(cache.get(key2)); + assertEquals(keysz*2 + length*2, cache.totalPacketSize()); + + // This should cause key1 (the eldest accessed) to be removed, as 3 is 1 2 many + cache.put(key3, createCacheEntry(key3)); + assertEquals(keysz*2 + length*2, cache.totalPacketSize()); + assertNull(cache.get(key1)); + assertNotNull(cache.get(key2)); + assertNotNull(cache.get(key3)); + assertEquals(keysz*2 + length*2, cache.totalPacketSize()); + } + + // more control that delete code does not change internal access order + public void testInternalOrdering() throws BufferTooSmallException { + // room for three entries + PacketCache cache = new PacketCache(0, length * 4 - 1, 1e64); + cache.setMaxCacheItemPercentage(50); + + cache.put(key1, createCacheEntry()); + cache.put(key2, createCacheEntry()); + cache.put(key3, createCacheEntry()); + cache.put(key4, createCacheEntry()); + + assertNull(cache.get(key1)); + assertEquals(3, cache.size()); + cache.get(key2); + cache.put(key1, createCacheEntry()); + assertNull(cache.get(key3)); + assertNotNull(cache.get(key1)); + assertNotNull(cache.get(key2)); + assertNotNull(cache.get(key4)); + assertNotNull(cache.get(key1)); + cache.put(key3, createCacheEntry()); + assertNotNull(cache.get(key1)); + assertNotNull(cache.get(key4)); + assertNotNull(cache.get(key3)); + } + + public void testTooLargeItem() throws BufferTooSmallException { + PacketCache cache = new PacketCache(0, 100, 1e64); // 100 bytes cache + + cache.setMaxCacheItemPercentage(50); + + cache.put(key1, createCacheEntry()); + assertNull(cache.get(key1)); // 68 is more than 50% of the size + assertEquals(0, cache.totalPacketSize()); + } + + public void testClearing() throws BufferTooSmallException { + PacketCache cache = new PacketCache(0, 140, 1e64); // 140 bytes cache + + cache.setMaxCacheItemPercentage(50); + + cache.put(key1, createCacheEntry()); + cache.put(key2, createCacheEntry()); + + cache.clear(); + assertNull(cache.get(key1)); + assertNull(cache.get(key2)); + assertEquals(0, cache.totalPacketSize()); + } + + public void testRemoving() throws BufferTooSmallException { + PacketCache cache = new PacketCache(0, length*2, 1e64); // 96*2 bytes cache + + cache.setMaxCacheItemPercentage(50); + + cache.put(key1, createCacheEntry()); + cache.put(key2, createCacheEntry()); + + cache.remove(key1); + assertNull(cache.get(key1)); + assertNotNull(cache.get(key2)); + assertEquals(length, cache.totalPacketSize()); + } + + public void testEntryAging() throws BufferTooSmallException { + // 1k bytes cache, 5h timeout + PacketCache cache = new PacketCache(0, 1024, 5 * 3600); + + cache.setMaxCacheItemPercentage(50); + cache.put(key1, createCacheEntry(), + System.currentTimeMillis() - 10 * 3600 * 1000); + cache.put(key2, createCacheEntry(), System.currentTimeMillis()); + assertNull(cache.get(key1)); + assertNotNull(cache.get(key2)); + } + + private PacketWrapper createCacheEntry() throws BufferTooSmallException { + return createCacheEntry(null); + } + + public void testTooBigCapacity() { + PacketCache cache = new PacketCache(2048, 0, 5 * 3600); + assertEquals(Integer.MAX_VALUE, cache.getByteCapacity()); + } + + /** Creates a 64-byte packet in an array wrapped in a PacketWrapper */ + private PacketWrapper createCacheEntry(CacheKey key) throws BufferTooSmallException { + ByteBuffer data = ByteBuffer.allocate(length); + + data.put(queryResultPacketData); + data.flip(); + BasicPacket[] content = new BasicPacket[] { + PacketDecoder.extractPacket( + data).packet }; + + return new PacketWrapper(key, content); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketWrapperTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketWrapperTestCase.java new file mode 100644 index 00000000000..e9dd9a89a1b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PacketWrapperTestCase.java @@ -0,0 +1,408 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + + +import java.util.Iterator; +import java.util.List; + +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.DocumentInfo; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.fs4.QueryResultPacket; +import com.yahoo.search.Query; +import com.yahoo.prelude.fastsearch.CacheKey; +import com.yahoo.prelude.fastsearch.PacketWrapper; + + +/** + * Tests the logic wrapping cache entries. + * + * @author Steinar Knutsen + */ +public class PacketWrapperTestCase extends junit.framework.TestCase { + public void testPartialOverlap() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + + // all docs at once + List l = w.getDocuments(0, 20); + assertNotNull(l); + assertEquals(20, l.size()); + int n = 0; + for (Iterator i = l.iterator(); i.hasNext(); ++n) { + DocumentInfo d = (DocumentInfo) i.next(); + assertEquals(DocsumDefinitionTestCase.createGlobalId(n), d.getGlobalId()); + } + + // too far out into the result set + l = w.getDocuments(15, 10); + assertNull(l); + + // only from first subdivision + l = w.getDocuments(3, 2); + assertNotNull(l); + assertEquals(2, l.size()); + n = 3; + for (Iterator i = l.iterator(); i.hasNext(); ++n) { + DocumentInfo d = (DocumentInfo) i.next(); + assertEquals(DocsumDefinitionTestCase.createGlobalId(n), d.getGlobalId()); + } + + // only from second subdivision + l = w.getDocuments(15, 5); + assertNotNull(l); + assertEquals(5, l.size()); + n = 15; + for (Iterator i = l.iterator(); i.hasNext(); ++n) { + DocumentInfo d = (DocumentInfo) i.next(); + assertEquals(DocsumDefinitionTestCase.createGlobalId(n), d.getGlobalId()); + } + + // overshoot by 1 + l = w.getDocuments(15, 6); + assertNull(l); + + // mixed subset + l = w.getDocuments(3, 12); + assertNotNull(l); + assertEquals(12, l.size()); + n = 3; + for (Iterator i = l.iterator(); i.hasNext(); ++n) { + DocumentInfo d = (DocumentInfo) i.next(); + assertEquals(DocsumDefinitionTestCase.createGlobalId(n), d.getGlobalId()); + } + + } + + public void testPacketTrimming1() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(5, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + + assertEquals(2, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(10, ((QueryResultPacket) l.get(1)).getOffset()); + } + + public void testPacketTrimming2() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(5, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(50, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(5, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(50, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming3() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(20, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(25, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(20, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(25, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming4() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(5, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(15, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(20, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(10, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(20, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming5() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(5, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(15, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(15, 85, 100); + w.addResultPacket(q); + q = createQueryResultPacket(20, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(25, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(10, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(15, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming6() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(5, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(60, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(65, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(70, 10, 100); + w.addResultPacket(q); + + assertEquals(4, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(10, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(60, ((QueryResultPacket) l.get(2)).getOffset()); + assertEquals(70, ((QueryResultPacket) l.get(3)).getOffset()); + } + + public void testPacketTrimming7() { + final Query query = new Query("/?query=key"); + query.setWindow(50, 10); + CacheKey key = new CacheKey(QueryPacket.create(query)); + PacketWrapper w = createResult(key, 50, 10, 100); + + QueryResultPacket q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(40, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(30, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(20, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(0, 10, 100); + w.addResultPacket(q); + + assertEquals(6, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(10, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(20, ((QueryResultPacket) l.get(2)).getOffset()); + assertEquals(30, ((QueryResultPacket) l.get(3)).getOffset()); + assertEquals(40, ((QueryResultPacket) l.get(4)).getOffset()); + assertEquals(50, ((QueryResultPacket) l.get(5)).getOffset()); + } + + public void testPacketTrimming8() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(50, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(90, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(50, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(90, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming9() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(10, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(11, 9, 100); + w.addResultPacket(q); + q = createQueryResultPacket(20, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(10, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(20, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming10() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(0, 11, 100); + w.addResultPacket(q); + q = createQueryResultPacket(11, 9, 100); + w.addResultPacket(q); + q = createQueryResultPacket(20, 10, 100); + w.addResultPacket(q); + + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(11, ((QueryResultPacket) l.get(0)).getDocumentCount()); + assertEquals(11, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(20, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming11() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(1, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(9, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(18, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(27, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(36, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(45, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(54, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(63, 10, 100); + w.addResultPacket(q); + + assertEquals(8, w.getResultPackets().size()); + q = createQueryResultPacket(10, 90, 100); + w.addResultPacket(q); + assertEquals(2, w.getResultPackets().size()); + } + + public void testPacketTrimming12() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(4, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(12, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(16, 10, 100); + w.addResultPacket(q); + + assertEquals(4, w.getResultPackets().size()); + q = createQueryResultPacket(8, 10, 100); + w.addResultPacket(q); + assertEquals(3, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(8, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(16, ((QueryResultPacket) l.get(2)).getOffset()); + } + + public void testPacketTrimming13() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(4, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(12, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(16, 10, 100); + w.addResultPacket(q); + + assertEquals(4, w.getResultPackets().size()); + q = createQueryResultPacket(11, 10, 100); + w.addResultPacket(q); + assertEquals(4, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(4, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(12, ((QueryResultPacket) l.get(2)).getOffset()); + assertEquals(16, ((QueryResultPacket) l.get(3)).getOffset()); + } + + public void testPacketTrimming14() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 100); + + QueryResultPacket q = createQueryResultPacket(4, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(12, 10, 100); + w.addResultPacket(q); + q = createQueryResultPacket(16, 10, 100); + w.addResultPacket(q); + + assertEquals(4, w.getResultPackets().size()); + q = createQueryResultPacket(5, 6, 100); + w.addResultPacket(q); + assertEquals(4, w.getResultPackets().size()); + List l = w.getResultPackets(); + assertEquals(0, ((QueryResultPacket) l.get(0)).getOffset()); + assertEquals(4, ((QueryResultPacket) l.get(1)).getOffset()); + assertEquals(12, ((QueryResultPacket) l.get(2)).getOffset()); + assertEquals(16, ((QueryResultPacket) l.get(3)).getOffset()); + } + + public void testZeroHits() { + CacheKey key = new CacheKey(QueryPacket.create(new Query("/?query=key"))); + PacketWrapper w = createResult(key, 0, 10, 0); + + final Query query = new Query("/?query=key"); + query.setWindow(5, 10); + key = new CacheKey(QueryPacket.create(query)); + + QueryResultPacket q = createQueryResultPacket(5, 10, 0); + w.addResultPacket(q); + assertEquals(1, w.getResultPackets().size()); + List l = w.getDocuments(3, 12); + assertNotNull(l); + assertEquals(0, l.size()); + l = w.getDocuments(0, 12); + assertNotNull(l); + assertEquals(0, l.size()); + l = w.getDocuments(0, 0); + assertNotNull(l); + assertEquals(0, l.size()); + } + + private PacketWrapper createResult(CacheKey key, + int offset, int hits, + int total) { + QueryResultPacket r = createQueryResultPacket(offset, hits, total); + return new PacketWrapper(key, new BasicPacket[] {r}); + } + + private QueryResultPacket createQueryResultPacket(int offset, int hits, + int total) { + QueryResultPacket r = QueryResultPacket.create(); + r.setDocstamp(1); + r.setChannel(0); + r.setTotalDocumentCount(total); + r.setOffset(offset); + for (int i = 0; i < hits && i < total; ++i) { + r.addDocument(new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(offset + i), + 1000 - offset - i, 1, 1)); + } + return r; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java new file mode 100644 index 00000000000..ea1494f6168 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/PartialFillTestCase.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.fastsearch.test; + + +import com.yahoo.component.chain.Chain; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.fastsearch.CacheKey; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * @author havardpe + */ +@SuppressWarnings("deprecation") +public class PartialFillTestCase extends junit.framework.TestCase { + + public static class FS4 extends VespaBackEndSearcher { + public List history = new ArrayList<>(); + protected Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) { + return new Result(query); + } + protected void doPartialFill(Result result, String summaryClass) { + history.add(result); + } + } + + public static class BadFS4 extends VespaBackEndSearcher { + protected Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) { + return new Result(query); + } + protected void doPartialFill(Result result, String summaryClass) { + if (result.hits().getErrorHit() == null) { + result.hits().setError(ErrorMessage.createUnspecifiedError("error")); + } + } + } + + public PartialFillTestCase(String name) { + super(name); + } + + public void testPartitioning() { + FS4 fs4 = new FS4(); + Query a = new Query("/?query=foo"); + Query b = new Query("/?query=bar"); + Query c = new Query("/?query=foo"); // equal to a + Result r = new Result(new Query("/?query=ignorethis")); + for (int i = 0; i < 7; i++) { + FastHit h = new FastHit(); + h.setQuery(a); + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 5; i++) { + FastHit h = new FastHit(); + h.setQuery(b); + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 3; i++) { + FastHit h = new FastHit(); + h.setQuery(c); + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 2; i++) { + FastHit h = new FastHit(); + // no query assigned + h.setFillable(); + r.hits().add(h); + } + for (int i = 0; i < 5; i++) { + FastHit h = new FastHit(); + // not fillable + h.setQuery(a); + r.hits().add(h); + } + for (int i = 0; i < 5; i++) { + FastHit h = new FastHit(); + // already filled + h.setQuery(a); + h.setFilled("default"); + r.hits().add(h); + } + doFill(fs4, r, "default"); + assertNull(r.hits().getErrorHit()); + assertEquals(4, fs4.history.size()); + assertEquals(a, fs4.history.get(0).getQuery()); + assertEquals(7, fs4.history.get(0).getHitCount()); + assertEquals(b, fs4.history.get(1).getQuery()); + assertEquals(5, fs4.history.get(1).getHitCount()); + assertEquals(c, fs4.history.get(2).getQuery()); + assertEquals(3, fs4.history.get(2).getHitCount()); + assertEquals(r.getQuery(), fs4.history.get(3).getQuery()); + assertEquals(2, fs4.history.get(3).getHitCount()); + } + + public void testMergeErrors() { + BadFS4 fs4 = new BadFS4(); + Query a = new Query("/?query=foo"); + Query b = new Query("/?query=bar"); + Result r = new Result(new Query("/?query=ignorethis")); + { + FastHit h = new FastHit(); + h.setQuery(a); + h.setFillable(); + r.hits().add(h); + } + { + FastHit h = new FastHit(); + h.setQuery(b); + h.setFillable(); + r.hits().add(h); + } + doFill(fs4, r, "default"); + ErrorHit eh = r.hits().getErrorHit(); + assertNotNull(eh); + ErrorMessage exp_sub = ErrorMessage.createUnspecifiedError("error"); + int n = 0; + for (Iterator i = eh.errorIterator(); i.hasNext();) { + com.yahoo.search.result.ErrorMessage error = i.next(); + switch (n) { + case 0: + assertEquals(exp_sub, error); + break; + case 1: + assertEquals(exp_sub, error); + break; + default: + assertTrue(false); + } + n++; + } + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private void doFill(Searcher searcher, Result result, String summaryClass) { + createExecution(searcher).fill(result, summaryClass); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/category.enum b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/category.enum new file mode 100644 index 00000000000..d42ae9943d6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/category.enum @@ -0,0 +1,20 @@ +19 +top +top/book +top/book/hardcover +top/book/paperback +top/book/textbook +top/music +top/music/cassette +top/music/cd +top/music/dvd +top/music/lp +top/video +top/video/dvd +top/video/vhs +top/wizard +top/wizard/car rental +top/wizard/cruise +top/wizard/flight +top/wizard/hotel +top/wizard/vacation diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg new file mode 100644 index 00000000000..f69e0ed7a54 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg @@ -0,0 +1,351 @@ +documentdb[1] +documentdb[0].name test +documentdb[0].summaryclass[7] +documentdb[0].summaryclass[0].name default +documentdb[0].summaryclass[0].id 0 +documentdb[0].summaryclass[0].fields[19] +documentdb[0].summaryclass[0].fields[0].name URL +documentdb[0].summaryclass[0].fields[0].type string +documentdb[0].summaryclass[0].fields[1].name TITLE +documentdb[0].summaryclass[0].fields[1].type string +documentdb[0].summaryclass[0].fields[2].name TEASER +documentdb[0].summaryclass[0].fields[2].type string +documentdb[0].summaryclass[0].fields[3].name TOPIC +documentdb[0].summaryclass[0].fields[3].type string +documentdb[0].summaryclass[0].fields[4].name FASTTOPIC +documentdb[0].summaryclass[0].fields[4].type string +documentdb[0].summaryclass[0].fields[5].name EXTINFO +documentdb[0].summaryclass[0].fields[5].type string +documentdb[0].summaryclass[0].fields[6].name EXTINFOSOURCE +documentdb[0].summaryclass[0].fields[6].type byte +documentdb[0].summaryclass[0].fields[7].name DSHOST +documentdb[0].summaryclass[0].fields[7].type integer +documentdb[0].summaryclass[0].fields[8].name DSKEY +documentdb[0].summaryclass[0].fields[8].type integer +documentdb[0].summaryclass[0].fields[9].name BYTES +documentdb[0].summaryclass[0].fields[9].type integer +documentdb[0].summaryclass[0].fields[10].name WORDS +documentdb[0].summaryclass[0].fields[10].type integer +documentdb[0].summaryclass[0].fields[11].name MODDATE +documentdb[0].summaryclass[0].fields[11].type integer +documentdb[0].summaryclass[0].fields[12].name CRAWLDATE +documentdb[0].summaryclass[0].fields[12].type integer +documentdb[0].summaryclass[0].fields[13].name LANG1 +documentdb[0].summaryclass[0].fields[13].type byte +documentdb[0].summaryclass[0].fields[14].name LANG2 +documentdb[0].summaryclass[0].fields[14].type byte +documentdb[0].summaryclass[0].fields[15].name LANG3 +documentdb[0].summaryclass[0].fields[15].type byte +documentdb[0].summaryclass[0].fields[16].name LANG4 +documentdb[0].summaryclass[0].fields[16].type byte +documentdb[0].summaryclass[0].fields[17].name IPADDRESS +documentdb[0].summaryclass[0].fields[17].type integer +documentdb[0].summaryclass[0].fields[18].name DOCVECTOR +documentdb[0].summaryclass[0].fields[18].type data +documentdb[0].summaryclass[1].name version1 +documentdb[0].summaryclass[1].id 1 +documentdb[0].summaryclass[1].fields[20] +documentdb[0].summaryclass[1].fields[0].name URL +documentdb[0].summaryclass[1].fields[0].type string +documentdb[0].summaryclass[1].fields[1].name TITLE +documentdb[0].summaryclass[1].fields[1].type string +documentdb[0].summaryclass[1].fields[2].name TEASER +documentdb[0].summaryclass[1].fields[2].type string +documentdb[0].summaryclass[1].fields[3].name TOPIC +documentdb[0].summaryclass[1].fields[3].type string +documentdb[0].summaryclass[1].fields[4].name FASTTOPIC +documentdb[0].summaryclass[1].fields[4].type string +documentdb[0].summaryclass[1].fields[5].name EXTINFO +documentdb[0].summaryclass[1].fields[5].type string +documentdb[0].summaryclass[1].fields[6].name EXTINFOSOURCE +documentdb[0].summaryclass[1].fields[6].type byte +documentdb[0].summaryclass[1].fields[7].name DSHOST +documentdb[0].summaryclass[1].fields[7].type integer +documentdb[0].summaryclass[1].fields[8].name DSKEY +documentdb[0].summaryclass[1].fields[8].type integer +documentdb[0].summaryclass[1].fields[9].name BYTES +documentdb[0].summaryclass[1].fields[9].type integer +documentdb[0].summaryclass[1].fields[10].name WORDS +documentdb[0].summaryclass[1].fields[10].type integer +documentdb[0].summaryclass[1].fields[11].name MODDATE +documentdb[0].summaryclass[1].fields[11].type integer +documentdb[0].summaryclass[1].fields[12].name CRAWLDATE +documentdb[0].summaryclass[1].fields[12].type integer +documentdb[0].summaryclass[1].fields[13].name LANG1 +documentdb[0].summaryclass[1].fields[13].type byte +documentdb[0].summaryclass[1].fields[14].name LANG2 +documentdb[0].summaryclass[1].fields[14].type byte +documentdb[0].summaryclass[1].fields[15].name LANG3 +documentdb[0].summaryclass[1].fields[15].type byte +documentdb[0].summaryclass[1].fields[16].name LANG4 +documentdb[0].summaryclass[1].fields[16].type byte +documentdb[0].summaryclass[1].fields[17].name IPADDRESS +documentdb[0].summaryclass[1].fields[17].type integer +documentdb[0].summaryclass[1].fields[18].name DOCVECTOR +documentdb[0].summaryclass[1].fields[18].type data +documentdb[0].summaryclass[1].fields[19].name PARTNERSITEIDS +documentdb[0].summaryclass[1].fields[19].type string +documentdb[0].summaryclass[2].name version2 +documentdb[0].summaryclass[2].id 2 +documentdb[0].summaryclass[2].fields[21] +documentdb[0].summaryclass[2].fields[0].name URL +documentdb[0].summaryclass[2].fields[0].type string +documentdb[0].summaryclass[2].fields[1].name TITLE +documentdb[0].summaryclass[2].fields[1].type string +documentdb[0].summaryclass[2].fields[2].name TEASER +documentdb[0].summaryclass[2].fields[2].type string +documentdb[0].summaryclass[2].fields[3].name TOPIC +documentdb[0].summaryclass[2].fields[3].type string +documentdb[0].summaryclass[2].fields[4].name FASTTOPIC +documentdb[0].summaryclass[2].fields[4].type string +documentdb[0].summaryclass[2].fields[5].name EXTINFO +documentdb[0].summaryclass[2].fields[5].type string +documentdb[0].summaryclass[2].fields[6].name EXTINFOSOURCE +documentdb[0].summaryclass[2].fields[6].type byte +documentdb[0].summaryclass[2].fields[7].name DSHOST +documentdb[0].summaryclass[2].fields[7].type integer +documentdb[0].summaryclass[2].fields[8].name DSKEY +documentdb[0].summaryclass[2].fields[8].type integer +documentdb[0].summaryclass[2].fields[9].name BYTES +documentdb[0].summaryclass[2].fields[9].type integer +documentdb[0].summaryclass[2].fields[10].name WORDS +documentdb[0].summaryclass[2].fields[10].type integer +documentdb[0].summaryclass[2].fields[11].name MODDATE +documentdb[0].summaryclass[2].fields[11].type integer +documentdb[0].summaryclass[2].fields[12].name CRAWLDATE +documentdb[0].summaryclass[2].fields[12].type integer +documentdb[0].summaryclass[2].fields[13].name LANG1 +documentdb[0].summaryclass[2].fields[13].type byte +documentdb[0].summaryclass[2].fields[14].name LANG2 +documentdb[0].summaryclass[2].fields[14].type byte +documentdb[0].summaryclass[2].fields[15].name LANG3 +documentdb[0].summaryclass[2].fields[15].type byte +documentdb[0].summaryclass[2].fields[16].name LANG4 +documentdb[0].summaryclass[2].fields[16].type byte +documentdb[0].summaryclass[2].fields[17].name IPADDRESS +documentdb[0].summaryclass[2].fields[17].type integer +documentdb[0].summaryclass[2].fields[18].name DOCVECTOR +documentdb[0].summaryclass[2].fields[18].type data +documentdb[0].summaryclass[2].fields[19].name PARTNERSITEIDS +documentdb[0].summaryclass[2].fields[19].type string +documentdb[0].summaryclass[2].fields[20].name DYNTEASER +documentdb[0].summaryclass[2].fields[20].type string +documentdb[0].summaryclass[3].name version3 +documentdb[0].summaryclass[3].id 3 +documentdb[0].summaryclass[3].fields[23] +documentdb[0].summaryclass[3].fields[0].name URL +documentdb[0].summaryclass[3].fields[0].type string +documentdb[0].summaryclass[3].fields[1].name TITLE +documentdb[0].summaryclass[3].fields[1].type string +documentdb[0].summaryclass[3].fields[2].name TEASER +documentdb[0].summaryclass[3].fields[2].type string +documentdb[0].summaryclass[3].fields[3].name TOPIC +documentdb[0].summaryclass[3].fields[3].type string +documentdb[0].summaryclass[3].fields[4].name FASTTOPIC +documentdb[0].summaryclass[3].fields[4].type string +documentdb[0].summaryclass[3].fields[5].name EXTINFO +documentdb[0].summaryclass[3].fields[5].type string +documentdb[0].summaryclass[3].fields[6].name EXTINFOSOURCE +documentdb[0].summaryclass[3].fields[6].type byte +documentdb[0].summaryclass[3].fields[7].name DSHOST +documentdb[0].summaryclass[3].fields[7].type integer +documentdb[0].summaryclass[3].fields[8].name DSKEY +documentdb[0].summaryclass[3].fields[8].type integer +documentdb[0].summaryclass[3].fields[9].name BYTES +documentdb[0].summaryclass[3].fields[9].type integer +documentdb[0].summaryclass[3].fields[10].name WORDS +documentdb[0].summaryclass[3].fields[10].type integer +documentdb[0].summaryclass[3].fields[11].name MODDATE +documentdb[0].summaryclass[3].fields[11].type integer +documentdb[0].summaryclass[3].fields[12].name CRAWLDATE +documentdb[0].summaryclass[3].fields[12].type integer +documentdb[0].summaryclass[3].fields[13].name LANG1 +documentdb[0].summaryclass[3].fields[13].type byte +documentdb[0].summaryclass[3].fields[14].name LANG2 +documentdb[0].summaryclass[3].fields[14].type byte +documentdb[0].summaryclass[3].fields[15].name LANG3 +documentdb[0].summaryclass[3].fields[15].type byte +documentdb[0].summaryclass[3].fields[16].name LANG4 +documentdb[0].summaryclass[3].fields[16].type byte +documentdb[0].summaryclass[3].fields[17].name IPADDRESS +documentdb[0].summaryclass[3].fields[17].type integer +documentdb[0].summaryclass[3].fields[18].name DOCVECTOR +documentdb[0].summaryclass[3].fields[18].type data +documentdb[0].summaryclass[3].fields[19].name PARTNERSITEIDS +documentdb[0].summaryclass[3].fields[19].type string +documentdb[0].summaryclass[3].fields[20].name MIMETYPE +documentdb[0].summaryclass[3].fields[20].type string +documentdb[0].summaryclass[3].fields[21].name STATICRANKLOG +documentdb[0].summaryclass[3].fields[21].type string +documentdb[0].summaryclass[3].fields[22].name DYNTEASER +documentdb[0].summaryclass[3].fields[22].type longstring +documentdb[0].summaryclass[4].name version4 +documentdb[0].summaryclass[4].id 4 +documentdb[0].summaryclass[4].fields[24] +documentdb[0].summaryclass[4].fields[0].name URL +documentdb[0].summaryclass[4].fields[0].type string +documentdb[0].summaryclass[4].fields[1].name CCURL +documentdb[0].summaryclass[4].fields[1].type string +documentdb[0].summaryclass[4].fields[2].name TITLE +documentdb[0].summaryclass[4].fields[2].type string +documentdb[0].summaryclass[4].fields[3].name TEASER +documentdb[0].summaryclass[4].fields[3].type string +documentdb[0].summaryclass[4].fields[4].name TOPIC +documentdb[0].summaryclass[4].fields[4].type string +documentdb[0].summaryclass[4].fields[5].name FASTTOPIC +documentdb[0].summaryclass[4].fields[5].type string +documentdb[0].summaryclass[4].fields[6].name EXTINFO +documentdb[0].summaryclass[4].fields[6].type string +documentdb[0].summaryclass[4].fields[7].name EXTINFOSOURCE +documentdb[0].summaryclass[4].fields[7].type byte +documentdb[0].summaryclass[4].fields[8].name DSHOST +documentdb[0].summaryclass[4].fields[8].type integer +documentdb[0].summaryclass[4].fields[9].name DSKEY +documentdb[0].summaryclass[4].fields[9].type integer +documentdb[0].summaryclass[4].fields[10].name BYTES +documentdb[0].summaryclass[4].fields[10].type integer +documentdb[0].summaryclass[4].fields[11].name WORDS +documentdb[0].summaryclass[4].fields[11].type integer +documentdb[0].summaryclass[4].fields[12].name MODDATE +documentdb[0].summaryclass[4].fields[12].type integer +documentdb[0].summaryclass[4].fields[13].name CRAWLDATE +documentdb[0].summaryclass[4].fields[13].type integer +documentdb[0].summaryclass[4].fields[14].name LANG1 +documentdb[0].summaryclass[4].fields[14].type byte +documentdb[0].summaryclass[4].fields[15].name LANG2 +documentdb[0].summaryclass[4].fields[15].type byte +documentdb[0].summaryclass[4].fields[16].name LANG3 +documentdb[0].summaryclass[4].fields[16].type byte +documentdb[0].summaryclass[4].fields[17].name LANG4 +documentdb[0].summaryclass[4].fields[17].type byte +documentdb[0].summaryclass[4].fields[18].name IPADDRESS +documentdb[0].summaryclass[4].fields[18].type integer +documentdb[0].summaryclass[4].fields[19].name DOCVECTOR +documentdb[0].summaryclass[4].fields[19].type data +documentdb[0].summaryclass[4].fields[20].name PARTNERSITEIDS +documentdb[0].summaryclass[4].fields[20].type string +documentdb[0].summaryclass[4].fields[21].name MIMETYPE +documentdb[0].summaryclass[4].fields[21].type string +documentdb[0].summaryclass[4].fields[22].name STATICRANKLOG +documentdb[0].summaryclass[4].fields[22].type string +documentdb[0].summaryclass[4].fields[23].name DYNTEASER +documentdb[0].summaryclass[4].fields[23].type longstring +documentdb[0].summaryclass[5].name version5 +documentdb[0].summaryclass[5].id 5 +documentdb[0].summaryclass[5].fields[25] +documentdb[0].summaryclass[5].fields[0].name URL +documentdb[0].summaryclass[5].fields[0].type string +documentdb[0].summaryclass[5].fields[1].name URLLIST +documentdb[0].summaryclass[5].fields[1].type string +documentdb[0].summaryclass[5].fields[2].name CCURL +documentdb[0].summaryclass[5].fields[2].type string +documentdb[0].summaryclass[5].fields[3].name TITLE +documentdb[0].summaryclass[5].fields[3].type string +documentdb[0].summaryclass[5].fields[4].name TEASER +documentdb[0].summaryclass[5].fields[4].type string +documentdb[0].summaryclass[5].fields[5].name TOPIC +documentdb[0].summaryclass[5].fields[5].type string +documentdb[0].summaryclass[5].fields[6].name FASTTOPIC +documentdb[0].summaryclass[5].fields[6].type string +documentdb[0].summaryclass[5].fields[7].name EXTINFO +documentdb[0].summaryclass[5].fields[7].type string +documentdb[0].summaryclass[5].fields[8].name EXTINFOSOURCE +documentdb[0].summaryclass[5].fields[8].type byte +documentdb[0].summaryclass[5].fields[9].name DSHOST +documentdb[0].summaryclass[5].fields[9].type integer +documentdb[0].summaryclass[5].fields[10].name DSKEY +documentdb[0].summaryclass[5].fields[10].type integer +documentdb[0].summaryclass[5].fields[11].name BYTES +documentdb[0].summaryclass[5].fields[11].type integer +documentdb[0].summaryclass[5].fields[12].name WORDS +documentdb[0].summaryclass[5].fields[12].type integer +documentdb[0].summaryclass[5].fields[13].name MODDATE +documentdb[0].summaryclass[5].fields[13].type integer +documentdb[0].summaryclass[5].fields[14].name CRAWLDATE +documentdb[0].summaryclass[5].fields[14].type integer +documentdb[0].summaryclass[5].fields[15].name LANG1 +documentdb[0].summaryclass[5].fields[15].type byte +documentdb[0].summaryclass[5].fields[16].name LANG2 +documentdb[0].summaryclass[5].fields[16].type byte +documentdb[0].summaryclass[5].fields[17].name LANG3 +documentdb[0].summaryclass[5].fields[17].type byte +documentdb[0].summaryclass[5].fields[18].name LANG4 +documentdb[0].summaryclass[5].fields[18].type byte +documentdb[0].summaryclass[5].fields[19].name IPADDRESS +documentdb[0].summaryclass[5].fields[19].type integer +documentdb[0].summaryclass[5].fields[20].name DOCVECTOR +documentdb[0].summaryclass[5].fields[20].type data +documentdb[0].summaryclass[5].fields[21].name PARTNERSITEIDS +documentdb[0].summaryclass[5].fields[21].type string +documentdb[0].summaryclass[5].fields[22].name MIMETYPE +documentdb[0].summaryclass[5].fields[22].type string +documentdb[0].summaryclass[5].fields[23].name STATICRANKLOG +documentdb[0].summaryclass[5].fields[23].type string +documentdb[0].summaryclass[5].fields[24].name DYNTEASER +documentdb[0].summaryclass[5].fields[24].type longstring +documentdb[0].summaryclass[6].name withranklog +documentdb[0].summaryclass[6].id 237 +documentdb[0].summaryclass[6].fields[31] +documentdb[0].summaryclass[6].fields[0].name BYTES +documentdb[0].summaryclass[6].fields[0].type integer +documentdb[0].summaryclass[6].fields[1].name CCURL +documentdb[0].summaryclass[6].fields[1].type string +documentdb[0].summaryclass[6].fields[2].name CRAWLDATE +documentdb[0].summaryclass[6].fields[2].type integer +documentdb[0].summaryclass[6].fields[3].name DOCVECTOR +documentdb[0].summaryclass[6].fields[3].type data +documentdb[0].summaryclass[6].fields[4].name DSHOST +documentdb[0].summaryclass[6].fields[4].type integer +documentdb[0].summaryclass[6].fields[5].name DSKEY +documentdb[0].summaryclass[6].fields[5].type integer +documentdb[0].summaryclass[6].fields[6].name DYNTEASER +documentdb[0].summaryclass[6].fields[6].type longstring +documentdb[0].summaryclass[6].fields[7].name DYNTEASERINPUT +documentdb[0].summaryclass[6].fields[7].type longstring +documentdb[0].summaryclass[6].fields[8].name EXTINFO +documentdb[0].summaryclass[6].fields[8].type string +documentdb[0].summaryclass[6].fields[9].name EXTINFOSOURCE +documentdb[0].summaryclass[6].fields[9].type byte +documentdb[0].summaryclass[6].fields[10].name FASTTOPIC +documentdb[0].summaryclass[6].fields[10].type string +documentdb[0].summaryclass[6].fields[11].name IPADDRESS +documentdb[0].summaryclass[6].fields[11].type integer +documentdb[0].summaryclass[6].fields[12].name JUNIPER +documentdb[0].summaryclass[6].fields[12].type longstring +documentdb[0].summaryclass[6].fields[13].name JUNIPERMETRIC +documentdb[0].summaryclass[6].fields[13].type integer +documentdb[0].summaryclass[6].fields[14].name LABEL +documentdb[0].summaryclass[6].fields[14].type string +documentdb[0].summaryclass[6].fields[15].name LANG1 +documentdb[0].summaryclass[6].fields[15].type byte +documentdb[0].summaryclass[6].fields[16].name LANG2 +documentdb[0].summaryclass[6].fields[16].type byte +documentdb[0].summaryclass[6].fields[17].name LANG3 +documentdb[0].summaryclass[6].fields[17].type byte +documentdb[0].summaryclass[6].fields[18].name LANG4 +documentdb[0].summaryclass[6].fields[18].type byte +documentdb[0].summaryclass[6].fields[19].name MIMETYPE +documentdb[0].summaryclass[6].fields[19].type string +documentdb[0].summaryclass[6].fields[20].name MODDATE +documentdb[0].summaryclass[6].fields[20].type integer +documentdb[0].summaryclass[6].fields[21].name PARTNERSITEIDS +documentdb[0].summaryclass[6].fields[21].type string +documentdb[0].summaryclass[6].fields[22].name RANKLOG +documentdb[0].summaryclass[6].fields[22].type string +documentdb[0].summaryclass[6].fields[23].name STATICRANK +documentdb[0].summaryclass[6].fields[23].type integer +documentdb[0].summaryclass[6].fields[24].name STATICRANKLOG +documentdb[0].summaryclass[6].fields[24].type string +documentdb[0].summaryclass[6].fields[25].name TEASER +documentdb[0].summaryclass[6].fields[25].type string +documentdb[0].summaryclass[6].fields[26].name TITLE +documentdb[0].summaryclass[6].fields[26].type string +documentdb[0].summaryclass[6].fields[27].name TOPIC +documentdb[0].summaryclass[6].fields[27].type string +documentdb[0].summaryclass[6].fields[28].name URL +documentdb[0].summaryclass[6].fields[28].type string +documentdb[0].summaryclass[6].fields[29].name URLLIST +documentdb[0].summaryclass[6].fields[29].type string +documentdb[0].summaryclass[6].fields[30].name WORDS +documentdb[0].summaryclass[6].fields[30].type integer +documentdb[0].rankprofile[0] diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/updated-qr-summary-dummy.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/updated-qr-summary-dummy.cfg new file mode 100644 index 00000000000..dca2c0db2b6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/updated-qr-summary-dummy.cfg @@ -0,0 +1,15 @@ +idtype BYTE +classes[1] +classes[0].name default +classes[0].id 0 +classes[0].fields[5] +classes[0].fields[0].name URL +classes[0].fields[0].type string +classes[0].fields[1].name TITLE +classes[0].fields[1].type string +classes[0].fields[2].name TEASER +classes[0].fields[2].type string +classes[0].fields[3].name TOPIC_UPDATED +classes[0].fields[3].type string +classes[0].fields[4].name FASTTOPIC +classes[0].fields[4].type string diff --git a/container-search/src/test/java/com/yahoo/prelude/grouping/legacy/test/.gitignore b/container-search/src/test/java/com/yahoo/prelude/grouping/legacy/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/prelude/hitfield/XmlRendererTestCase.java b/container-search/src/test/java/com/yahoo/prelude/hitfield/XmlRendererTestCase.java new file mode 100644 index 00000000000..6b3b48cd098 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/hitfield/XmlRendererTestCase.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.hitfield; + +import com.yahoo.data.access.simple.Value; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; + +public class XmlRendererTestCase extends junit.framework.TestCase { + + public void testWeightedSet1() { + Value.ArrayValue top = new Value.ArrayValue(); + top + .add(new Value.ArrayValue() + .add(new Value.StringValue("per")) + .add(new Value.LongValue(10))) + .add(new Value.ArrayValue() + .add(new Value.StringValue("paal")) + .add(new Value.LongValue(20))) + .add(new Value.ArrayValue() + .add(new Value.StringValue("espen")) + .add(new Value.LongValue(30))); + String rendered = XmlRenderer.render(new StringBuilder(), top).toString(); +//System.err.println("rendered >>>"); +//System.err.println(rendered); +//System.err.println("<<< rendered"); + String correct = "\n" + + " per\n" + + " paal\n" + + " espen\n" + + " "; + assertEquals(correct, rendered); + } + + public void testWeightedSet2() { + Value.ObjectValue top = new Value.ObjectValue(); + top + .put("foo", new Value.ArrayValue() + .add(new Value.ArrayValue() + .add(new Value.StringValue("per")) + .add(new Value.LongValue(10))) + .add(new Value.ArrayValue() + .add(new Value.StringValue("paal")) + .add(new Value.LongValue(20))) + .add(new Value.ArrayValue() + .add(new Value.StringValue("espen")) + .add(new Value.LongValue(30)))) + .put("bar", new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("item",new Value.StringValue("per")) + .put("weight",new Value.LongValue(10))) + .add(new Value.ObjectValue() + .put("item",new Value.StringValue("paal")) + .put("weight",new Value.LongValue(20))) + .add(new Value.ObjectValue() + .put("weight",new Value.LongValue(30)) + .put("item",new Value.StringValue("espen")))); + String rendered = XmlRenderer.render(new StringBuilder(), top).toString(); +//System.err.println("rendered >>>"); +//System.err.println(rendered); +//System.err.println("<<< rendered"); + String correct = "\n" + + " \n" + + " per\n" + + " paal\n" + + " espen\n" + + " \n" + + " \n" + + " per\n" + + " paal\n" + + " espen\n" + + " \n" + + " "; + assertEquals(correct, rendered); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/HitFieldTestCase.java b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/HitFieldTestCase.java new file mode 100644 index 00000000000..3fc0a52382e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/HitFieldTestCase.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.hitfield.test; + +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.prelude.hitfield.HitField; +import com.yahoo.prelude.hitfield.StringFieldPart; + +/** + * Tests the HitField class + * + * @author Lars Chr Jensen + */ +@SuppressWarnings({"unchecked", "rawtypes"}) +public class HitFieldTestCase extends junit.framework.TestCase { + + public HitFieldTestCase (String name) { + super(name); + } + + public void testHitField() { + HitField hf = new HitField("boo", "hei paa deg"); + assertEquals(3, hf.getTokenizedContent().size()); + List l = new ArrayList(); + l.add(new StringFieldPart("foo", true)); + l.add(new StringFieldPart(" ", false)); + l.add(new StringFieldPart("bar", true)); + hf.setTokenizedContent(l); + assertEquals("foo bar", hf.getContent()); + assertEquals("hei paa deg", hf.getRawContent()); + } + + public void testCjk() { + HitField hf = new HitField("boo", "hmm\u001fgr"); + assertEquals(2, hf.getTokenizedContent().size()); + assertEquals("hmmgr", hf.getContent()); + List l = new ArrayList(); + l.add(new StringFieldPart("foo", true)); + l.add(new StringFieldPart("bar", true)); + hf.setTokenizedContent(l); + assertEquals("foobar", hf.getContent()); + } + + public void testAnnotateField() { + HitField hf = new HitField("boo", "The Eclipse SDK \uFFF9include\uFFFAincludes\uFFFB the Eclipse Platform"); + assertEquals(11, hf.getTokenizedContent().size()); + hf = new HitField("boo", "\uFFF9include\uFFFAincludes\uFFFB the Eclipse Platform"); + assertEquals(6, hf.getTokenizedContent().size()); + hf = new HitField("boo", "clude\uFFFAincludes\uFFFB the Eclipse Platform"); + assertEquals(5, hf.getTokenizedContent().size()); + hf = new HitField("boo", "\uFFFAincludes\uFFFB the Eclipse Platform"); + assertEquals(5, hf.getTokenizedContent().size()); + hf = new HitField("boo", "cludes\uFFFB the Eclipse Platform"); + assertEquals(5, hf.getTokenizedContent().size()); + hf = new HitField("boo", "\uFFFB the Eclipse Platform"); + assertEquals(5, hf.getTokenizedContent().size()); + hf = new HitField("boo", "The Eclipse SDK \uFFF9include\uFFFAincludes\uFFFB"); + assertEquals(6, hf.getTokenizedContent().size()); + hf = new HitField("boo", "The Eclipse SDK \uFFF9include\uFFFAincl"); + assertEquals(6, hf.getTokenizedContent().size()); + hf = new HitField("boo", "The Eclipse SDK \uFFF9include\uFFFA"); + assertEquals(6, hf.getTokenizedContent().size()); + hf = new HitField("boo", "The Eclipse SDK \uFFF9incl"); + assertEquals(6, hf.getTokenizedContent().size()); + hf = new HitField("boo", "The Eclipse SDK \uFFF9"); + assertEquals(6, hf.getTokenizedContent().size()); + hf = new HitField("boo", "The Eclipse SDK \uFFF9include\uFFFAincludes\uFFFB the Eclipse \uFFF9platform\uFFFAPlatforms\uFFFB test"); + assertEquals(12, hf.getTokenizedContent().size()); + + + //hf = new HitField("boo", "The Eclipse SDK \uFFF9include\uFFFAincludes\uFFFB the Eclipse Platform"); + } + public void testEmptyField() { + HitField hf = new HitField("boo", ""); + assertEquals(0, hf.getTokenizedContent().size()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java new file mode 100644 index 00000000000..a3e1a183d1a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java @@ -0,0 +1,862 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.hitfield.test; + +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.data.access.simple.Value; +import com.yahoo.data.access.slime.SlimeAdapter; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; +import com.yahoo.slime.Slime; +import com.yahoo.slime.Cursor; + +/** + * Tests the JSONString XML rendering. + * + * TODO: Add correct answers. These are not added because this code was checked in before sync with system test + * + * @author Steinar Knutsen + */ +public class JSONStringTestCase extends junit.framework.TestCase { + public void testWeightedSet() { + String json = "[[{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788997,\"d1\":87.790001,\"i1\":7,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" + + "espa\u00F1a\\n" + + "wssf1.s1[0]\"},10]," + + "[{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788998,\"d1\":88.790001,\"i1\":8,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" + + "espa\u00F1a wssf1.s1[1]\"},20]]"; + JSONString js = new JSONString(json); + String o1 = " \n"; + String[] o1Fields = { + " 1122334455667788997\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 7\n", + " 87.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " string\n" + "españa\n" + + "wssf1.s1[0]\n" }; + String o2 = " \n"; + String[] o2Fields = { + " 1122334455667788998\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 8\n", + " 88.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " string\n" + + "españa wssf1.s1[1]\n" }; + String rendered = js.toString(); + int o1Offset = rendered.indexOf(o1); + assertTrue(-1 < o1Offset); + int o2Offset = rendered.indexOf(o2); + assertTrue(-1 < o2Offset); + + checkSubstrings(o1Fields, rendered.substring(o1Offset, o2Offset)); + checkSubstrings(o2Fields, rendered, o2Offset); + + } + + public void testWeightedSetFromInspector() { + Value.ArrayValue top = new Value.ArrayValue(); + top.add(new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("d1", new Value.DoubleValue(87.790001)) + .put("s1", new Value.StringValue("string\n" + "espa\u00F1a\n" + "wssf1.s1[0]")) + .put("al1", new Value.ArrayValue() + .add(new Value.LongValue(11223344556677881L)) + .add(new Value.LongValue(11223344556677883L))) + .put("l1", new Value.LongValue(1122334455667788997L)) + .put("as1", new Value.ArrayValue() + .add(new Value.StringValue("per")) + .add(new Value.StringValue("paal"))) + .put("i1", new Value.LongValue(7))) + .add(new Value.LongValue(10))) + .add(new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("d1", new Value.DoubleValue(88.790001)) + .put("s1", new Value.StringValue("string\n" + "espa\u00F1a wssf1.s1[1]")) + .put("al1", new Value.ArrayValue() + .add(new Value.LongValue(11223344556677881L)) + .add(new Value.LongValue(11223344556677883L))) + .put("l1", new Value.LongValue(1122334455667788998L)) + .put("as1", new Value.ArrayValue() + .add(new Value.StringValue("per")) + .add(new Value.StringValue("paal"))) + .put("i1", new Value.LongValue(8))) + .add(new Value.LongValue(20))); + + JSONString js = new JSONString(top); + String correct = "\n" + + " \n" + + " 87.790001\n" + + " string\n" + + "espa\u00F1a\n" + + "wssf1.s1[0]\n" + + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n" + + " 1122334455667788997\n" + + " \n" + + " per\n" + + " paal\n" + + " \n" + + " 7\n" + + " \n" + + " \n" + + " 88.790001\n" + + " string\n" + + "espa\u00F1a wssf1.s1[1]\n" + + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n" + + " 1122334455667788998\n" + + " \n" + + " per\n" + + " paal\n" + + " \n" + + " 8\n" + + " \n" + + " "; + assertEquals(correct, js.renderFromInspector()); + + top = new Value.ArrayValue(); + top.add(new Value.ArrayValue() + .add(new Value.StringValue("s1")) + .add(new Value.LongValue(10))) + .add(new Value.ArrayValue() + .add(new Value.StringValue("s2")) + .add(new Value.LongValue(20))); + js = new JSONString(top); +// System.err.println("js.toString() is: >>>" + js.toString() + "<<<"); + correct = "\n" + + " s1\n" + + " s2\n" + + " "; + assertEquals(correct, js.renderFromInspector()); + } + + public void testStruct() { + { + String json = "{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788991,\"d1\":81.790001,\"i1\":1,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" + + "espa\u00F1a ssf1.s1\"}"; + JSONString js = new JSONString(json); + String[] renderedFields = { + " 1122334455667788991\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 1\n", + " 81.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " string\n" + + "españa ssf1.s1\n" }; + String rendered = js.toString(); + checkSubstrings(renderedFields, rendered); + } + { + Value.ObjectValue top = new Value.ObjectValue(); + top.put("d1", new Value.DoubleValue(81.790001)) + .put("s1", + new Value.StringValue("string\nespa\u00F1a ssf1.s1")) + .put("al1", + new Value.ArrayValue() + .add(new Value.LongValue(11223344556677881L)) + .add(new Value.LongValue(11223344556677883L))) + .put("l1", new Value.LongValue(1122334455667788991L)) + .put("as1", + new Value.ArrayValue().add( + new Value.StringValue("per")).add( + new Value.StringValue("paal"))) + .put("i1", new Value.LongValue(1)); + JSONString js = new JSONString(top); + + String[] renderedFields = { + " 81.790001\n", + " string\n" + + "españa ssf1.s1\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 1122334455667788991\n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " 1\n" }; + + String rendered = js.renderFromInspector(); + checkSubstrings(renderedFields, rendered); + } + { + String json = "{\"as1\":[\"per\",\"paal\"],\"d1\":84.790001,\"i1\":4,\"al1\":[11223344556677881,11223344556677883]}"; + JSONString js = new JSONString(json); + String[] renderedFields = { + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 4\n", + " 84.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n " }; + String rendered = js.toString(); + + checkSubstrings(renderedFields, rendered); + } + { + Value.ObjectValue top = new Value.ObjectValue(); + top.put("d1", new Value.DoubleValue(84.790001)) + .put("al1", + new Value.ArrayValue() + .add(new Value.LongValue(11223344556677881L)) + .add(new Value.LongValue(11223344556677883L))) + .put("as1", + new Value.ArrayValue().add( + new Value.StringValue("per")).add( + new Value.StringValue("paal"))) + .put("i1", new Value.LongValue(4)); + JSONString js = new JSONString(top); + + String[] renderedFields = { + " 84.790001\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " 4\n " }; + + String rendered = js.renderFromInspector(); + checkSubstrings(renderedFields, rendered); + + } + { + String json = "{\"s2\":\"string espa\u00F1a\\n" + + "ssf5.s2\",\"nss1\":{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788995,\"d1\":85.790001,\"i1\":5,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" + + "espa\u00F1a ssf5.nss1.s1\"}}"; + JSONString js = new JSONString(json); + String[] renderedFields = { + " \n", + " string\n" + + "españa ssf5.nss1.s1\n", + " string españa\n" + + "ssf5.s2\n " }; + String nss1Fields[] = { + " 1122334455667788995\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 5\n", + " 85.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n" }; + + String rendered = js.toString(); + checkSubstrings(renderedFields, rendered); + int nss1Offset = rendered.indexOf(renderedFields[0]) + + renderedFields[0].length(); + checkSubstrings(nss1Fields, rendered, nss1Offset); + } + { + Value.ObjectValue top = new Value.ObjectValue(); + top.put("s2", "string espa\u00F1a\nssf5.s2").put( + "nss1", + new Value.ObjectValue() + .put("d1", new Value.DoubleValue(85.790001)) + .put("s1", "string\nespa\u00F1a ssf5.nss1.s1") + .put("al1", + new Value.ArrayValue().add( + new Value.LongValue( + 11223344556677881L)).add( + new Value.LongValue( + 11223344556677883L))) + .put("l1", 1122334455667788995L) + .put("as1", + new Value.ArrayValue().add( + new Value.StringValue("per")).add( + new Value.StringValue("paal"))) + .put("i1", 5)); + JSONString js = new JSONString(top); + + String f1 = " string españa\n" + + "ssf5.s2"; + String f2 = " \n"; + String f2_1 = " 85.790001\n"; + String f2_2 = " string\n" + + "españa ssf5.nss1.s1\n"; + String f2_3 = " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n"; + String f2_4 = " 1122334455667788995\n"; + String f2_5 = " \n" + + " per\n" + + " paal\n" + + " \n"; + String f2_6 = " 5\n"; + String f2_end = " \n "; + String rendered = js.renderFromInspector(); + + assertTrue(-1 < rendered.indexOf(f1)); + int offsetF2; + assertTrue(-1 < (offsetF2 = rendered.indexOf(f2))); + offsetF2 += f2.length(); + assertTrue(-1 < rendered.indexOf(f2_1, offsetF2)); + assertTrue(-1 < rendered.indexOf(f2_2, offsetF2)); + assertTrue(-1 < rendered.indexOf(f2_3, offsetF2)); + assertTrue(-1 < rendered.indexOf(f2_4, offsetF2)); + assertTrue(-1 < rendered.indexOf(f2_5, offsetF2)); + assertTrue(-1 < rendered.indexOf(f2_6, offsetF2)); + final int expectedEnd = offsetF2 + f2_1.length() + f2_2.length() + f2_3.length() + + f2_4.length() + f2_5.length() + f2_6.length(); + assertEquals( + expectedEnd, + rendered.indexOf( + f2_end, + expectedEnd)); + } + { + String json = "{\"s2\":\"string espa\u00F1a\\n" + + "ssf8.s2\",\"nss1\":{\"as1\":[\"per\",\"paal\"],\"d1\":88.790001,\"i1\":8,\"al1\":[11223344556677881,11223344556677883]}}"; + JSONString js = new JSONString(json); + + String[] renderedFields = { + " \n", + " string españa\n" + + "ssf8.s2\n " }; + String nss1Fields[] = { + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 8\n", + " 88.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n" }; + + String rendered = js.toString(); + checkSubstrings(renderedFields, rendered); + int nss1Offset = rendered.indexOf(renderedFields[0]) + + renderedFields[0].length(); + checkSubstrings(nss1Fields, rendered, nss1Offset); + + } + { + Value.ObjectValue top = new Value.ObjectValue(); + top.put("s2", "string espa\u00F1a\nssf8.s2").put( + "nss1", + new Value.ObjectValue() + .put("d1", new Value.DoubleValue(88.790001)) + .put("al1", + new Value.ArrayValue().add( + new Value.LongValue( + 11223344556677881L)).add( + new Value.LongValue( + 11223344556677883L))) + .put("as1", + new Value.ArrayValue().add( + new Value.StringValue("per")).add( + new Value.StringValue("paal"))) + .put("i1", 8)); + JSONString js = new JSONString(top); + String rendered = js.renderFromInspector(); + String[] renderedFields = { + " \n", + " string españa\n" + + "ssf8.s2\n " }; + String nss1Fields[] = { + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 8\n", + " 88.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n" }; + + checkSubstrings(renderedFields, rendered); + int nss1Offset = rendered.indexOf(renderedFields[0]) + + renderedFields[0].length(); + checkSubstrings(nss1Fields, rendered, nss1Offset); + + } + } + + public void testMap() { + String json = "[{\"key\":\"k1\",\"value\":\"v1\"},{\"key\":\"k2\",\"value\":\"v2\"}]"; + JSONString js = new JSONString(json); + String correct = "\n" + + " k1v1\n" + + " k2v2\n "; + assertEquals(correct,js.toString()); + + Inspector top = new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("key", "k1") + .put("value", "v1")) + .add(new Value.ObjectValue() + .put("key", "k2") + .put("value", "v2")); + js = new JSONString(top); + assertEquals(correct, js.renderFromInspector()); + } + + public void testWithData() { + byte[] d1 = { (byte)0x41, (byte)0x42, (byte)0x43 }; + byte[] d2 = { (byte)0x00, (byte)0x01, (byte)0x00, (byte)0x02 }; + byte[] d3 = { (byte)0x12, (byte)0x34 }; + byte[] d4 = { (byte)0xff, (byte)0x80, (byte)0x7f }; + Inspector top = new Value.ObjectValue() + .put("simple", new Value.DataValue(d1)) + .put("array", new Value.ArrayValue() + .add(new Value.DataValue(d2)) + .add(new Value.DataValue(d3)) + .add(new Value.DataValue(d4))); + JSONString js = new JSONString(top); + String correct = "\n" + + " " + + "414243" + + "\n" + + " \n" + + " " + + "00010002" + + "\n" + + " " + + "1234" + + "\n" + + " " + + "FF807F" + + "\n" + + " \n "; + assertEquals(correct, js.renderFromInspector()); + } + + public void testArrayOfArray() { + String json = "[[\"c1\", 0], [\"c2\", 2, 3], [\"c3\", 3, 4, 5], [\"c4\", 4,5,6,7]]"; + JSONString js = new JSONString(json); + Inspector outer = js.inspect(); + assertEquals(4, outer.entryCount()); + + assertEquals(2, outer.entry(0).entryCount()); + assertEquals("c1", outer.entry(0).entry(0).asString()); + assertEquals(0, outer.entry(0).entry(1).asLong()); + + assertEquals(3, outer.entry(1).entryCount()); + assertEquals("c2", outer.entry(1).entry(0).asString()); + assertEquals(2, outer.entry(1).entry(1).asLong()); + assertEquals(3, outer.entry(1).entry(2).asLong()); + + assertEquals(4, outer.entry(2).entryCount()); + assertEquals("c3", outer.entry(2).entry(0).asString()); + assertEquals(3, outer.entry(2).entry(1).asLong()); + assertEquals(4, outer.entry(2).entry(2).asLong()); + assertEquals(5, outer.entry(2).entry(3).asLong()); + + assertEquals(5, outer.entry(3).entryCount()); + assertEquals("c4", outer.entry(3).entry(0).asString()); + assertEquals(4, outer.entry(3).entry(1).asLong()); + assertEquals(5, outer.entry(3).entry(2).asLong()); + assertEquals(6, outer.entry(3).entry(3).asLong()); + assertEquals(7, outer.entry(3).entry(4).asLong()); + } + + public void testSimpleArrays() { + String json = "[1, 2, 3]"; + JSONString js = new JSONString(json); + String correct = "\n" + + " 1\n" + + " 2\n" + + " 3\n "; + assertEquals(correct, js.toString()); + + Inspector top = new Value.ArrayValue() + .add(1).add(2).add(3); + js = new JSONString(top); + assertEquals(correct, js.renderFromInspector()); + + json = "[1.0, 2.0, 3.0]"; + js = new JSONString(json); + correct = "\n" + + " 1.0\n" + + " 2.0\n" + + " 3.0\n "; + assertEquals(correct, js.toString()); + top = new Value.ArrayValue() + .add(1.0).add(2.0).add(3.0); + js = new JSONString(top); + assertEquals(correct, js.renderFromInspector()); + + json = "[\"a\", \"b\", \"c\"]"; + correct = "\n" + + " a\n" + + " b\n" + + " c\n "; + js = new JSONString(json); + assertEquals(correct, js.toString()); + + top = new Value.ArrayValue() + .add("a").add("b").add("c"); + js = new JSONString(top); + assertEquals(correct, js.renderFromInspector()); + } + + public void testArrayOfStruct() { + String json = "[{\"as1\":[\"per\",\"paal\"]," + + "\"l1\":1122334455667788994,\"d1\":74.790001," + + "\"i1\":14,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" + + "espa\u00F1a\\n" + + "asf1[0].s1\"},{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788995,\"d1\":75.790001,\"i1\":15,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" + + "espa\u00F1a asf1[1].s1\"}]"; + JSONString js = new JSONString(json); + String[] o1Fields = { + "\n \n", + " 1122334455667788994\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 14\n", + " 74.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " string\n" + "españa\n" + + "asf1[0].s1\n" }; + String separator = " \n" + " \n"; + String[] o2Fields = { + " 1122334455667788995\n", + " \n" + + " 11223344556677881\n" + + " 11223344556677883\n" + + " \n", + " 15\n", + " 75.790001\n", + " \n" + + " per\n" + + " paal\n" + + " \n", + " string\n" + + "españa asf1[1].s1\n" }; + String rendered = js.toString(); + + int o2Offset = rendered.indexOf(separator); + assertTrue(-1 < o2Offset); + + checkSubstrings(o1Fields, rendered); + checkSubstrings(o2Fields, rendered, o2Offset); + + Inspector top = new Value.ArrayValue().add( + new Value.ObjectValue() + .put("d1", 74.790001) + .put("s1", "string\n" + "espa\u00F1a\n" + "asf1[0].s1") + .put("al1", + new Value.ArrayValue().add(11223344556677881L) + .add(11223344556677883L)) + .put("l1", 1122334455667788994L) + .put("as1", + new Value.ArrayValue().add("per").add("paal")) + .put("i1", 14)).add( + new Value.ObjectValue() + .put("d1", 75.790001) + .put("s1", "string\n" + "espa\u00F1a asf1[1].s1") + .put("al1", + new Value.ArrayValue().add(11223344556677881L) + .add(11223344556677883L)) + .put("l1", 1122334455667788995L) + .put("as1", + new Value.ArrayValue().add( + new Value.StringValue("per")).add( + new Value.StringValue("paal"))) + .put("i1", 15)); + js = new JSONString(top); + + rendered = js.renderFromInspector(); + + o2Offset = rendered.indexOf(separator); + assertTrue(-1 < o2Offset); + + checkSubstrings(o1Fields, rendered.substring(0, o2Offset)); + checkSubstrings(o2Fields, rendered, o2Offset); + } + + private void checkSubstrings(String[] fields, String haystack) { + for (String field : fields) { + assertTrue(-1 < haystack.indexOf(field)); + } + } + + private void checkSubstrings(String[] fields, String haystack, int offset) { + for (String field : fields) { + assertTrue(-1 < haystack.indexOf(field, offset)); + } + } + +/*** here is some json for you + + [{"asf":"here is 1st simple string field", + "map":[{"key":"one key string","value":["one value string","embedded array"]}, + {"key":"two key string","value":["two value string","embedded array"]}], + "sf2":"here is 2nd simple string field"}, + {"asf":"here is 3rd simple string field", + "map":[{"key":"three key string","value":["three value string","embedded array"]}, + {"key":"four key string","value":["four value string","embedded array"]}], + "sf2":"here is 4th simple string field"}, + ] + +***/ + +/*** and here is some corresponding XML + + + here is 1st simple string field + + one key string + one value string + embedded array + + two key string + two value string + embedded array + + + here is 2nd simple string field + + + here is 3rd simple string field + + three key string + three value string + embedded array + + four key string + four value string + embedded array + + + here is 4th simple string field + + +***/ + + public void testArrayOfStructWithMap() { + String json = "[{\"asf\":\"here is 1st simple string field\",\"map\":[{\"key\":\"one key string\",\"value\":[\"one value string\",\"embedded array\"]},{\"key\":\"two key string\",\"value\":[\"two value string\",\"embedded array\"]}],\"sf2\":\"here is 2nd simple string field\"},{\"asf\":\"here is 3rd simple string field\",\"map\":[{\"key\":\"three key string\",\"value\":[\"three value string\",\"embedded array\"]},{\"key\":\"four key string\",\"value\":[\"four value string\",\"embedded array\"]}],\"sf2\":\"here is 4th simple string field\"}]"; + + + JSONString js = new JSONString(json); + // System.err.println("got:>>>"); + // System.err.println(js.toString()); + // System.err.println("<<<:got"); + String correct = "\n" + + " \n" + + " here is 1st simple string field\n" + + " \n" + + " one key string\n" + + " one value string\n" + + " embedded array\n" + + " \n" + + " two key string\n" + + " two value string\n" + + " embedded array\n" + + " \n" + + " \n" + + " here is 2nd simple string field\n" + + " \n" + + " \n" + + " here is 3rd simple string field\n" + + " \n" + + " three key string\n" + + " three value string\n" + + " embedded array\n" + + " \n" + + " four key string\n" + + " four value string\n" + + " embedded array\n" + + " \n" + + " \n" + + " here is 4th simple string field\n" + + " \n" + + " "; + assertEquals(correct, js.toString()); + + Inspector top = new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("asf", "here is 1st simple string field") + .put("map", new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("key", "one key string") + .put("value", new Value.ArrayValue() + .add("one value string") + .add("embedded array"))) + .add(new Value.ObjectValue() + .put("key", "two key string") + .put("value", new Value.ArrayValue() + .add("two value string") + .add("embedded array")))) + .put("sf2", "here is 2nd simple string field")) + .add(new Value.ObjectValue() + .put("asf", "here is 3rd simple string field") + .put("map", new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("key", "three key string") + .put("value", new Value.ArrayValue() + .add("three value string") + .add("embedded array"))) + .add(new Value.ObjectValue() + .put("key", "four key string") + .put("value", new Value.ArrayValue() + .add("four value string") + .add("embedded array")))) + .put("sf2", "here is 4th simple string field")); + js = new JSONString(top); +// System.err.println(">>>"+js.renderFromInspector()+"<<<"); + assertEquals(correct, js.renderFromInspector()); + } + + public void testArrayOfStructWithEmptyMap() { + String json = "[{\"asf\":\"here is 1st simple string field\",\"map\":[],\"sf2\":\"here is 2nd simple string field\"},{\"asf\":\"here is 3rd simple string field\",\"map\":[],\"sf2\":\"here is 4th simple string field\"}]"; + + + JSONString js = new JSONString(json); + // System.err.println("got:>>>"); + // System.err.println(js.toString()); + // System.err.println("<<<:got"); + String correct = "\n" + + " \n" + + " here is 1st simple string field\n" + + " \n" + + " here is 2nd simple string field\n" + + " \n" + + " \n" + + " here is 3rd simple string field\n" + + " \n" + + " here is 4th simple string field\n" + + " \n" + + " "; + assertEquals(correct, js.toString()); + + Inspector top = new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("asf", "here is 1st simple string field") + .put("map", new Value.ArrayValue()) + .put("sf2", "here is 2nd simple string field")) + .add(new Value.ObjectValue() + .put("asf", "here is 3rd simple string field") + .put("map", new Value.ArrayValue()) + .put("sf2", "here is 4th simple string field")); + js = new JSONString(top); +// System.err.println(">>>"+js.renderFromInspector()+"<<<"); + assertEquals(correct, js.renderFromInspector()); + + } + + + private Inspector getSlime1() { + Slime slime = new Slime(); + slime.setNix(); + return new SlimeAdapter(slime.get()); + } + private Inspector getSlime2() { + Slime slime = new Slime(); + slime.setString("foo"); + return new SlimeAdapter(slime.get()); + } + private Inspector getSlime3() { + Slime slime = new Slime(); + slime.setLong(123); + return new SlimeAdapter(slime.get()); + } + private Inspector getSlime4() { + Slime slime = new Slime(); + Cursor obj = slime.setObject(); + obj.setLong("foo", 1); + return new SlimeAdapter(slime.get()); + } + private Inspector getSlime5() { + Slime slime = new Slime(); + Cursor arr = slime.setArray(); + arr.addLong(1); + arr.addLong(2); + arr.addLong(3); + return new SlimeAdapter(slime.get()); + } + + public void testInspectorToContentMapping() { + String content1 = new JSONString(getSlime1()).getContent(); + String content2 = new JSONString(getSlime2()).getContent(); + String content3 = new JSONString(getSlime3()).getContent(); + String content4 = new JSONString(getSlime4()).getContent(); + String content5 = new JSONString(getSlime5()).getContent(); + assertEquals("", content1); + assertEquals("foo", content2); + assertEquals("123", content3); + assertEquals("{\"foo\":1}", content4); + assertEquals("[1,2,3]", content5); + } + + public void testContentToInspectorMapping() { + Inspector value1 = new JSONString("").inspect(); + Inspector value2 = new JSONString("foo").inspect(); + Inspector value3 = new JSONString("\"foo\"").inspect(); + Inspector value4 = new JSONString("123").inspect(); + Inspector value5 = new JSONString("{\"foo\":1}").inspect(); + Inspector value6 = new JSONString("[1,2,3]").inspect(); + + System.out.println("1: " + value1); + System.out.println("2: " + value2); + System.out.println("3: " + value3); + System.out.println("4: " + value4); + System.out.println("5: " + value5); + System.out.println("6: " + value6); + + assertEquals(Type.STRING, value1.type()); + assertEquals("", value1.asString()); + + assertEquals(value2.type(), Type.STRING); + assertEquals("foo", value2.asString()); + + assertEquals(value3.type(), Type.STRING); + assertEquals("\"foo\"", value3.asString()); + + assertEquals(value4.type(), Type.STRING); + assertEquals("123", value4.asString()); + + assertEquals(value5.type(), Type.OBJECT); + assertEquals(1L, value5.field("foo").asLong()); + assertEquals("{\"foo\":1}", value5.toString()); + + assertEquals(value6.type(), Type.ARRAY); + assertEquals(1L, value6.entry(0).asLong()); + assertEquals(2L, value6.entry(1).asLong()); + assertEquals(3L, value6.entry(2).asLong()); + assertEquals("[1,2,3]", value6.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/TokenFieldIteratorTestCase.java b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/TokenFieldIteratorTestCase.java new file mode 100644 index 00000000000..d61d0edcf2e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/TokenFieldIteratorTestCase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.hitfield.test; + +import java.util.ListIterator; + +import com.yahoo.prelude.hitfield.FieldPart; +import com.yahoo.prelude.hitfield.HitField; +import com.yahoo.prelude.hitfield.StringFieldPart; + + +/** + * Tests the FieldTokenIterator class + * + * @author Steinar Knutsen + */ +public class TokenFieldIteratorTestCase extends junit.framework.TestCase { + + public TokenFieldIteratorTestCase (String name) { + super(name); + } + + public void testTokenIteratorNext() { + HitField hf = new HitField("boo", "hei paa deg"); + assertEquals(3, hf.getTokenizedContent().size()); + ListIterator l = hf.tokenIterator(); + FieldPart p = (FieldPart)l.next(); + assertEquals("hei", p.getContent()); + p = (FieldPart)l.next(); + assertEquals("paa", p.getContent()); + p = (FieldPart)l.next(); + assertEquals("deg", p.getContent()); + assertEquals(false, l.hasNext()); + } + public void testTokenIteratorPrevious() { + HitField hf = new HitField("boo", "hei paa"); + ListIterator l = hf.tokenIterator(); + FieldPart p = (FieldPart)l.next(); + assertEquals("hei", p.getContent()); + p = (FieldPart)l.next(); + assertEquals("paa", p.getContent()); + p = (FieldPart)l.previous(); + assertEquals("paa", p.getContent()); + p = (FieldPart)l.previous(); + assertEquals("hei", p.getContent()); + } + public void testTokenIteratorSet() { + HitField hf = new HitField("boo", "hei paa deg"); + assertEquals(3, hf.getTokenizedContent().size()); + ListIterator l = hf.tokenIterator(); + l.next(); + l.next(); + l.set(new StringFieldPart("aap", true)); + l.next(); + assertEquals(false, l.hasNext()); + l.previous(); + l.set(new StringFieldPart("ged", true)); + assertEquals("hei aap ged", hf.getContent()); + } + public void testTokenIteratorAdd() { + HitField hf = new HitField("boo", "hei paa deg"); + assertEquals(3, hf.getTokenizedContent().size()); + ListIterator l = hf.tokenIterator(); + l.add(new StringFieldPart("a", true)); + l.next(); + l.next(); + l.add(new StringFieldPart("b", true)); + l.next(); + l.add(new StringFieldPart("c", true)); + assertEquals(false, l.hasNext()); + assertEquals("ahei paab degc", hf.getContent()); + } + public void testTokenIteratorRemove() { + HitField hf = new HitField("boo", "hei paa deg"); + ListIterator l = hf.tokenIterator(); + l.next(); + l.next(); + l.remove(); + l.add(new StringFieldPart("hallo", true)); + assertEquals(3, hf.getTokenizedContent().size()); + assertEquals("hei hallo deg", hf.getContent()); + l.next(); + l.previous(); + l.previous(); + l.remove(); + assertEquals("hei deg", hf.getContent()); + l.add(new StringFieldPart("paa", true)); + assertEquals("hei paa deg", hf.getContent()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java new file mode 100644 index 00000000000..35adc50a556 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemHelperTestCase.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import static org.junit.Assert.*; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import com.yahoo.search.Query; + + +/** + * Unit test for the helper methods placed in + * com.yahoo.prelude.query.ItemHelper. + * + * @author Steinar Knutsen + */ +public class ItemHelperTestCase { + + @Test + public final void testGetNumTerms() { + ItemHelper helper = new ItemHelper(); + Query q = new Query("/?query=" + enc("a b c")); + assertEquals(3, helper.getNumTerms(q.getModel().getQueryTree().getRoot())); + } + + @Test + public final void testGetPositiveTerms() { + ItemHelper helper = new ItemHelper(); + Query q = new Query("/?query=" + enc("a b c \"d e\" -f")); + List l = new ArrayList<>(); + System.out.println(q.getModel()); + helper.getPositiveTerms(q.getModel().getQueryTree().getRoot(), l); + assertEquals(4, l.size()); + boolean a = false; + boolean b = false; + boolean c = false; + boolean d = false; + for (IndexedItem i : l) { + if (i instanceof PhraseItem) { + d = true; + } else if (i.getIndexedString().equals("a")) { + a = true; + } else if (i.getIndexedString().equals("b")) { + b = true; + } else if (i.getIndexedString().equals("c")) { + c = true; + } + } + assertFalse("An item is missing.", (a & b & c & d) == false); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemLabelTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemLabelTestCase.java new file mode 100644 index 00000000000..62ef9c28ce6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemLabelTestCase.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import java.lang.reflect.Modifier; + +import static org.junit.Assert.*; +import org.junit.Test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.query.textualrepresentation.Discloser; + +public class ItemLabelTestCase { + + private static final class LabelCatcher implements Discloser { + public String label = null; + public void addProperty(String key, Object value) { + if (key.equals("label")) { + if (value == null) { + label = "null"; + } else { + label = (String) value; + } + } + } + public void setValue(Object value) {} + public void addChild(Item item) {} + } + + @Test + public final void testLabelVisibility() throws Exception { + assertTrue(Modifier.isPublic(Item.class.getMethod("setLabel", String.class).getModifiers())); + assertTrue(Modifier.isPublic(Item.class.getMethod("getLabel").getModifiers())); + } + + @Test + public final void testLabelAccess() { + Item item = new WordItem("word"); + assertFalse(item.hasUniqueID()); + assertNull(item.getLabel()); + item.setLabel("my_label"); + assertTrue(item.hasUniqueID()); + assertEquals("my_label", item.getLabel()); + } + + @Test + public final void testLabelDisclose() { + LabelCatcher catcher = new LabelCatcher(); + Item item = new WordItem("word"); + item.disclose(catcher); + assertNull(catcher.label); + item.setLabel("my_other_label"); + item.disclose(catcher); + assertEquals("my_other_label", item.getLabel()); + } + + @Test + public final void testLabelEncode() { + Item w1 = new WordItem("w1"); + Item w2 = new WordItem("w2"); + Item w3 = new WordItem("w3"); + AndItem and = new AndItem(); + Query query = new Query(); + + w1.setLabel("bar"); + w3.setLabel("foo"); + and.addItem(w1); + and.addItem(w2); + and.addItem(w3); + and.setLabel("missing"); + query.getModel().getQueryTree().setRoot(and); + query.prepare(); + assertEquals("3", query.getRanking().getProperties().get("vespa.label.foo.id").get(0)); + assertEquals("1", query.getRanking().getProperties().get("vespa.label.bar.id").get(0)); + + // Conceptually, any node can have a label. However, only + // taggable nodes are allowed to have a unique id. Taggable + // nodes act as leaf nodes, but labels should be possible for + // any combination of nodes in the query tree. Thus, generic + // labeling is appropriate, but only those items that are also + // taggable will propagate their labels to the bank-end. We + // can live with this weakness for now, as the nodes we + // typically need to label in the back-end are leaf-ish nodes. + assertNull(query.getRanking().getProperties().get("vespa.label.missing.id")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java new file mode 100644 index 00000000000..c1b18a8ae31 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java @@ -0,0 +1,398 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.ListIterator; +import java.util.NoSuchElementException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.prelude.query.Item.ItemType; + +/** + * Check basic contracts common to "many" item implementations. + * + * @author Steinar Knutsen + */ +public class ItemsCommonStuffTestCase { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testLoops() { + AndSegmentItem as = new AndSegmentItem("farmyards", false, false); + boolean caught = false; + try { + as.addItem(as); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + AndItem a = new AndItem(); + caught = false; + try { + a.addItem(a); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + caught = false; + a.addItem(as); + try { + as.addItem(a); + } catch (QueryException e) { + caught = true; + } + assertTrue(caught); + caught = false; + a.removeItem(as); + as.addItem(a); + try { + a.addItem(as); + } catch (QueryException e) { + caught = true; + } + assertTrue(caught); + } + + @Test + public final void testIndexName() { + WordItem w = new WordItem("nalle"); + AndItem a = new AndItem(); + a.addItem(w); + final String expected = "mobil"; + a.setIndexName(expected); + assertEquals(expected, w.getIndexName()); + } + + @Test + public final void testBoundaries() { + WordItem w = new WordItem("nalle"); + AndItem a = new AndItem(); + boolean caught = false; + try { + a.addItem(-1, w); + } catch (IndexOutOfBoundsException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + a.addItem(1, w); + } catch (IndexOutOfBoundsException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + a.setItem(-1, w); + } catch (IndexOutOfBoundsException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + a.setItem(0, w); + } catch (IndexOutOfBoundsException e) { + caught = true; + } + assertTrue(caught); + } + + @Test + public final void testRemoving() { + AndItem other = new AndItem(); + WordItem w = new WordItem("nalle"); + AndItem a = new AndItem(); + WordItem v = new WordItem("bamse"); + v.setParent(other); + a.addItem(w); + assertFalse(a.removeItem(null)); + assertTrue(a.removeItem(w)); + assertNull(w.getParent()); + a.removeItem(v); + assertSame(other, v.getParent()); + } + + @Test + public final void testGeneralMutability() { + AndItem a = new AndItem(); + assertFalse(a.isLocked()); + a.lock(); + assertFalse(a.isLocked()); + } + + @Test + public final void testCounting() { + WordItem w = new WordItem("nalle"); + AndItem a = new AndItem(); + WordItem v = new WordItem("bamse"); + AndItem other = new AndItem(); + assertEquals(0, a.getTermCount()); + a.addItem(w); + assertEquals(1, a.getTermCount()); + other.addItem(v); + a.addItem(other); + assertEquals(2, a.getTermCount()); + } + + @Test + public final void testIteratorJuggling() { + AndItem a = new AndItem(); + WordItem w0 = new WordItem("nalle"); + WordItem w1 = new WordItem("bamse"); + WordItem w2 = new WordItem("teddy"); + boolean caught = false; + a.addItem(w0); + a.addItem(w1); + ListIterator i = a.getItemIterator(); + assertFalse(i.hasPrevious()); + try { + i.previous(); + } catch (NoSuchElementException e) { + caught = true; + } + assertTrue(caught); + assertEquals(-1, i.previousIndex()); + assertEquals(0, i.nextIndex()); + i.next(); + WordItem wn = (WordItem) i.next(); + assertSame(w1, wn); + assertSame(w1, i.previous()); + assertSame(w0, i.previous()); + assertEquals(0, i.nextIndex()); + i.add(w2); + assertEquals(1, i.nextIndex()); + } + + @Test + public final void testIdStuff() { + Item i; + final String expected = "i"; + i = new ExactstringItem(expected); + assertEquals(ItemType.EXACT, i.getItemType()); + assertEquals("EXACTSTRING", i.getName()); + assertEquals(expected, ((ExactstringItem) i).stringValue()); + i = new PrefixItem("p"); + assertEquals(ItemType.PREFIX, i.getItemType()); + assertEquals("PREFIX", i.getName()); + i = new SubstringItem("p"); + assertEquals(ItemType.SUBSTRING, i.getItemType()); + assertEquals("SUBSTRING", i.getName()); + i = new SuffixItem("p"); + assertEquals(ItemType.SUFFIX, i.getItemType()); + assertEquals("SUFFIX", i.getName()); + i = new WeightedSetItem("nalle"); + assertEquals(ItemType.WEIGHTEDSET, i.getItemType()); + assertEquals("WEIGHTEDSET", i.getName()); + i = new AndSegmentItem("",false, false); + assertEquals(ItemType.AND, i.getItemType()); + assertEquals("SAND", i.getName()); + i = new WeakAndItem(); + assertEquals(ItemType.WEAK_AND, i.getItemType()); + assertEquals("WAND", i.getName()); + } + + @Test + public final void testEquivBuilding() { + WordItem w = new WordItem("nalle"); + WordItem v = new WordItem("bamse"); + w.setConnectivity(v, 1.0); + EquivItem e = new EquivItem(w); + assertEquals(1.0, e.getConnectivity(), 1e-9); + assertSame(v, e.getConnectedItem()); + } + + @Test + public final void testEquivBuildingFromCollection() { + WordItem w = new WordItem("nalle"); + WordItem v = new WordItem("bamse"); + w.setConnectivity(v, 1.0); + final String expected = "puppy"; + final String expected2 = "kvalp"; + EquivItem e = new EquivItem(w, Arrays.asList(new String[] { expected, expected2 })); + assertEquals(1.0, e.getConnectivity(), 1e-9); + assertSame(v, e.getConnectedItem()); + assertEquals(expected, ((WordItem) e.getItem(1)).getWord()); + assertEquals(expected2, ((WordItem) e.getItem(2)).getWord()); + } + + @Test + public final void testSegment() { + AndSegmentItem as = new AndSegmentItem("farmyards", false, false); + assertFalse(as.isLocked()); + final WordItem firstItem = new WordItem("nalle"); + as.addItem(firstItem); + final WordItem item = new WordItem("bamse"); + as.addItem(1, item); + assertTrue(as.removeItem(item)); + assertFalse(as.isFromUser()); + as.setFromUser(true); + assertTrue(as.isFromUser()); + as.lock(); + boolean caught = false; + try { + as.removeItem(firstItem); + } catch (QueryException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + as.addItem(new WordItem("puppy")); + } catch (QueryException e) { + caught= true; + } + assertTrue(caught); + caught = false; + try { + as.addItem(1, new WordItem("kvalp")); + } catch (QueryException e) { + caught = true; + } + assertTrue(caught); + } + + @Test + public final void testMarkersVsWords() { + WordItem mw0 = MarkerWordItem.createEndOfHost(); + WordItem mw1 = MarkerWordItem.createStartOfHost(); + WordItem w0 = new WordItem("$"); + WordItem w1 = new WordItem("^"); + assertEquals(w0.getWord(), mw0.getWord()); + assertEquals(w1.getWord(), mw1.getWord()); + assertFalse(mw0.equals(w0)); + assertTrue(mw0.equals(MarkerWordItem.createEndOfHost())); + assertFalse(w1.hashCode() == mw1.hashCode()); + } + + @Test + public final void testNumberBasics() { + final String expected = "12"; + IntItem i = new IntItem(expected, "num"); + assertEquals(expected, i.stringValue()); + final String expected2 = "34"; + i.setNumber(expected2); + assertEquals(expected2, i.stringValue()); + String expected3 = "56"; + i.setValue(expected3); + assertEquals(expected3, i.stringValue()); + assertTrue(i.isStemmed()); + assertFalse(i.isWords()); + assertEquals(1, i.getNumWords()); + assertFalse(i.equals(new IntItem(expected3))); + assertTrue(i.equals(new IntItem(expected3, "num"))); + } + + @Test + public final void testNullItemFailsProperly() { + NullItem n = new NullItem(); + n.setIndexName("nalle"); + boolean caught = false; + try { + n.encode(ByteBuffer.allocate(100)); + } catch (RuntimeException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + n.getItemType(); + } catch (RuntimeException e) { + caught = true; + } + assertTrue(caught); + assertEquals(0, n.getTermCount()); + } + + private void fill(CompositeItem c) { + for (String w : new String[] { "nalle", "bamse", "teddy" }) { + c.addItem(new WordItem(w)); + } + } + + @Test + public final void testNearisNotAnd() { + AndItem a = new AndItem(); + NearItem n = new NearItem(); + n.setDistance(2); + NearItem n2 = new NearItem(); + n2.setDistance(2); + NearItem n3 = new NearItem(); + n3.setDistance(3); + fill(a); + fill(n); + fill(n2); + fill(n3); + assertFalse(a.hashCode() == n.hashCode()); + assertFalse(n.equals(a)); + assertTrue(n.equals(n2)); + assertFalse(n.equals(n3)); + } + + @Test + public final void testPhraseSegmentBasics() { + AndSegmentItem a = new AndSegmentItem("gnurk", "gurk", false, false); + fill(a); + a.lock(); + PhraseSegmentItem p = new PhraseSegmentItem(a); + assertEquals("SPHRASE", p.getName()); + p.addItem(new WordItem("blbl")); + boolean caught = false; + try { + p.addItem(new AndItem()); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + assertEquals("blbl", p.getWordItem(3).getWord()); + ByteBuffer b = ByteBuffer.allocate(5000); + int i = p.encode(b); + assertEquals(5, i); + assertEquals("nalle bamse teddy blbl", p.getIndexedString()); + } + + @Test + public final void testPhraseConnectivity() { + WordItem w = new WordItem("a"); + PhraseItem p = new PhraseItem(); + fill(p); + p.setConnectivity(w, 500.0d); + assertEquals(500.0d, p.getConnectivity(), 1e-9); + assertSame(w, p.getConnectedItem()); + } + + @Test + public final void testBaseClassPhraseSegments() { + PhraseSegmentItem p = new PhraseSegmentItem("g", false, true); + fill(p); + assertEquals(4, p.encode(ByteBuffer.allocate(5000))); + p.setIndexName(null); + assertEquals("", p.getIndexName()); + PhraseSegmentItem p2 = new PhraseSegmentItem("g", false, true); + fill(p2); + } + + @Test + public final void testTermTypeBasic() { + assertFalse(TermType.AND.equals(TermType.DEFAULT)); + assertFalse(TermType.AND.equals(new Integer(10))); + assertTrue(TermType.AND.equals(TermType.AND)); + assertSame(AndItem.class, TermType.DEFAULT.createItemClass().getClass()); + assertSame(CompositeItem.class, TermType.DEFAULT.getItemClass()); + assertFalse(TermType.AND.hashCode() == TermType.PHRASE.hashCode()); + assertEquals("term type 'not'", TermType.NOT.toString()); + } +} + diff --git a/container-search/src/test/java/com/yahoo/prelude/query/TaggableItemsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/TaggableItemsTestCase.java new file mode 100644 index 00000000000..aabcc0e54aa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/TaggableItemsTestCase.java @@ -0,0 +1,158 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Method; +import java.util.HashSet; +import java.util.Set; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Keep CompositeTaggableItem, SimpleTaggableItem and TaggableSegmentItem in + * lockstep. + * + * @author Steinar Knutsen + */ +public class TaggableItemsTestCase { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + private static class ApiMethod { + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final ApiMethod other = (ApiMethod) obj; + if (!name.equals(other.name)) { + return false; + } + if (parameterTypes.length != other.parameterTypes.length) { + return false; + } + for (int i = 0; i < parameterTypes.length; ++i) { + if (parameterTypes[i] != other.parameterTypes[i]) { + return false; + } + } + if (returnType != other.returnType) { + return false; + } + return true; + } + + public ApiMethod(final Method method) { + if (method == null) { + throw new IllegalArgumentException(); + } + name = method.getName(); + returnType = method.getReturnType(); + parameterTypes = method.getParameterTypes(); + } + + @Override + public String toString() { + final StringBuilder s = new StringBuilder(); + s.append(returnType.getSimpleName()).append(' ').append(name) + .append('('); + final int initLen = s.length(); + for (final Class c : parameterTypes) { + if (s.length() != initLen) { + s.append(", "); + } + s.append(c.getSimpleName()); + } + s.append(')'); + return s.toString(); + } + + private final String name; + private final Class returnType; + private final Class[] parameterTypes; + + } + + @Test + public void requireSimilarAPIs() { + final Method[] composite = CompositeTaggableItem.class + .getDeclaredMethods(); + final Method[] simple = SimpleTaggableItem.class.getDeclaredMethods(); + final Method[] segment = TaggableSegmentItem.class.getDeclaredMethods(); + final int numberOfMethods = 10; + assertEquals(numberOfMethods, composite.length); + assertEquals(numberOfMethods, simple.length); + assertEquals(numberOfMethods, segment.length); + final Set compositeSet = methodSet(composite); + final Set simpleSet = methodSet(simple); + final Set segmentSet = methodSet(segment); + assertEquals(compositeSet, simpleSet); + assertEquals(simpleSet, segmentSet); + + } + + public Set methodSet(final Method[] methods) { + final Set methodSet = new HashSet<>(); + for (final Method m : methods) { + methodSet.add(new ApiMethod(m)); + } + return methodSet; + } + + @Test + public final void testSetUniqueID() { + final PhraseSegmentItem p = new PhraseSegmentItem("farmyards", false, + false); + assertFalse(p.hasUniqueID()); + p.setUniqueID(10); + assertEquals(10, p.getUniqueID()); + assertTrue(p.hasUniqueID()); + } + + @Test + public final void testSetConnectivity() { + final PhraseSegmentItem p = new PhraseSegmentItem("farmyards", false, + false); + assertEquals(0.0d, p.getConnectivity(), 1e-9); + final WordItem w = new WordItem("nalle"); + final double expectedConnectivity = 37e9; + p.setConnectivity(w, expectedConnectivity); + assertSame(w, p.getConnectedItem()); + assertEquals(expectedConnectivity, p.getConnectivity(), 1e0); + } + + @Test + public final void testSetSignificance() { + final PhraseSegmentItem p = new PhraseSegmentItem("farmyards", false, + false); + // unset + assertEquals(0.0d, p.getSignificance(), 1e-9); + assertFalse(p.hasExplicitSignificance()); + p.setSignificance(500.0d); + assertTrue(p.hasExplicitSignificance()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/WordAlternativesItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/WordAlternativesItemTestCase.java new file mode 100644 index 00000000000..63f8af96bd0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/WordAlternativesItemTestCase.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import com.yahoo.prelude.query.WordAlternativesItem.Alternative; + +/** + * Functional test for the contracts in WordAlternativesItem. + * + * @author steinar + */ +public class WordAlternativesItemTestCase { + + @Test + public final void testWordAlternativesItem() { + List terms = new ArrayList<>(); + List expected; + terms.add(new Alternative("1", 1.0)); + terms.add(new Alternative("2", 1.0)); + terms.add(new Alternative("3", 1.0)); + terms.add(new Alternative("4", 1.0)); + expected = new ArrayList<>(terms); + terms.add(new Alternative("1", .1)); + terms.add(new Alternative("2", .2)); + terms.add(new Alternative("3", .3)); + terms.add(new Alternative("4", .4)); + WordAlternativesItem w = new WordAlternativesItem("", true, null, terms); + assertEquals(expected, w.getAlternatives()); + } + + @Test + public final void testSetAlternatives() { + List terms = new ArrayList<>(); + terms.add(new Alternative("1", 1.0)); + terms.add(new Alternative("2", 1.0)); + WordAlternativesItem w = new WordAlternativesItem("", true, null, terms); + terms.add(new Alternative("1", 1.5)); + terms.add(new Alternative("2", 0.5)); + w.setAlternatives(terms); + assertTrue("Could not overwrite alternative", + w.getAlternatives().stream().anyMatch((a) -> a.word.equals("1") && a.exactness == 1.5)); + assertTrue("Old alternative unexpectedly removed", + w.getAlternatives().stream().anyMatch((a) -> a.word.equals("2") && a.exactness == 1.0)); + assertEquals(2, w.getAlternatives().size()); + terms.add(new Alternative("3", 0.5)); + w.setAlternatives(terms); + assertTrue("Could not add new term", + w.getAlternatives().stream().anyMatch((a) -> a.word.equals("3") && a.exactness == 0.5)); + } + + @Test + public final void testAddTerm() { + List terms = new ArrayList<>(); + terms.add(new Alternative("1", 1.0)); + terms.add(new Alternative("2", 1.0)); + WordAlternativesItem w = new WordAlternativesItem("", true, null, terms); + w.addTerm("1", 0.1); + assertEquals(terms, w.getAlternatives()); + w.addTerm("1", 2.0); + assertTrue("Could not add new alternative", + w.getAlternatives().stream().anyMatch((a) -> a.word.equals("1") && a.exactness == 2.0)); + assertEquals(2, w.getAlternatives().size()); + w.addTerm("3", 0.5); + assertTrue("Could not add new term", + w.getAlternatives().stream().anyMatch((a) -> a.word.equals("3") && a.exactness == 0.5)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java new file mode 100644 index 00000000000..139a8cb1a2e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; +import com.yahoo.language.Linguistics; +import com.yahoo.language.detect.Detector; +import com.yahoo.language.process.*; +import com.yahoo.language.simple.SimpleLinguistics; + +/** + * @author Simon Thoresen + */ +public class TestLinguistics implements Linguistics { + + public static final Linguistics INSTANCE = new TestLinguistics(); + private final Linguistics linguistics = new SimpleLinguistics(); + + private TestLinguistics() { + // hide + } + + @Override + public Stemmer getStemmer() { + return linguistics.getStemmer(); + } + + @Override + public com.yahoo.language.process.Tokenizer getTokenizer() { + return linguistics.getTokenizer(); + } + + @Override + public Normalizer getNormalizer() { + return linguistics.getNormalizer(); + } + + @Override + public Transformer getTransformer() { + return linguistics.getTransformer(); + } + + @Override + public Segmenter getSegmenter() { + return new TestSegmenter(); + } + + @Override + public Detector getDetector() { + return linguistics.getDetector(); + } + + @Override + public GramSplitter getGramSplitter() { + return linguistics.getGramSplitter(); + } + + @Override + public CharacterClasses getCharacterClasses() { + return linguistics.getCharacterClasses(); + } + + @Override + public Tuple2 getVersion(Linguistics.Component component) { + return linguistics.getVersion(component); + } + +} + + diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/TestSegmenter.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestSegmenter.java new file mode 100644 index 00000000000..6906c48f2ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestSegmenter.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Segmenter; + +import java.util.List; + +/** + * @author bratseth + */ +public class TestSegmenter implements Segmenter { + + /** + *

Splits "cd" and "fg" and every other single letter into separate tokens.

+ *

+ *

Special case for testing overlapping tokens: + * Any occurence of the string "bcd" will not split into the tokens + * "bc" and "d", but will instead split into "bc" and "cd".

+ */ + @Override + public List segment(String string, Language language) { + List tokens = new java.util.ArrayList<>(); + + // Tokenize + for (int i = 0; i < string.length(); i++) { + String token = startsByTestToken(string, i); + if (token != null) { + tokens.add(token); + i = i + token.length() - 1; + } else { + tokens.add(string.substring(i, i + 1)); + } + } + + // Special case + for (int i = 0; i < tokens.size(); i++) { + String token = tokens.get(i); + if (token.equals("bc") && tokens.size() > i + 1 && tokens.get(i + 1).equals("d")) { + tokens.set(i + 1, "cd"); + } + } + + return tokens; + } + + private static final String[] testTokens = new String[] { "bc", "fg", "first", "second", "third" }; + + private static String startsByTestToken(String string, int index) { + for (String testToken : testTokens) { + if (string.startsWith(testToken, index)) { + return testToken; + } + } + return null; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/UnicodePropertyDumpTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/UnicodePropertyDumpTestCase.java new file mode 100644 index 00000000000..e1b705616ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/UnicodePropertyDumpTestCase.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser; + +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +import com.yahoo.java7compat.Util; +import org.junit.Test; + +/** + * Test UnicodePropertyDump gives expected data. + * + * @author Steinar Knutsen + */ +public class UnicodePropertyDumpTestCase { + + @Test + public final void testMain() throws IOException { + ByteArrayOutputStream toCheck; + PrintStream out; + toCheck = new ByteArrayOutputStream(); + out = new PrintStream(toCheck, false, "UTF-8"); + // 002E;FULL STOP;Po;0;CS;;;;;N;PERIOD;;;; + UnicodePropertyDump.dumpProperties(0x2E, 0x2E + 1, true, out); + // 00C5;LATIN CAPITAL LETTER A WITH RING ABOVE;Lu;0;L;0041 030A;;;;N;LATIN CAPITAL LETTER A RING;;;00E5; + UnicodePropertyDump.dumpProperties(0xC5, 0xC5 + 1, true, out); + // 1D7D3;MATHEMATICAL BOLD DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; + UnicodePropertyDump.dumpProperties(0x1D7D3, 0x1D7D3 + 1, true, out); + out.flush(); + toCheck.flush(); + final String result = toCheck.toString("UTF-8"); + + String expected; + + if (Util.isJava7Compatible()) { + expected = "0000002e 0000 24\n000000c5 0002 1\n0001d7d3 0006 5\n"; + } else { + expected = "0000002e 0000 24\n000000c5 0002 1\n0001d7d3 0010 0\n"; + } + + assertEquals(expected, result); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java new file mode 100644 index 00000000000..9b216331551 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser.test; + + +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +/** + * Check default index propagates correctly to the tokenizer. + * + * @author Steinar Knutsen + */ +public class ExactMatchAndDefaultIndexTestCase extends junit.framework.TestCase { + + public ExactMatchAndDefaultIndexTestCase(String name) { + super(name); + } + + public void testExactMatchTokenization() { + Index index = new Index("testexact"); + index.setExact(true, null); + IndexFacts facts = new IndexFacts(); + facts.addIndex("testsd", index); + Query q = new Query("?query=" + enc("a/b foo.com") + "&default-index=testexact"); + q.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, null))); + assertEquals("AND testexact:a/b testexact:foo.com", q.getModel().getQueryTree().getRoot().toString()); + q = new Query("?query=" + enc("a/b foo.com")); + assertEquals("AND \"a b\" \"foo com\"", q.getModel().getQueryTree().getRoot().toString()); + } + + // From Flickr, which had problems with this as they didn't use a default-index + // (query is dog & cat) + public void testDefaultIndexSpecialChars() { + Query q = new Query("?query=" + enc("dog & cat") + "&default-index=textsearch"); + assertEquals("AND textsearch:dog textsearch:cat", q.getModel().getQueryTree().getRoot().toString()); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + +} + + diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java new file mode 100644 index 00000000000..be9a6b50ff2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -0,0 +1,2507 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser.test; + +import com.yahoo.language.Language; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.prelude.query.parser.TestLinguistics; +import com.yahoo.search.Query; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests query parsing. + * + * @author bratseth + */ +public class ParseTestCase { + + private ParsingTester tester = new ParsingTester(); + + @Test + public void testSimpleTermQuery() { + tester.assertParsed("foobar", "foobar", Query.Type.ANY); + } + + @Test + public void testTermWithIndexPrefix() { + tester.assertParsed("url:foobar", "url:foobar", Query.Type.ANY); + } + + @Test + public void testTermWithCatalogAndIndexPrefix() { + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ANY); + } + + @Test + public void testMultipleTermsWithUTF8EncodingOred() { + tester.assertParsed("OR l\u00e5gen delta M\u00dcNICH M\u00fcnchen", + "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", Query.Type.ANY); + } + + @Test + public void testMultipleTermsWithMultiplePrefixes() { + tester.assertParsed("RANK (+bar -normal.title:foo -baz) url:foobar", + "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); + } + + @Test + public void testSimpleQueryDefaultOr() { + tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", + Query.Type.ANY); + } + + @Test + public void testOrAndNot() { + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); + } + + @Test + public void testSimpleOrNestedAnd() { + tester.assertParsed("RANK (OR foo bar baz) foobar xyzzy", + "foobar +(foo bar baz) xyzzy", Query.Type.ANY); + } + + @Test + public void testSimpleOrNestedNot() { + tester.assertParsed("+(OR foobar xyzzy) -(AND foo bar baz)", + "foobar -(foo bar baz) xyzzy", Query.Type.ANY); + } + + @Test + public void testOrNotNestedAnd() { + tester.assertParsed( + "RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", + Query.Type.ANY); + } + + @Test + public void testOrAndNotNestedNot() { + tester.assertParsed( + "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", + Query.Type.ANY); + } + + @Test + public void testOrMultipleNestedAnd() { + tester.assertParsed( + "RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", + "foobar +(fo ba foba) foo bar +(foz baraz) baz", Query.Type.ANY); + } + + @Test + public void testOrMultipleNestedNot() { + tester.assertParsed( + "+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", + "foobar -(fo ba foba) foo bar -(foz baraz) baz", Query.Type.ANY); + } + + @Test + public void testOrAndNotMultipleNestedAnd() { + tester.assertParsed( + "RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", + Query.Type.ANY); + } + + @Test + public void testOrAndNotMultipleNestedNot() { + tester.assertParsed( + "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", + Query.Type.ANY); + } + + @Test + public void testOrMultipleNestedAndNot() { + tester.assertParsed( + "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", + "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", + Query.Type.ANY); + } + + @Test + public void testOrAndNotMultipleNestedAndNot() { + tester.assertParsed( + "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", + "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", + Query.Type.ANY); + } + + @Test + public void testExplicitPhrase() { + Item root=tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); + assertTrue(root instanceof PhraseItem); + assertTrue(((PhraseItem)root).isExplicit()); + } + + @Test + public void testPhraseWithIndex() { + tester.assertParsed("normal.title:\"foo bar foobar\"", + "normal.title:\"foo bar foobar\"", Query.Type.ANY); + } + + @Test + public void testPhrasesAndTerms() { + tester.assertParsed("OR \"foo bar foobar\" xyzzy \"baz gaz faz\"", + "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); + } + + @Test + public void testPhrasesAndTermsWithOperators() { + tester.assertParsed( + "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", + "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", + Query.Type.ANY); + } + + @Test + public void testSimpleTermQueryDefaultAnd() { + tester.assertParsed("foobar", "foobar", Query.Type.ALL); + } + + @Test + public void testTermWithCatalogAndIndexPrefixDefaultAnd() { + tester.assertParsed("normal.title:foobar", "normal.title:foobar", + Query.Type.ALL); + } + + @Test + public void testMultipleTermsWithMultiplePrefixesDefaultAnd() { + tester.assertParsed("+(AND url:foobar bar) -normal.title:foo -baz", + "url:foobar +bar -normal.title:foo -baz", Query.Type.ALL); + } + + @Test + public void testSimpleQueryDefaultAnd() { + tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", + Query.Type.ALL); + } + + @Test + public void testNotDefaultAnd() { + tester.assertParsed( + "+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", + "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", Query.Type.ALL); + } + + @Test + public void testSimpleTermQueryDefaultPhrase() { + tester.assertParsed("foobar", "foobar", Query.Type.PHRASE); + } + + @Test + public void testSimpleQueryDefaultPhrase() { + Item root=tester.assertParsed("\"foobar foo bar baz\"", "foobar foo bar baz", + Query.Type.PHRASE); + assertTrue(root instanceof PhraseItem); + assertFalse(((PhraseItem)root).isExplicit()); + } + + @Test + public void testMultipleTermsWithMultiplePrefixesDefaultPhrase() { + tester.assertParsed("\"url foobar bar normal title foo baz\"", + "url:foobar +bar -normal.title:foo -baz", Query.Type.PHRASE); + } + + @Test + public void testOdd1() { + tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.ALL); + } + + @Test + public void testOdd2() { + tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",Query.Type.ALL); + } + + @Test + public void testOdd2Uppercase() { + tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", + Query.Type.ALL); + } + + @Test + public void testOdd3() { + tester.assertParsed("AND foo (OR size.all:[200;300] date.all:512)", + "foo +(size.all:[200;300] date.all:512)", Query.Type.ALL); + } + + @Test + public void testNullQuery() { + tester.assertParsed(null, null, Query.Type.ALL); + } + + @Test + public void testEmptyQuery() { + tester.assertParsed(null, "", Query.Type.ALL); + } + + @Test + public void testNotOnly() { + tester.assertParsed(null, "-foobar", Query.Type.ALL); + } + + @Test + public void testMultipleNotsOnlt() { + tester.assertParsed(null, "-foo -bar -foobar", Query.Type.ALL); + } + + @Test + public void testOnlyNotComposite() { + tester.assertParsed(null, "-(foo bar baz)", Query.Type.ALL); + } + + @Test + public void testNestedCompositesDefaultOr() { + tester.assertParsed("RANK (OR foobar bar baz) foo xyzzy", + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); + } + + @Test + public void testNestedCompositesDefaultAnd() { + tester.assertParsed("AND foo (OR foobar bar baz) xyzzy", + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); + } + + @Test + public void testNestedCompositesPhraseDefault() { + tester.assertParsed("\"foo foobar bar baz xyzzy\"", + "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); + } + + @Test + public void testNumeric() { + tester.assertParsed("34", "34", Query.Type.ANY); + } + + @Test + public void testGreaterNumeric() { + tester.assertParsed("<454", "<454", Query.Type.ANY); + } + + @Test + public void testSmallerNumeric() { + tester.assertParsed(">454", ">454", Query.Type.ANY); + } + + @Test + public void testFullRange() { + tester.assertParsed("[34;454]", "[34;454]", Query.Type.ANY); + } + + public void testFullRangeLimit() { + tester.assertParsed("[34;454;7]", "[34;454;7]", Query.Type.ANY); + tester.assertParsed("[34;454;-7]", "[34;454;-7]", Query.Type.ANY); + } + + @Test + public void testLowOpenRange() { + tester.assertParsed("[;454]", "[;454]", Query.Type.ANY); + } + + @Test + public void testHiOpenRange() { + tester.assertParsed("[34;]", "[34;]", Query.Type.ANY); + } + + @Test + public void testNumericWithIndex() { + tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", + Query.Type.ANY); + } + + @Test + public void testMultipleNumeric() { + tester.assertParsed("OR [34;454] <34", "[34;454] <34", Query.Type.ANY); + } + + @Test + public void testMultipleIntegerWithIndex() { + tester.assertParsed("OR document.size:[34;454] date:>1234567890", + "document.size:[34;454] date:>1234567890", Query.Type.ANY); + } + + @Test + public void testMixedNumericAndOtherTerms() { + tester.assertParsed("RANK (AND document.size:<1024 xyzzy) foo date:>123456890", + "foo +document.size:<1024 +xyzzy date:>123456890", + Query.Type.ANY); + } + + /** Test 50. Semantics changed: Old parser: OR to be or not */ + public void testEmptyPhrase() { + tester.assertParsed("\"to be or not\"", "\"\"to be or not", Query.Type.ANY); + } + + @Test + public void testItemPhraseEmptyPhrase() { + tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", + Query.Type.ANY); + } + + @Test + public void testSimpleQuery() { + tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", + Query.Type.ANY); + } + + @Test + public void testExcessivePluses() { + tester.assertParsed("+(AND other is nothing) -test", + "++other +++++is ++++++nothing -test", Query.Type.ANY); + } + + @Test + public void testMinusAndPluses() { + tester.assertParsed(null, "--test+-if", Query.Type.ANY); + } + + @Test + public void testPlusesAndMinuses() { + Item root=tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ANY); + assertTrue(root instanceof PhraseItem); + assertFalse(((PhraseItem)root).isExplicit()); + } + + @Test + public void testNumbers() { + tester.assertParsed("\"123 2132odfd 934032 32423\"", + "123+2132odfd.934032,,32423", Query.Type.ANY); + } + + @Test + public void testOtherSignsInQuote() { + tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", + Query.Type.ANY); + } + + @Test + public void testGribberish() { + tester.assertParsed("1349832840234l3040roer\u00e6lf12", + ",1349832840234l3040roer\u00e6lf12", Query.Type.ANY); + } + + @Test + public void testUrl() { + tester.assertParsed("www:\"www hotelaiguablava com\"", + "+www:www.hotelaiguablava:com", Query.Type.ANY); + } + + @Test + public void testUrlGribberish() { + tester.assertParsed("OR \"3 16\" fast.type:lycosoffensive", + "[ 3:16 fast.type:lycosoffensive", Query.Type.ANY); + } + + @Test + public void testBracedWordAny() { + tester.assertParsed("foo", "(foo)", Query.Type.ANY); + } + + @Test + public void testBracedWordAll() { + tester.assertParsed("foo", "(foo)", Query.Type.ALL); + + } + + @Test + public void testBracedWords() { + tester.assertParsed("OR (OR foo bar) (OR xyzzy foobar)", + "(foo bar) (xyzzy foobar)", Query.Type.ANY); + } + + @Test + public void testNullAdvanced() { + tester.assertParsed(null, null, Query.Type.ADVANCED); + } + + @Test + public void testEmptyAdvanced() { + tester.assertParsed(null, "", Query.Type.ADVANCED); + } + + @Test + public void testSimpleAdvanced() { + tester.assertParsed("foobar", "foobar", Query.Type.ADVANCED); + } + + @Test + public void testPrefixAdvanced() { + tester.assertParsed("url:foobar", "url:foobar", Query.Type.ADVANCED); + } + + @Test + public void testPrefixWithDotAdvanced() { + tester.assertParsed("normal.title:foobar", "normal.title:foobar", + Query.Type.ADVANCED); + } + + @Test + public void testUTF8Advanced() { + tester.assertParsed("m\u00fcnchen", "m\u00fcnchen", Query.Type.ADVANCED); + } + + @Test + public void testSimplePhraseAdvanced() { + tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", + Query.Type.ADVANCED); + } + + @Test + public void testSimplePhraseWithIndexAdvanced() { + tester.assertParsed("normal.title:\"foo bar foobar\"", + "normal.title:\"foo bar foobar\"", Query.Type.ADVANCED); + } + + @Test + public void testMultiplePhrasesAdvanced() { + tester.assertParsed("AND \"foo bar foobar\" \"baz gaz faz\"", + "\"foo bar foobar\" and \"baz gaz faz\"", Query.Type.ADVANCED); + } + + @Test + public void testNumberAdvanced() { + tester.assertParsed("34", "34", Query.Type.ADVANCED); + } + + @Test + public void testLargerNumberAdvanced() { + tester.assertParsed("<454", "<454", Query.Type.ADVANCED); + } + + @Test + public void testLesserNumberAdvanced() { + tester.assertParsed(">454", ">454", Query.Type.ADVANCED); + } + + @Test + public void testRangeAdvanced() { + tester.assertParsed("[34;454]", "[34;454]", Query.Type.ADVANCED); + } + + @Test + public void testLowOpenRangeAdvanced() { + tester.assertParsed("[;454]", "[;454]", Query.Type.ADVANCED); + } + + @Test + public void testHighOpenRangeAdvanced() { + tester.assertParsed("[34;]", "[34;]", Query.Type.ADVANCED); + } + + @Test + public void testIdexedRangeAdvanced() { + tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", Query.Type.ADVANCED); + } + + @Test + public void testSimpleAndAdvanced() { + tester.assertParsed("AND foo bar", "foo and bar", Query.Type.ADVANCED); + } + + @Test + public void testSimpleOrAdvanced() { + tester.assertParsed("OR foo bar", "foo or bar", Query.Type.ADVANCED); + } + + @Test + public void testSimpleAndNotAdvanced() { + tester.assertParsed("+foo -bar", "foo andnot bar", Query.Type.ADVANCED); + } + + @Test + public void testSimpleRankAdvanced() { + tester.assertParsed("RANK foo bar", "foo rank bar", Query.Type.ADVANCED); + } + + @Test + public void testMultipleAndAdvanced() { + tester.assertParsed("AND foo bar foobar", "foo and bar and foobar", Query.Type.ADVANCED); + } + + @Test + public void testMultipleOrAdvanced() { + tester.assertParsed("OR foo bar foobar", "foo or bar or foobar", Query.Type.ADVANCED); + } + + @Test + public void testMultipleAndnotAdvanced() { + tester.assertParsed("+foo -bar -foobar", "foo andnot bar andnot foobar", Query.Type.ADVANCED); + } + + @Test + public void testMultipleRankAdvanced() { + tester.assertParsed("RANK foo bar foobar", "foo rank bar rank foobar", Query.Type.ADVANCED); + } + + @Test + public void testMixedAdvanced() { + tester.assertParsed("OR (AND foo bar) foobar", "foo and bar or foobar", Query.Type.ADVANCED); + } + + @Test + public void testNestedAdvanced() { + tester.assertParsed("AND foo (OR bar foobar)", "foo and (bar or foobar)", Query.Type.ADVANCED); + } + + @Test + public void testMultipleNestedAdvanced() { + tester.assertParsed("+(AND foo xyzzy) -(OR bar foobar)", + "(foo and xyzzy) andnot (bar or foobar)", Query.Type.ADVANCED); + } + + @Test + public void testDoubleNestedAdvanced() { + tester.assertParsed("AND foo (OR bar (OR xyzzy foobar))", + "foo and (bar or (xyzzy or foobar))", Query.Type.ADVANCED); + } + + @Test + public void testDeeplyAdvanced() { + tester.assertParsed( + "AND foo (OR bar (OR (AND (AND baz (+(OR bazar zyxxy) -fozbaz)) (OR boz bozor) xyzzy) foobar))", + "foo and (bar or ((baz and ((bazar or zyxxy) andnot fozbaz)) and (boz or bozor) and xyzzy or foobar))", + Query.Type.ADVANCED); + } + + @Test + public void testDeeplyAdvancedUppercase() { + tester.assertParsed( + "AND FOO (OR BAR (OR (AND (AND BAZ (+(OR BAZAR ZYXXY) -FOZBAZ)) (OR BOZ BOZOR) XYZZY) FOOBAR))", + "FOO AND (BAR OR ((BAZ AND ((BAZAR OR ZYXXY) ANDNOT FOZBAZ)) AND (BOZ OR BOZOR) AND XYZZY OR FOOBAR))", + Query.Type.ADVANCED); + } + + @Test + public void testAbortedIntegerRange() { + tester.assertParsed("AND audio.audall:744 audio.audall:ph", + "+audio.audall:[744 +audio.audall:ph", Query.Type.ANY); + } + + @Test + public void testJunk() { + tester.assertParsed("+l -fast.type:offensive", + ",;'/.? -fast.type:offensive", Query.Type.ALL); + } + + @Test + public void testOneTermPhraseWithIndex() { + tester.assertParsed("normal.title:foo", "normal.title:\"foo\"", Query.Type.ANY); + } + + @Test + public void testOneTermPhraseWithIndexAdvanced() { + tester.assertParsed("normal.title:foo", "normal.title:\"foo\"", Query.Type.ADVANCED); + } + + @Test + public void testIncorrect1Advanced() { + tester.assertParsed("\"to be or not\"", "\"\"to be or not", Query.Type.ADVANCED); + } + + @Test + public void testIncorrect2Advanced() { + tester.assertParsed("AND to \"or not to be\"", "+to\"or not to be\"\"\"", Query.Type.ADVANCED); + } + + @Test + public void testIncorrect3Advanced() { + tester.assertParsed("AND if am \"f g 4 2\" maybe", + "if am \" f g 4 2\"\" maybe", Query.Type.ADVANCED); + } + + @Test + public void testIncorrect4Advanced() { + tester.assertParsed("AND other is nothing test", + "++other +++++is ++++++nothing -test", Query.Type.ADVANCED); + } + + @Test + public void testImplicitPhrase1Advanced() { + tester.assertParsed("\"test if\"", "--test+-if", Query.Type.ADVANCED); + } + + @Test + public void testImplicitPhrase2Advanced() { + tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ADVANCED); + } + + @Test + public void testImplicitPhrase3Advanced() { + tester.assertParsed("\"123 2132odfd 934032 32423\"", + "123+2132odfd.934032,,32423", Query.Type.ADVANCED); + } + + @Test + public void testImplicitPhrase4Advanced() { + tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", Query.Type.ADVANCED); + } + + @Test + public void testUtf8Advanced() { + tester.assertParsed("1349832840234l3040roer\u00e6lf12", + ",1349832840234l3040roer\u00e6lf12", Query.Type.ADVANCED); + } + + @Test + public void testOperatorSearchAdvanced() { + tester.assertParsed("RANK (OR (AND and and) or andnot) rank", + "and and and or or or andnot rank rank", Query.Type.ADVANCED); + } + + @Test + public void testIncorrectParenthesisAdvanced() { + tester.assertParsed("AND foo bar", "foo and bar )", Query.Type.ADVANCED); + } + + @Test + public void testOpeningParenthesisOnlyAdvanced() { + tester.assertParsed("AND foo (OR bar (AND foobar xyzzy))", + "(foo and (bar or (foobar and xyzzy", Query.Type.ADVANCED); + } + + @Test + public void testSimpleWeight() { + tester.assertParsed("foo!150", "foo!", Query.Type.ANY); + } + + @Test + public void testMultipleWeight() { + tester.assertParsed("foo!250", "foo!!!", Query.Type.ANY); + } + + @Test + public void testExplicitWeight() { + tester.assertParsed("foo!200", "foo!200", Query.Type.ANY); + } + + @Test + public void testExplicitZeroWeight() { + tester.assertParsed("foo!0", "foo!0", Query.Type.ANY); + } + + @Test + public void testSimplePhraseWeight() { + tester.assertParsed("\"foo bar\"!150", "\"foo bar\"!", Query.Type.ANY); + } + + @Test + public void testSingleHyphen() { + tester.assertParsed("\"a b\"", "a-b", Query.Type.ALL); + } + + @Test + public void testUserCase() { + tester.assertParsed("\"a a\"", "\"a- a-*\"", Query.Type.ALL); + } + + @Test + public void testMultiplePhraseWeight() { + tester.assertParsed("\"foo bar\"!250", "\"foo bar\"!!!", Query.Type.ANY); + } + + @Test + public void testExplicitPhraseWeight() { + tester.assertParsed("\"foo bar\"!200", "\"foo bar\"!200", Query.Type.ANY); + } + + @Test + public void testUrlSubmodeHyphen() { + assertTrue(ParsingTester.createIndexFacts().newSession(new Query()).getIndex("url.all").isUriIndex()); + tester.assertParsed("url.all:\"www-microsoft com\"", "url.all:www-microsoft.com", Query.Type.ANY); + } + + @Test + public void testUrlSubmodeUnderscore() { + tester.assertParsed("url.all:\"www_microsoft com\"", "url.all:www_microsoft.com", Query.Type.ANY); + } + + @Test + public void testUrlSubmode() { + tester.assertParsed("host.all:\"www-microsoft com $\"", "host.all:www-microsoft.com", Query.Type.ANY); + } + + @Test + public void testExplicitHostNameAnchoringHost() { + tester.assertParsed("host.all:\"^ www-microsoft com $\"", "host.all:^www-microsoft.com$", Query.Type.ANY); + } + + @Test + public void testExplicitHostNameAnchoringSite() { + tester.assertParsed("site:\"^ www-microsoft com $\"", "site:^www-microsoft.com$", Query.Type.ANY); + } + + @Test + public void testExplicitHostNameAnchoring() { + tester.assertParsed("host.all:\"^ http www krangaz-central com index html $\"", + "host.all:^http://www.krangaz-central.com/index.html$", + Query.Type.ANY); + } + + @Test + public void testExplicitHostAnchoringRemoval() { + tester.assertParsed("host.all:\"www-microsoft com\"", + "host.all:www-microsoft.com*", Query.Type.ANY); + + } + + @Test + public void testQuery1Any() { + tester.assertParsed("RANK (AND fast \"search engine\") kernel", + "+fast +\"search engine\" kernel", Query.Type.ANY); + + } + + @Test + public void testQuery1Advanced() { + tester.assertParsed("RANK (AND fast \"search engine\") kernel", + "+fast and \"search engine\" rank kernel", Query.Type.ADVANCED); + + } + + @Test + public void testQuery2Any() { + tester.assertParsed("+(OR title:car bmw) -z3", "title:car bmw -z3", + Query.Type.ANY); + + } + + @Test + public void testQuery2Advanced() { + tester.assertParsed("+(OR title:car bmw) -z3", "title:car or bmw andnot z3", Query.Type.ADVANCED); + } + + @Test + public void testQuery3All() { + tester.assertParsed("+(AND FAST search domain:no pagedepth:0) -title:phrase", + "FAST search -title:phrase domain:no Pagedepth:0", + Query.Type.ALL); + } + + @Test + public void testQuery4Advanced() { + tester.assertParsed("AND (+(AND FAST search) -title:phrase) domain:no pagedepth:0", + "FAST and search andnot title:phrase and domain:no and Pagedepth:0", + Query.Type.ADVANCED); + } + + @Test + public void testQuery5Any() { + tester.assertParsed("AND alltheweb fast search", "+alltheweb +fast +search", Query.Type.ANY); + } + + @Test + public void testQuery6Any() { + tester.assertParsed("RANK (+(AND query language) -sql) search", "+query +language -sql search", Query.Type.ANY); + } + + @Test + public void testQuery7Any() { + tester.assertParsed( + "+(AND alltheweb (OR search engine)) -(OR excite altavista)", + "(alltheweb and (search or engine)) andnot (excite or altavista)", + Query.Type.ADVANCED); + } + + @Test + public void testQuery8Advanced() { + tester.assertParsed( + "RANK (AND \"search engines\" \"query processing\") \"fast search\"", + "(\"search engines\" and \"query processing\") rank \"fast search\"", + Query.Type.ADVANCED); + } + + @Test + public void testPStrangeAdvanced() { + tester.assertParsed("AND AND r.s:jnl", "( AND +r.s:jnl) ", Query.Type.ADVANCED); + } + + @Test + public void testEmptyNestAdvanced() { + tester.assertParsed(null, "() ", Query.Type.ADVANCED); + } + + @Test + public void testNestedBeginningAdvanced() { + tester.assertParsed("AND (OR a b) c", "(a or b) and c", Query.Type.ADVANCED); + } + + @Test + public void testNestedPositiveAny() { + tester.assertParsed("AND (OR a b) c", "+(a b) +c", Query.Type.ANY); + } + + @Test + public void testParseAdvancedQuery() { + tester.assertParsed("AND joplin remediation r.s:jnl", + "(joplin and + and remediation and +r.s:jnl)", + Query.Type.ADVANCED); + } + + @Test + public void testSimpleDotPhraseAny() { + tester.assertParsed("OR a \"b c\" d", "a b.c d", Query.Type.ANY); + } + + @Test + public void testSimpleHyphenPhraseAny() { + tester.assertParsed("OR a \"b c\" d", "a b-c d", Query.Type.ANY); + } + + @Test + public void testAnotherSimpleDotPhraseAny() { + tester.assertParsed("OR \"a b\" c d", "a.b c d", Query.Type.ANY); + } + + @Test + public void testYetAnotherSimpleDotPhraseAny() { + tester.assertParsed("OR a b \"c d\"", "a b c.d", Query.Type.ANY); + } + + @Test + public void testVariousSeparatorsPhraseAny() { + tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ANY); + } + + @Test + public void testDoublyMarkedPhraseAny() { + tester.assertParsed("OR a \"b c\" d", "a \"b.c\" d", Query.Type.ANY); + } + + @Test + public void testPartlyDoublyMarkedPhraseAny() { + tester.assertParsed("OR a \"b c d\"", "a \"b.c d\"", Query.Type.ANY); + } + + @Test + public void testIndexedDottedPhraseAny() { + tester.assertParsed("OR a url:\"b c\" d", "a url:b.c d", Query.Type.ANY); + } + + @Test + public void testIndexedPlusedPhraseAny() { + tester.assertParsed("OR a normal.title:\"b c\" d", "a normal.title:b+c d", + Query.Type.ANY); + } + + @Test + public void testNestedNotAny() { + tester.assertParsed( + "RANK (+(OR normal.title:foobar url:\"www pvv org\") -foo) a", + "a +(normal.title:foobar url:www.pvv.org) -foo", Query.Type.ANY); + } + + @Test + public void testDottedPhraseAdvanced() { + tester.assertParsed("OR a \"b c\"", "a or b.c", Query.Type.ADVANCED); + } + + @Test + public void testHyphenPhraseAdvanced() { + tester.assertParsed("OR (AND a \"b c\") d", "a and b-c or d", Query.Type.ADVANCED); + } + + @Test + public void testAnotherDottedPhraseAdvanced() { + tester.assertParsed("OR \"a b\" c", "a.b or c", Query.Type.ADVANCED); + } + + @Test + public void testNottedDottedPhraseAdvanced() { + tester.assertParsed("+a -\"c d\"", "a andnot c.d", Query.Type.ADVANCED); + } + + @Test + public void testVariousSeparatorsPhraseAdvanced() { + tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ADVANCED); + } + + @Test + public void testDoublyPhrasedAdvanced() { + tester.assertParsed("OR (AND a \"b c\") d", "a and \"b.c\" or d", Query.Type.ADVANCED); + } + + @Test + public void testPartlyDoublyPhrasedAdvanced() { + tester.assertParsed("OR a \"b c d\"", "a or \"b.c d\"", Query.Type.ADVANCED); + } + + @Test + public void testNestedDottedPhraseAdvanced() { + tester.assertParsed("AND a (OR url:\"b c\" d)", "a and(url:\"b.c\" or d)", Query.Type.ADVANCED); + } + + @Test + public void testNestedPlussedPhraseAdvanced() { + tester.assertParsed("AND (OR a normal.title:\"b c\") d", + "a or normal.title:b+c and d", Query.Type.ADVANCED); + } + + @Test + public void testNottedNestedDottedPhraseAdvanced() { + tester.assertParsed( + "+(AND a (OR normal.title:foobar url:\"www pvv org\")) -foo", + "a and (normal.title:foobar or url:www.pvv.org) andnot foo", + Query.Type.ADVANCED); + } + + @Test + public void testPlusedThenQuotedPhraseAny() { + tester.assertParsed("\"a b c\"", "a+\"b c\"", Query.Type.ANY); + } + + @Test + public void testPlusedTwiceThenQuotedPhraseAny() { + tester.assertParsed("\"a b c d\"", "a+b+\"c d\"", Query.Type.ANY); + } + + @Test + public void testPlusedThenQuotedPhraseAdvanced() { + tester.assertParsed("\"a b c\"", "a+\"b c\"", Query.Type.ADVANCED); + } + + @Test + public void testPhrasesInBraces() { + tester.assertParsed("url.domain:\"microsoft com\"", + "+(url.domain:microsoft.com)", Query.Type.ALL); + } + + @Test + public void testDoublyPhrasedPhrasesInBraces() { + tester.assertParsed("url.domain:\"microsoft com\"", + "+(url.domain:\"microsoft.com\")", Query.Type.ALL); + } + + @Test + public void testSinglePrefixTerm() { + Item root = tester.assertParsed("prefix*", "prefix*", Query.Type.ANY); + assertTrue(root instanceof PrefixItem); + } + + @Test + public void testSingleSubstringTerm() { + Item root = tester.assertParsed("*substring*", "*substring*", Query.Type.ANY); + assertTrue(root instanceof SubstringItem); + } + + @Test + public void testSingleSuffixTerm() { + Item root = tester.assertParsed("*suffix", "*suffix", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + @Test + public void testPrefixAndWordTerms() { + Item root = tester.assertParsed("OR foo prefix* bar", "foo prefix* bar", Query.Type.ANY); + assertTrue(((OrItem)root).getItem(1) instanceof PrefixItem); + } + + @Test + public void testSubstringAndWordTerms() { + Item root = tester.assertParsed("OR foo *substring* bar", "foo *substring* bar", Query.Type.ANY); + assertTrue(((OrItem)root).getItem(1) instanceof SubstringItem); + } + + @Test + public void testSuffixAndWordTerms() { + Item root = tester.assertParsed("OR foo *suffix bar", "foo *suffix bar", Query.Type.ANY); + assertTrue(((OrItem)root).getItem(1) instanceof SuffixItem); + } + + @Test + public void testPhraseNotPrefix() { + tester.assertParsed("OR foo \"prefix bar\"", "foo prefix*bar", Query.Type.ANY); + } + + @Test + public void testPhraseNotSubstring() { + tester.assertParsed("OR foo \"substring bar\"", "foo *substring*bar", Query.Type.ANY); + } + + @Test + public void testPhraseNotSuffix() { + tester.assertParsed("OR \"foo suffix\" bar", "foo*suffix bar", Query.Type.ANY); + } + + @Test + public void testIndexedPrefix() { + Item root = tester.assertParsed("foo.bar:prefix*", "foo.bar:prefix*", Query.Type.ANY); + assertTrue(root instanceof PrefixItem); + } + + @Test + public void testIndexedSubstring() { + Item root = tester.assertParsed("foo.bar:*substring*", "foo.bar:*substring*", Query.Type.ANY); + assertTrue(root instanceof SubstringItem); + } + + @Test + public void testIndexedSuffix() { + Item root = tester.assertParsed("foo.bar:*suffix", "foo.bar:*suffix", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + @Test + public void testIndexedPhraseNotPrefix() { + tester.assertParsed("foo.bar:\"prefix xyzzy\"", "foo.bar:prefix*xyzzy", + Query.Type.ANY); + } + + @Test + public void testIndexedPhraseNotSubstring() { + tester.assertParsed("foo.bar:\"substring xyzzy\"", "foo.bar:*substring*xyzzy", + Query.Type.ANY); + } + + @Test + public void testIndexedPhraseNotSuffix() { + tester.assertParsed("foo.bar:\"xyzzy suffix\"", "foo.bar:xyzzy*suffix", + Query.Type.ANY); + } + + @Test + public void testPrefixWithWeight() { + Item root = tester.assertParsed("prefix*!200", "prefix*!200", Query.Type.ANY); + assertTrue(root instanceof PrefixItem); + } + + @Test + public void testSubstringWithWeight() { + Item root = tester.assertParsed("*substring*!200", "*substring*!200", Query.Type.ANY); + assertTrue(root instanceof SubstringItem); + } + + @Test + public void testSuffixWithWeight() { + Item root = tester.assertParsed("*suffix!200", "*suffix!200", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + /** Non existing index → phrase **/ + @Test + public void testNonIndexPhraseNotPrefix() { + tester.assertParsed("\"void prefix\"", "void:prefix*", Query.Type.ANY); + } + + @Test + public void testNonIndexPhraseNotSubstring() { + tester.assertParsed("\"void substring\"", "void:*substring*", Query.Type.ANY); + } + + @Test + public void testNonIndexPhraseNotSuffix() { + tester.assertParsed("\"void suffix\"", "void:*suffix", Query.Type.ANY); + } + + /** Explicit phrase → remove '*' **/ + @Test + public void testExplicitPhraseNotPrefix() { + tester.assertParsed("\"prefix bar\"", "\"prefix* bar\"", Query.Type.ANY); + } + + @Test + public void testExplicitPhraseNotSubstring() { + tester.assertParsed("\"substring bar\"", "\"*substring* bar\"", Query.Type.ANY); + } + + @Test + public void testExplicitPhraseNotSuffix() { + tester.assertParsed("\"suffix bar\"", "\"*suffix bar\"", Query.Type.ANY); + } + + /** Extra star is ignored */ + @Test + public void testPrefixExtraStar() { + Item root = tester.assertParsed("prefix*", "prefix**", Query.Type.ANY); + assertTrue(root instanceof PrefixItem); + } + + @Test + public void testSubstringExtraStar() { + Item root = tester.assertParsed("*substring*", "**substring**", Query.Type.ANY); + assertTrue(root instanceof SubstringItem); + } + + @Test + public void testSuffixExtraStar() { + Item root = tester.assertParsed("*suffix", "**suffix", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + @Test + public void testPrefixExtraSpace() { + Item root = tester.assertParsed("prefix", "prefix *", Query.Type.ANY); + assertTrue(root instanceof WordItem); + } + + @Test + public void testSubstringExtraSpace() { + Item root = tester.assertParsed("*substring*", "* substring*", Query.Type.ANY); + assertTrue(root instanceof SubstringItem); + } + + @Test + public void testSubstringExtraSpace2() { + Item root = tester.assertParsed("*substring", "* substring *", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + @Test + public void testSuffixExtraSpace() { + Item root = tester.assertParsed("*suffix", "* suffix", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + /** Extra spaces with index **/ + @Test + public void testIndexPrefixExtraSpace() { + tester.assertParsed("\"foo prefix\"", "foo:prefix *", Query.Type.ANY); + } + + @Test + public void testIndexSubstringExtraSpace() { + Item root = tester.assertParsed("OR foo substring*", "foo:* substring*", Query.Type.ANY); + assertTrue(((OrItem)root).getItem(0) instanceof WordItem); + assertTrue(((OrItem)root).getItem(1) instanceof PrefixItem); + } + + @Test + public void testIndexSubstringExtraSpace2() { + Item root = tester.assertParsed("OR foo substring", "foo:* substring *", Query.Type.ANY); + assertTrue(((OrItem)root).getItem(0) instanceof WordItem); + assertTrue(((OrItem)root).getItem(1) instanceof WordItem); + } + + @Test + public void testIndexSuffixExtraSpace() { + Item root = tester.assertParsed("OR foo suffix", "foo:* suffix", Query.Type.ANY); + assertTrue(((OrItem)root).getItem(0) instanceof WordItem); + assertTrue(((OrItem)root).getItem(1) instanceof WordItem); + } + + /** Various tests for prefix, substring, and suffix terms **/ + @Test + public void testTermsWithStarsAndSpaces() { + tester.assertParsed("OR foo *bar", "foo * bar", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndSpaces2() { + tester.assertParsed("OR foo *bar *baz", "foo * * bar * * baz", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndPlussAndMinus() { + tester.assertParsed("+(AND *bar baz*) -*foo*", "+*bar -*foo* +baz*", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndPlussAndMinus2() { + tester.assertParsed("OR *bar *foo baz", "+ * bar - * foo * + baz *", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndExclamation() { + tester.assertParsed("OR foo* 200 *bar* 200 *baz 200", "foo* !200 *bar* !200 *baz !200", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndExclamation2() { + tester.assertParsed("OR foo 200 *bar 200", "foo *!200 *bar *!200", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndParenthesis() { + tester.assertParsed("RANK *baz *bar* foo*", "(foo*) (*bar*) (*baz)", Query.Type.ANY); + } + + @Test + public void testTermsWithStarsAndParenthesis2() { + tester.assertParsed("RANK baz bar foo", "(foo)* *(bar)* *(baz)", Query.Type.ANY); + } + + + @Test + public void testSimpleAndFilter() { + tester.assertParsed("AND bar |foo", "bar", "+foo", Query.Type.ANY); + } + + @Test + public void testSimpleRankFilter() { + tester.assertParsed("RANK bar |foo", "bar", "foo", Query.Type.ANY); + } + + @Test + public void testSimpleNotFilter() { + tester.assertParsed("+bar -|foo", "bar", "-foo", Query.Type.ANY); + } + + @Test + public void testSimpleCompoundFilter1() { + tester.assertParsed("RANK (AND bar |foo1) |foo2", "bar", "+foo1 foo2", + Query.Type.ANY); + } + + @Test + public void testSimpleCompoundFilter2() { + tester.assertParsed("+(AND bar |foo1) -|foo3", "bar", "+foo1 -foo3", + Query.Type.ANY); + } + + @Test + public void testSimpleCompoundFilter3() { + tester.assertParsed("RANK (+bar -|foo3) |foo2", "bar", "foo2 -foo3", + Query.Type.ANY); + } + + @Test + public void testSimpleCompoundFilter4() { + tester.assertParsed("RANK (+(AND bar |foo1) -|foo3) |foo2", "bar", + "+foo1 foo2 -foo3", Query.Type.ANY); + } + + @Test + public void testAndFilterEmptyQuery() { + tester.assertParsed("|foo", "", "+foo", Query.Type.ANY); + } + + @Test + public void testRankFilterEmptyQuery() { + tester.assertParsed("|foo", "", "foo", Query.Type.ANY); + } + + @Test + public void testNotFilterEmptyQuery() { + tester.assertParsed(null, "", "-foo", Query.Type.ANY); + } + + @Test + public void testCompoundFilter1EmptyQuery() { + tester.assertParsed("RANK |foo1 |foo2", "", "+foo1 foo2", Query.Type.ANY); + } + + @Test + public void testCompoundFilter2EmptyQuery() { + tester.assertParsed("+|foo1 -|foo3", "", "+foo1 -foo3", Query.Type.ANY); + } + + @Test + public void testCompoundFilter3EmptyQuery() { + tester.assertParsed("+|foo2 -|foo3", "", "foo2 -foo3", Query.Type.ANY); + } + + @Test + public void testCompoundFilter4EmptyQuery() { + tester.assertParsed("RANK (+|foo1 -|foo3) |foo2", "", "+foo1 foo2 -foo3", + Query.Type.ANY); + } + + @Test + public void testMultitermAndFilter() { + tester.assertParsed("AND bar |foo |foz", "bar", "+foo +foz", Query.Type.ANY); + } + + @Test + public void testMultitermRankFilter() { + tester.assertParsed("RANK bar |foo |foz", "bar", "foo foz", Query.Type.ANY); + } + + @Test + public void testMultitermNotFilter() { + tester.assertParsed("+bar -|foo -|foz", "bar", "-foo -foz", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter1() { + tester.assertParsed("RANK (AND bar |foo1 |foz1) |foo2 |foz2", "bar", + "+foo1 +foz1 foo2 foz2", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter2() { + tester.assertParsed("+(AND bar |foo1 |foz1) -|foo3 -|foz3", "bar", + "+foo1 +foz1 -foo3 -foz3", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter3() { + tester.assertParsed("RANK (+bar -|foo3 -|foz3) |foo2 |foz2", "bar", + "foo2 foz2 -foo3 -foz3", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter4() { + tester.assertParsed("RANK (+(AND bar |foo1 |foz1) -|foo3 -|foz3) |foo2 |foz2", + "bar", "+foo1 +foz1 foo2 foz2 -foo3 -foz3", Query.Type.ANY); + } + + @Test + public void testMultitermAndFilterEmptyQuery() { + tester.assertParsed("AND |foo |foz", "", "+foo +foz", Query.Type.ANY); + } + + @Test + public void testMultitermRankFilterEmptyQuery() { + tester.assertParsed("OR |foo |foz", "", "foo foz", Query.Type.ANY); + } + + @Test + public void testMultitermNotFilterEmptyQuery() { + tester.assertParsed(null, "", "-foo -foz", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter1EmptyQuery() { + tester.assertParsed("RANK (AND |foo1 |foz1) |foo2 |foz2", "", + "+foo1 +foz1 foo2 foz2", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter2EmptyQuery() { + tester.assertParsed("+(AND |foo1 |foz1) -|foo3 -|foz3", "", + "+foo1 +foz1 -foo3 -foz3", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter3EmptyQuery() { + tester.assertParsed("+(OR |foo2 |foz2) -|foo3 -|foz3", "", + "foo2 foz2 -foo3 -foz3", Query.Type.ANY); + } + + @Test + public void testMultitermCompoundFilter4EmptyQuery() { + tester.assertParsed("RANK (+(AND |foo1 |foz1) -|foo3 -|foz3) |foo2 |foz2", "", + "+foo1 +foz1 foo2 foz2 -foo3 -foz3", Query.Type.ANY); + } + + @Test + public void testMultipleDifferentPhraseSeparators() { + tester.assertParsed("\"foo bar\"", "foo.-.bar", Query.Type.ANY); + } + + @Test + public void testNoisyFilter() { + tester.assertParsed("RANK (+(AND foobar kanoo) -|foo) |bar", "foobar and kanoo", + "-foo ;+;bar", Query.Type.ADVANCED); + } + + @Test + public void testReallyNoisyQuery1() { + tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", + Query.Type.ALL); + } + + @Test + public void testReallyNoisyQuery2() { + tester.assertParsed("\"\u03bc\u03bc hei\"", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); + } + + @Test + public void testReallyNoisyQuery3() { + tester.assertParsed("AND \"hei hallo\" du der", "hei-hallo;du;der", + Query.Type.ALL); + } + + @Test + public void testNumberParsing() { + Item root = tester.parseQuery("normal:400", null, Language.UNKNOWN, Query.Type.ANY, TestLinguistics.INSTANCE); + assertEquals(root.getCode(), 5); + } + + @Test + public void testRangeParsing() { + Item root = tester.parseQuery("normal:[5;400]", null, Language.UNKNOWN, Query.Type.ANY, TestLinguistics.INSTANCE); + assertEquals(root.toString(), "normal:[5;400]"); + assertEquals(root.getCode(), 5); + } + + @Test + public void testNumberAsPrefix() { + Item root = tester.assertParsed("89*", "89*", Query.Type.ANY); + assertTrue(root instanceof PrefixItem); + } + + @Test + public void testNumberAsSubstring() { + Item root = tester.assertParsed("*89*", "*89*", Query.Type.ANY); + assertTrue(root instanceof SubstringItem); + } + + @Test + public void testNumberAsSuffix() { + Item root = tester.assertParsed("*89", "*89", Query.Type.ANY); + assertTrue(root instanceof SuffixItem); + } + + @Test + public void testTheStupidSymbolsWhichAreNowWordCharactersInUnicode() { + tester.assertParsed("\"yz a\"", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); + } + + @Test + public void testTWoWords() { + tester.assertParsed("\"hei h\u00e5\"", "\"hei h\u00e5\"", Query.Type.ANY); + } + + @Test + public void testLoneStar() { + assertNull(tester.parseQuery("*", null, Language.UNKNOWN, Query.Type.ANY, TestLinguistics.INSTANCE)); + } + + @Test + public void testLoneStarWithFilter() { + tester.assertParsed("|a", "*", "+a", Query.Type.ANY); + } + + @Test + public void testImplicitPhrasingWithIndex() { + tester.assertParsed("a:\"b c\"", "a:/b/c", Query.Type.ANY); + } + + @Test + public void testSingleNoisyTermWithIndex() { + tester.assertParsed("a:b", "a:/b", Query.Type.ANY); + } + + @Test + public void testSingleNoisyPhraseWithIndex() { + tester.assertParsed("mail:\"yahoo com\"", "mail:@yahoo.com", Query.Type.ANY); + } + + @Test + public void testPhraseWithWeightAndIndex() { + tester.assertParsed("to:\"a b\"!150", "to:\"a b\"!150", Query.Type.ANY); + } + + @Test + public void testTermWithWeightAndIndex() { + tester.assertParsed("to:a!150", "to:a!150", Query.Type.ANY); + } + + @Test + public void testPhrasingWithIndexAndQuerySyntax() { + tester.assertParsed("to:\"a b c\"", "to:\"a (b c)\"", Query.Type.ANY); + } + + @Test + public void testPhrasingWithIndexAndHalfBrokenQuerySyntax() { + tester.assertParsed("to:\"a b c\"", "to:\"a +b c)\"", Query.Type.ANY); + } + + @Test + public void testURLHostQueryOneTerm1() { + tester.assertParsed("site:\"com $\"", "site:com", Query.Type.ANY); + } + + @Test + public void testURLHostQueryOneTerm2() { + tester.assertParsed("site:com", "site:com*", Query.Type.ANY); + } + + @Test + public void testURLHostQueryOneTerm3() { + tester.assertParsed("site:\"com $\"", "site:.com", Query.Type.ANY); + } + + @Test + public void testURLHostQueryOneTerm4() { + tester.assertParsed("site:\"^ com $\"", "site:^com", Query.Type.ANY); + } + + @Test + public void testURLHostQueryOneTerm5() { + tester.assertParsed("site:\"^ com\"", "site:^com*", Query.Type.ANY); + } + + @Test + public void testFullURLQuery() { + tester.assertParsed( + "url.all:\"http shopping yahoo-inc com 1080 this is a path shop d hab id 1804905709 frag1\"", + "url.all:http://shopping.yahoo-inc.com:1080/this/is/a/path/shop?d=hab&id=1804905709#frag1", + Query.Type.ANY); + } + + @Test + public void testURLQueryHyphen() { + tester.assertParsed( + "url.all:\"http news bbc co uk go rss test - sport1 hi tennis 4112866 stm\"", + "url.all:http://news.bbc.co.uk/go/rss/test/-/sport1/hi/tennis/4112866.stm", + Query.Type.ANY); + } + + @Test + public void testURLQueryUnderScoreNumber() { + tester.assertParsed( + "url.all:\"ap 20050621 45_ap_on_re_la_am_ca aruba_missing_teen_5\"", + "url.all:/ap/20050621/45_ap_on_re_la_am_ca/aruba_missing_teen_5", + Query.Type.ANY); + } + + @Test + public void testOtherComplexUrls() { + tester.assertParsed( + "url.all:\"http redir folha com br redir online dinheiro rss091 http www1 folha uol com br folha dinheiro ult91u96593 shtml\"", + "url.all:http://redir.folha.com.br/redir/online/dinheiro/rss091/*http://www1.folha.uol.com.br/folha/dinheiro/ult91u96593.shtml", + Query.Type.ALL); + tester.assertParsed( + "url.all:\"http economista com mx online4 nsf all 6FC11CB53F8A305B0625702700709029 OpenDocument\"", + "url.all:http://economista.com.mx/online4.nsf/(all)/6FC11CB53F8A305B0625702700709029?OpenDocument", + Query.Type.ALL); + tester.assertParsed( + "url.all:\"http www tierradelfuego info index php s AR13xbyxg espectaculos programa ARc7hzxb\"", + "url.all:http://www.tierradelfuego.info/index.php?s=AR13xbyxg$$espectaculos/programa$ARc7hzxb", + Query.Type.ALL); + tester.assertParsed( + "url.all:\"http www newsadvance com servlet Satellite pagename LNA MGArticle IMD_BasicArticle c MGArticle cid 1031782787014 path mgnetwork diversions\"", + "url.all:http://www.newsadvance.com/servlet/Satellite?pagename=LNA/MGArticle/IMD_BasicArticle&c=MGArticle&cid=1031782787014&path=!mgnetwork!diversions", + Query.Type.ALL); + tester.assertParsed( + "AND ull:\"http www neue oz de information pub Boulevard index html file a 3 s 4 file\" s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", + "ull:http://www.neue-oz.de/information/pub_Boulevard/index.html?file=a:3:{s:4:\"file\";s:37:\"iptc-bdt-20050607-294-dpa_9001170.txt\";s:3:\"dir\";s:26:\"/opt/DPA/parsed/boulevard/\";s:7:\"bereich\";s:9:\"Boulevard\";}", + Query.Type.ALL); + } + + @Test + public void testTooGreedyUrlParsing() { + tester.assertParsed("AND site:\"nypost com $\" about", "site:nypost.com about", + Query.Type.ALL); + } + + @Test + public void testTooGreedyUrlParsing2() { + tester.assertParsed("AND site:\"nypost com $\" about foo", + "site:nypost.com about foo", Query.Type.ALL); + } + + @Test + public void testSimplerDurbin() { + tester.assertParsed("+(OR language:en \"Durbin said\" a) -newstype:rssexclude", + "( a (\"Durbin said\" ) -newstype:rssexclude (language:en )", + Query.Type.ALL); + } + + @Test + public void testSimplerDurbin2() { + tester.assertParsed("+(AND \"Durbin said\" language:en) -newstype:rssexclude", + "( , (\"Durbin said\" ) -newstype:rssexclude (language:en )", + Query.Type.ALL); + } + + @Test + public void testDurbin() { + tester.assertParsed( + "AND \"Richard Durbin\" Welfare (+(OR language:en (OR \"Durbin said\" \"Durbin says\" \"Durbin added\" \"Durbin agreed\" \"Durbin questioned\") date:>1109664000) -newstype:rssexclude)", + "(\"Richard Durbin\" ) \"Welfare\" ((\"Durbin said\" \"Durbin says\" \"Durbin added\" \"Durbin agreed\" \"Durbin questioned\" ) -newstype:rssexclude date:>1109664000 (language:en )", + Query.Type.ALL); + } + + @Test + public void testTooLongQueryTerms() { + tester.assertParsed("AND \"545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"!1000 \"2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"", + "+/545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof&filter=ew:545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof!1000 =.2b..2f.545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", + Query.Type.ALL); + } + + @Test + public void testNonSpecialTokenParsing() { + ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); + customTester.assertParsed("OR c or c with \"tcp ip\"", "c# or c++ with tcp/ip", Query.Type.ANY); + } + + @Test + public void testNonIndexWithColons1() { + tester.assertParsed("OR this is \"notan iindex\"", "this is notan:iindex", Query.Type.ANY); + } + + @Test + public void testNonIndexWithColons2() { + tester.assertParsed("OR this is \"notan iindex either\"", "this is notan:iindex:either", Query.Type.ANY); + } + + @Test + public void testIndexThenUnderscoreTermBecomesIndex() { + tester.assertParsed("name:\"batch article\"", "name:batch_article", Query.Type.ANY); + } + + @Test + public void testFakeCJKSegmenting() { + // "first" "second" and "third" are segments in the test language + Item item = tester.parseQuery("name:firstsecondthird", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); + + assertTrue(item instanceof PhraseSegmentItem); + PhraseSegmentItem phrase = (PhraseSegmentItem) item; + + assertEquals(3, phrase.getItemCount()); + assertEquals("name:first", phrase.getItem(0).toString()); + assertEquals("name:second", phrase.getItem(1).toString()); + assertEquals("name:third", phrase.getItem(2).toString()); + + assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(2)).getIndexName()); + } + + @Test + public void testFakeCJKSegmentingOfPhrase() { + // "first" "second" and "third" are segments in the test language + Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); + + assertTrue(item instanceof PhraseSegmentItem); + PhraseSegmentItem phrase = (PhraseSegmentItem) item; + + assertEquals(3, phrase.getItemCount()); + assertEquals("name:first", phrase.getItem(0).toString()); + assertEquals("name:second", phrase.getItem(1).toString()); + assertEquals("name:third", phrase.getItem(2).toString()); + + assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem)phrase.getItem(2)).getIndexName()); + } + + @Test + public void testAndItemAndImplicitPhrase() { + tester.assertParsed("\"\u00d8 \u00d8 \u00d8 \u00d9\"", + "\u00d8\u00b9\u00d8\u00b1\u00d8\u00a8\u00d9", "", + Query.Type.ALL, Language.CHINESE_SIMPLIFIED); + } + + @Test + public void testAvoidMultiLevelAndForLongCJKQueries() { + Item root = tester.parseQuery( + "\u30d7\u30ed\u91ce\u7403\u962a\u795e\u306e\u672c\u62e0\u5730\u3001\u7532\u5b50\u5712\u7403\u5834\uff08\u5175\u5eab\u770c\u897f\u5bae\u5e02\uff09\u306f\uff11\u65e5\u3001\uff11\uff19\uff12\uff14\u5e74\u30d7\u30ed\u91ce\u7403\u962a\u795e\u306e\u672c\u62e0\u5730\u3001\u7532\u5b50\u5712\u7403\u5834\uff08\u5175\u5eab\u770c\u897f\u5bae\u5e02\uff09\u306f\uff11\u65e5\u3001\uff11\uff19\uff12\uff14\u5e74\u30d7\u30ed\u91ce\u7403\u962a\u795e\u306e\u672c\u62e0\u5730\u3001\u7532\u5b50\u5712\u7403\u5834\uff08\u5175\u5eab\u770c\u897f\u5bae\u5e02\uff09\u306f\uff11\u65e5\u3001\uff11\uff19\uff12\uff14\u5e74\u30d7\u30ed\u91ce\u7403\u962a\u795e\u306e\u672c\u62e0\u5730\u3001\u7532\u5b50\u5712\u7403\u5834\uff08\u5175\u5eab\u770c\u897f\u5bae\u5e02\uff09\u306f\uff11\u65e5\u3001\uff11\uff19\uff12\uff14\u5e74\u30d7\u30ed\u91ce\u7403\u962a\u795e\u306e\u672c\u62e0\u5730\u3001\u7532\u5b50\u5712\u7403\u5834\uff08\u5175\u5eab\u770c\u897f\u5bae\u5e02\uff09\u306f\uff11\u65e5\u3001\uff11\uff19\uff12\uff14\u5e74\u30d7\u30ed\u91ce\u7403\u962a\u795e\u306e\u672c\u62e0\u5730\u3001\u7532\u5b50\u5712\u7403\u5834\uff08\u5175\u5eab\u770c\u897f\u5bae\u5e02\uff09\u306f\uff11\u65e5\u3001\uff11\uff19\uff12\uff14\u5e74", + "", Language.UNKNOWN, Query.Type.ALL, TestLinguistics.INSTANCE); + + assertTrue("Query tree too deep when parsing CJK queries.", + 4 > stackDepth(0, root)); + } + + private int stackDepth(int i, Item root) { + if (root instanceof CompositeItem) { + int maxDepth = i; + + for (Iterator j = ((CompositeItem) root).getItemIterator(); j.hasNext();) { + int newDepth = stackDepth(i + 1, j.next()); + + maxDepth = java.lang.Math.max(maxDepth, newDepth); + } + return maxDepth; + } else { + return i; + } + } + + @Test + public void testFakeCJKSegmentingOfMultiplePhrases() { + Item item = tester.parseQuery("name:firstsecond.s", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); + assertEquals("name:\"'first second' s\"", item.toString()); + } + + @Test + public void testOrFilter() { + tester.assertParsed("AND d (OR |a |b)", "d", "+(a b)", Query.Type.ALL); + } + + @Test + public void testOrFilterWithTypeAdv() { + tester.assertParsed("AND d (OR |a |b)", "d", "+(a b)", Query.Type.ADVANCED); + } + + @Test + public void testPhraseFilter() { + tester.assertParsed("AND d |\"a b\"", "d", "+\"a b\"", Query.Type.ALL); + } + + @Test + public void testMinusAndFilter() { + tester.assertParsed("+d -(AND |a |b)", "d", "-(a b)", Query.Type.ALL); + } + + @Test + public void testOrAndSomeTermsFilter() { + tester.assertParsed("RANK d |c |a |b |e", "d", "c (a b) e", Query.Type.ALL); + } + + // This is an ugly parse tree, but it's at least reasonable + @Test + public void testOrAndSomeTermsFilterAndAnAnd() { + AndItem root=(AndItem)tester.assertParsed("AND (RANK d |c |a |b |e) (OR |e |f)", "d", "c (a b) e +(e f)", Query.Type.ALL); + assertFalse(root.isFilter()); // AND + assertFalse(root.getItem(0).isFilter()); // RANK + assertFalse(((RankItem)root.getItem(0)).getItem(0).isFilter()); // d + assertTrue(((RankItem)root.getItem(0)).getItem(1).isFilter()); // c + assertTrue(((RankItem)root.getItem(0)).getItem(2).isFilter()); // a + assertTrue(((RankItem)root.getItem(0)).getItem(3).isFilter()); // b + assertTrue(((RankItem)root.getItem(0)).getItem(4).isFilter()); // e + assertFalse(root.getItem(1).isFilter()); // OR + assertTrue(((OrItem)root.getItem(1)).getItem(0).isFilter()); // e + assertTrue(((OrItem)root.getItem(1)).getItem(1).isFilter()); // f + } + + @Test + public void testUrlNotConsumingBrace() { + tester.assertParsed("AND A (OR url.all:B url.all:C) D E", "A (url.all:B url.all:C) D E", Query.Type.ALL); + } + + // Really a syntax error on part of the user, but it's part of + // the logic where balanced braces are allowed in URLs + @Test + public void testUrlNotConsumingBrace2() { + tester.assertParsed("AND A (OR url.all:B url.all:C) D E", + "A (url.all:B url.all:C)D E", Query.Type.ALL); + } + + @Test + public void testSiteNotConsumingBrace() { + tester.assertParsed("AND A (OR site:\"B $\" site:\"C $\") D E", + "A (site:B site:C) D E", Query.Type.ALL); + } + + @Test + public void testCommaOnlyLeadsToImplicitPhrasing() { + tester.assertParsed("\"A B C\"", "A,B,C", Query.Type.ALL); + } + + @Test + public void testBangDoesNotBindAcrossSpace() { + tester.assertParsed("word", "word !", Query.Type.ALL); + } + + @Test + public void testLotsOfPlusMinus() { + tester.assertParsed("OR word term", "word - + - + - term", Query.Type.ANY); + } + + @Test + public void testLotsOfMinusPlus() { + tester.assertParsed("OR word term", "word - + - + term", Query.Type.ANY); + } + + @Test + public void testMinusDoesNotBindAcrossSpace() { + tester.assertParsed("OR word term", "word - term", Query.Type.ANY); + } + + @Test + public void testPlusDoesNotBindAcrossSpace() { + tester.assertParsed("OR word term", "word + term", Query.Type.ANY); + } + + @Test + public void testMinusDoesNotBindAcrossSpaceAllQuery() { + tester.assertParsed("AND word term", "word - term", Query.Type.ALL); + } + + @Test + public void testPlusDoesNotBindAcrossSpaceAllQuery() { + tester.assertParsed("AND word term", "word + term", Query.Type.ALL); + } + + @Test + public void testNoSpaceInIndexPrefix() { + tester.assertParsed("AND url domain:s url.domain:b", + "url. domain:s url.domain:b", Query.Type.ALL); + } + + @Test + public void testNestedParensAndLittleElse() { + tester.assertParsed("OR (OR a b) (OR c d)", "((a b) (c d))", Query.Type.ALL); + } + + // This is simply to control it doesn't crash + @Test + public void testNestedParensAndLittleElseMoreBroken() { + tester.assertParsed("AND (OR a b) (OR c d)", "(a b) +(c d))", Query.Type.ALL); + } + + @Test + public void testNestedUnbalancedParensAndLittleElseMoreBroken() { + tester.assertParsed("OR (OR a b) c d", "((a b) +(c d)", Query.Type.ALL); + } + + @Test + public void testUnbalancedParens() { + tester.assertParsed("AND a b (OR c d)", "a b) +(c d))", Query.Type.ALL); + } + + @Test + public void testUnbalancedStartingParens() { + tester.assertParsed("OR (OR a b) c d", "((a b) +(c d", Query.Type.ALL); + } + + @Test + public void testJPMobileExceptionQuery() { + tester.assertParsed("OR (OR concat and) (OR \"make string\" 1 47) or", + "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); + } + + @Test + public void testColonDoesNotBindAcrossSpace() { + tester.assertParsed("b", "a: b", Query.Type.ALL); + tester.assertParsed("AND a b", "a : b", Query.Type.ALL); + tester.assertParsed("AND a b", "a :b", Query.Type.ALL); + tester.assertParsed("\"a b\"", "a.:b", Query.Type.ALL); + tester.assertParsed("a:b", "a:b", Query.Type.ALL); + } + + @Test + public void testGermanUriDecompounding() { + tester.assertParsed("url.all:\"kommunikationsfehler de\"", + "url.all:kommunikationsfehler.de", "", Query.Type.ALL, Language.GERMAN); + } + + // Check the parser doesn't fail on these horrible query strings + @Test + public void testTicket443882() { + tester.assertParsed( + "AND australian LOTTERY (+(OR language:en (OR IN AFFILIATION WITH THE UK NATIONAL LOTTERY) date:>1125475200) -newstype:rssexclude)", + "australian LOTTERY (IN AFFILIATION WITH THE UK NATIONAL LOTTERY -newstype:rssexclude date:>1125475200 (language:en )", + Query.Type.ALL); + tester.assertParsed( + "AND AND consulting (+(OR language:en (OR albuquerque \"new mexico\") date:>1125475200) -newstype:rssexclude)", + ") AND (consulting) ((albuquerque \"new mexico\" ) -newstype:rssexclude date:>1125475200 (language:en )", + Query.Type.ALL); + tester.assertParsed( + "AND the church of Jesus Christ of latter Day Saints (+(OR language:en (OR Mormon temples) date:>1125475200) -newstype:rssexclude)", + "the church of Jesus Christ of latter Day Saints (Mormon temples -newstype:rssexclude date:>1125475200 (language:en )", + Query.Type.ALL); + } + + // Ticket 523571 + @Test + public void testParensInQuotes() { + tester.assertParsed("AND ringtone (OR a:\"Delivery SMAF large max 150kB 063\" a:\"RealMusic Delivery\")", + "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", + Query.Type.ADVANCED); + tester.assertParsed("AND ringtone AND (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", + "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", + Query.Type.ALL); + // The last one here is a little weird, but it's not a problem, + // so I let it pass for now... + tester.assertParsed("OR (OR ringtone AND) (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", + "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", + Query.Type.ANY); + } + + @Test + public void testMixedCaseIndexNames() { + tester.assertParsed("AND mixedCase:a mixedCase:b \"notAnIndex c\" mixedCase:d", + "mixedcase:a MIXEDCASE:b notAnIndex:c mixedCase:d", + Query.Type.ALL); + } + + /** CJK special tokens should be recognized also on non-boundaries */ + @Test + public void testChineseSpecialTokens() { + tester.assertParsed("AND \"cat tcp/ip zu\" \"foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz\"", + "cattcp/ipzu foo.netbar.net.netC#c++bar.net.netwiz","",Query.Type.ALL,Language.CHINESE_SIMPLIFIED); + } + + /** + * If a cjk special token replace is multi-segment, that token should perhaps be segmented + * but right now it is not + */ + @Test + public void testChineseSpecialTokensWithMultiSegmentReplace() { + // special-token-fs is a special token, to be replaced by firstsecond, first and second are segments in test + tester.assertParsed("AND \"tcp/ip firstsecond dotnet\" firstsecond 'first second'","tcp/ipspecial-token-fs.net special-token-fs firstsecond", + "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, TestLinguistics.INSTANCE); + } + + @Test + public void testSpaceAndTermWeights() { + tester.assertParsed("AND yahoo!360 yahoo 360 yahoo!150 360 yahoo 360 yahoo yahoo!150 yahoo!200", + "yahoo!360 yahoo !360 yahoo! 360 yahoo ! 360 yahoo !!! yahoo! ! yahoo!!", Query.Type.ALL); + } + + @Test + public void testNumbersAndNot() { + tester.assertParsed("+a -12", "a -12", Query.Type.ALL); + } + + @Test + public void testNegativeNumberWithIndex() { + tester.assertParsed("normal:-12", "normal:-12", Query.Type.ALL); + } + + @Test + public void testSingleNegativeNumberLikeTerm() { + tester.assertParsed(null, "-12", Query.Type.ALL); + } + + @Test + public void testNegativeLessThan() { + tester.assertParsed("normal:<-3", "normal:<-3", Query.Type.ALL); + tester.assertParsed("<-3", "<-3", Query.Type.ALL); + } + + @Test + public void testNegativeBiggerThan() { + tester.assertParsed("normal:>-3", "normal:>-3", Query.Type.ALL); + tester.assertParsed(">-3", ">-3", Query.Type.ALL); + } + + @Test + public void testNegativesInRanges() { + tester.assertParsed("normal:[-4;9]", "normal:[-4;9]", Query.Type.ALL); + tester.assertParsed("[-4;9]", "[-4;9]", Query.Type.ALL); + tester.assertParsed("normal:[-4;-9]", "normal:[-4;-9]", Query.Type.ALL); + tester.assertParsed("[-4;-9]", "[-4;-9]", Query.Type.ALL); + } + + @Test + public void testDecimal() { + Item root=tester.assertParsed("2.2", "2.2", Query.Type.ALL); + assertTrue(root instanceof IntItem); + tester.assertParsed("normal:2.2", "normal:2.2", Query.Type.ALL); + } + + @Test + public void testVersionNumbers() { + tester.assertParsed("\"1 0 9\"", "1.0.9", Query.Type.ALL); + } + + @Test + public void testDecimalNumbersAndNot() { + tester.assertParsed("+a -12.2", "a -12.2", Query.Type.ALL); + } + + @Test + public void testNegativeDecimalNumberWithIndex() { + tester.assertParsed("normal:-12.2", "normal:-12.2", Query.Type.ALL); + } + + @Test + public void testSingleNegativeDecimalNumberLikeTerm() { + tester.assertParsed(null, "-12.2", Query.Type.ALL); + } + + @Test + public void testNegativeDecimalLessThan() { + tester.assertParsed("normal:<-3.14", "normal:<-3.14", Query.Type.ALL); + tester.assertParsed("<-3.14", "<-3.14", Query.Type.ALL); + } + + @Test + public void testNegativeDecimalBiggerThan() { + tester.assertParsed("normal:>-3.14", "normal:>-3.14", Query.Type.ALL); + tester.assertParsed(">-3.14", ">-3.14", Query.Type.ALL); + } + + @Test + public void testNegativesDecimalInRanges() { + tester.assertParsed("normal:[-4.16;9.2]", "normal:[-4.16;9.2]", Query.Type.ALL); + tester.assertParsed("[-4.16;9.2]", "[-4.16;9.2]", Query.Type.ALL); + tester.assertParsed("normal:[-4.16;-9.2]", "normal:[-4.16;-9.2]", Query.Type.ALL); + tester.assertParsed("[-4.16;-9.2]", "[-4.16;-9.2]", Query.Type.ALL); + } + + @Test + public void testRangesAndNoise() { + tester.assertParsed("[2;3]", "[2;3]]", Query.Type.ALL); + } + + @Test + public void testIndexNoise() { + tester.assertParsed("AND normal:a notanindex", "normal:a normal: notanindex:", Query.Type.ALL); + tester.assertParsed(null, "normal:", Query.Type.ALL); + tester.assertParsed(null, "normal:!", Query.Type.ALL); + tester.assertParsed(null, "normal::", Query.Type.ALL); + tester.assertParsed(null, "normal:_", Query.Type.ALL); + tester.assertParsed(null, "normal:", Query.Type.ANY); + tester.assertParsed(null, "normal:", Query.Type.ADVANCED); + } + + @Test + public void testIndexNoiseAndExplicitPhrases() { + tester.assertParsed("normal:\"a b\"", "normal:\" a b\"", Query.Type.ALL); + tester.assertParsed("normal:\"a b\"", "normal:\"a b\"", Query.Type.ALL); + } + + @Test + public void testExactMatchParsing1() { + IndexFacts indexFacts = ParsingTester.createIndexFacts(); + Index index1=new Index("testexact1"); + index1.setExact(true,null); + Index index2=new Index("testexact2"); + index2.setExact(true,"()/aa*::*&"); + indexFacts.addIndex("testsd",index1); + indexFacts.addIndex("testsd",index2); + ParsingTester customTester = new ParsingTester(indexFacts); + + customTester.assertParsed("testexact1:/,%&#", "testexact1:/,%&#", Query.Type.ALL); + customTester.assertParsed("testexact2:/,%&#!!", "testexact2:/,%&#!!()/aa*::*&", Query.Type.ALL); + customTester.assertParsed("AND word1 (OR testexact1:word2 testexact1:word3)","word1 AND (testexact1:word2 OR testexact1:word3 )",Query.Type.ADVANCED); + customTester.assertParsed("AND word (OR testexact1:AND testexact1:OR)","word AND (testexact1: AND OR testexact1: OR )",Query.Type.ADVANCED); + } + + /** Testing terminators containing control characters in conjunction with those control characters */ + @Test + public void testExactMatchParsing2() { + IndexFacts indexFacts = ParsingTester.createIndexFacts(); + Index index1=new Index("testexact1"); + index1.setExact(true,"*!*"); + indexFacts.addIndex("testsd",index1); + ParsingTester customTester = new ParsingTester(indexFacts); + + customTester.assertParsed("testexact1:_-_*!200","testexact1:_-_*!**!!",Query.Type.ALL); + } + + /** Testing terminators containing control characters in conjunction with those control characters */ + @Test + public void testExactMatchParsing3() { + IndexFacts indexFacts = ParsingTester.createIndexFacts(); + Index index1=new Index("testexact1"); + index1.setExact(true,"*"); + indexFacts.addIndex("testsd",index1); + ParsingTester customTester = new ParsingTester(indexFacts); + + customTester.assertParsed("testexact1:_-_*!200","testexact1:_-_**!!",Query.Type.ALL); + } + + // bug 1393139 + @Test + public void testMinimalBritneyFilter() { + tester.assertParsed("RANK (+a -|c) b", "a RANK b", "-c", Query.Type.ADVANCED); + } + + @Test + public void testBritneyFilter() { + tester.assertParsed("RANK (+(AND performernameall:britney performernameall:spears songnameall:toxic |SongConsumable:1 |country:us |collapsedrecord:1 |doctype:song) -|collapsecount:0) (AND metadata:britney metadata:spears metadata:toxic)", + "((performernameall:britney AND performernameall:spears AND songnameall:toxic) RANK (metadata:britney AND metadata:spears AND metadata:toxic))", + "+SongConsumable:1 +country:us +collapsedrecord:1 -collapsecount:0 +doctype:song", + Query.Type.ADVANCED); + + tester.assertParsed("AND (+(AND (RANK (OR (AND performernameall:britney performernameall:spears songnameall:toxic) (AND performernameall:britney performernameall:spears songnameall:toxic)) (AND metadata:britney metadata:spears metadata:toxic)) |SongConsumable:1 |country:us |collapsedrecord:1) -|collapsecount:0) |doctype:song", + "(((performernameall:britney AND performernameall:spears AND songnameall:toxic) OR (performernameall:britney AND performernameall:spears AND songnameall:toxic)) RANK (metadata:britney AND metadata:spears AND metadata:toxic)))", + "+SongConsumable:1 +country:us +collapsedrecord:1 -collapsecount:0 +doctype:song", + Query.Type.ADVANCED); + } + + // bug 1412840 + @Test + public void testGreedyPhrases() { + tester.assertParsed("AND title:why title:\"1 2\"", "title:\"why\" title:\"&\" title:\"1/2\"", Query.Type.ALL); + } + + // bug 1509347 + @Test + public void testUnderscoreInFieldNames() { + tester.assertParsed("AND title:why score_under:what score_under:>5000", + "title:why score_under:what score_under:>5000", + Query.Type.ALL); + } + + // bug 1509347 + @Test + public void testLeadingUnderscoreInFieldNames() { + tester.assertParsed("AND title:why _under_score_:what _under_score_:>5000", + "title:why _under_score_:what _under_score_:>5000", + Query.Type.ALL); + } + + // Re-add if underscore should be a word character + // @Test + // public void testUnderscoreAsWordCharacter() { + // tester.assertParsed("AND _a b_ a__b \"_a_b_c _d_e_f\"", + // "_a b_ a__b \"_a_b_c _d_e_f\"", + // Query.Type.ALL); + // } + + // Re-add if underscore should be a word character + // @Test + // public void testUnderscoreAsWordWithIndexName() { + // tester.assertParsed("AND title:_a title:a title:_a_", + // "title:_a title:a title:_a_", + // Query.Type.ALL); + // } + + // bug 524918 + @Test + public void testAdvancedSyntaxParensAndQuotes() { + tester.assertParsed("OR a (AND \"b c d\" e)", + "a OR (\"b (c) d\" AND e)", + Query.Type.ADVANCED); + } + + // bug 2530430 + // This test is here instead of in the query parser because + // this needs to become series of tests where the tokenizer + // and parser will step on each other's toes. + @Test + public void testStarFirstInAttributes() { + tester.assertParsed("exactindex:*test", + "exactindex:*test", + Query.Type.ALL); + + } + + @Test + public void testOneWordWebParsing() { + tester.assertParsed("a","a",Query.Type.WEB); + } + + @Test + public void testTwoWordWebParsing() { + tester.assertParsed("AND a b","a b",Query.Type.WEB); + } + + @Test + public void testPlusWordWebParsing1() { + Item root=tester.assertParsed("AND a b","+a b",Query.Type.WEB); + assertTrue(((AndItem)root).getItem(0).isProtected()); + assertFalse(((AndItem)root).getItem(1).isProtected()); + } + + @Test + public void testPlusWordWebParsing2() { + Item root=tester.assertParsed("AND a b","+a +b",Query.Type.WEB); + assertTrue(((AndItem)root).getItem(0).isProtected()); + assertTrue(((AndItem)root).getItem(1).isProtected()); + } + + @Test + public void testNegativeWordsParsing1() { + Item root=tester.assertParsed("+a -b","a -b",Query.Type.WEB); + assertFalse(((NotItem)root).getItem(0).isProtected()); + assertTrue(((NotItem)root).getItem(1).isProtected()); + } + + @Test + public void testNegativeWordsParsing2() { + Item root=tester.assertParsed("+a -b","+a -b",Query.Type.WEB); + assertTrue(((NotItem)root).getItem(0).isProtected()); + assertTrue(((NotItem)root).getItem(1).isProtected()); + } + + @Test + public void testNegativeWordsParsing3() { + tester.assertParsed("+a -b","-b a",Query.Type.WEB); + } + + @Test + public void testNegativeWordsParsing4() { + tester.assertParsed("+(AND a b) -c -d","a b -c -d",Query.Type.WEB); + } + + @Test + public void testNegativeWordsParsing5() { + tester.assertParsed("+(AND a \"b c\" d) -e -f","a -e \"b c\" d -f",Query.Type.WEB); + } + + @Test + public void testPhraseWebParsing() { + tester.assertParsed("\"a b\"","\"a b\"",Query.Type.WEB); + } + + @Test + public void testPhraseAndExtraTermWebParsing() { + tester.assertParsed("AND \"a b\" c","\"a b\" c",Query.Type.WEB); + } + + @Test + public void testNotOrWebParsing() { + tester.assertParsed("AND a or b","a or b",Query.Type.WEB); + } + + @Test + public void testNotOrALLParsing() { + tester.assertParsed("AND a OR b","a OR b",Query.Type.ALL); + } + + @Test + public void testOrParsing1() { + tester.assertParsed("OR a b","a OR b",Query.Type.WEB); + } + + @Test + public void testOrParsing2() { + tester.assertParsed("OR a b c","a OR b OR c",Query.Type.WEB); + } + + @Test + public void testOrParsing3() { + tester.assertParsed("OR a (AND b c) \"d e\"","a OR b c OR \"d e\"",Query.Type.WEB); + } + + @Test + public void testOrParsing4() { + tester.assertParsed("OR (AND or1 a) or2","or1 a OR or2",Query.Type.WEB); + } + + @Test + public void testOrCornerCase1() { + tester.assertParsed("AND OR a","OR a",Query.Type.WEB); + } + + @Test + public void testOrCornerCase2() { + tester.assertParsed("AND OR a","OR a OR",Query.Type.WEB); // Don't care + } + + @Test + public void testOrCornerCase3() { + tester.assertParsed("AND OR a","OR a OR OR",Query.Type.WEB); // Don't care + } + + @Test + public void testOrCornerCase4() { + tester.assertParsed("+(OR (AND a b) (AND d c) (AND g h)) -e -f","a b OR d c -e -f OR g h",Query.Type.WEB); + } + + @Test + public void testOdd1Web() { + tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.WEB); + } + + @Test + public void testNotOnlyWeb() { + tester.assertParsed(null, "-foobar", Query.Type.WEB); + } + + @Test + public void testMultipleNotsOnltWeb() { + tester.assertParsed(null, "-foo -bar -foobar", Query.Type.WEB); + } + + @Test + public void testOnlyNotCompositeWeb() { + tester.assertParsed(null, "-(foo bar baz)", Query.Type.WEB); + } + + @Test + public void testSingleNegativeNumberLikeTermWeb() { + tester.assertParsed(null, "-12", Query.Type.WEB); + } + + @Test + public void testSingleNegativeDecimalNumberLikeTermWeb() { + tester.assertParsed(null, "-12.2", Query.Type.WEB); + } + + /** These additions should be done by the YstSearcher */ + @Test + public void testDefaultWebIndices() { + tester.assertParsed("\"notanindex b\"","notanindex:b",Query.Type.WEB); + tester.assertParsed("site:\"b $\"","site:b",Query.Type.WEB); + tester.assertParsed("hostname:b","hostname:b",Query.Type.WEB); + tester.assertParsed("link:b","link:b",Query.Type.WEB); + tester.assertParsed("url:b","url:b",Query.Type.WEB); + tester.assertParsed("inurl:b","inurl:b",Query.Type.WEB); + tester.assertParsed("intitle:b","intitle:b",Query.Type.WEB); + } + + @Test + public void testHTMLWeb() { + tester.assertParsed("AND h2 Title h2","

Title

",Query.Type.WEB); + } + + /** + * Shortcut terms are represented as any other terms, but can be rewritten downstream. + * The information about added bangs is available from the origin as shown (do not use the weight to find this) + */ + @Test + public void testShortcutsWeb() { + tester.assertParsed("AND map new york","map new york",Query.Type.WEB); + + AndItem root=(AndItem)tester.assertParsed("AND map!150 new york","map! new york",Query.Type.WEB); + assertEquals('!',((WordItem)root.getItem(0)).getOrigin().charAfter(0)); + + root=(AndItem)tester.assertParsed("AND barack obama news!150","barack obama news!",Query.Type.WEB); + assertEquals('!',((WordItem)root.getItem(2)).getOrigin().charAfter(0)); + } + + @Test + public void testZipCodeShortcutWeb() { + tester.assertParsed("12345","12345",Query.Type.WEB); + IntItem root=(IntItem)tester.assertParsed("00012!150","00012!",Query.Type.WEB); + assertEquals('!',root.getOrigin().charAfter(0)); + } + + @Test + public void testDouble() { + Item number=tester.assertParsed("123456789.987654321","123456789.987654321",Query.Type.ALL); + assertTrue(number instanceof IntItem); + } + + @Test + public void testLong() { + Item number=tester.assertParsed("3000000000000","3000000000000",Query.Type.ALL); + assertTrue(number instanceof IntItem); + } + + @Test + public void testNear1() { + tester.assertParsed("NEAR(2) new york","new NEAR york",Query.Type.ADVANCED); + } + + @Test + public void testNear2() { + tester.assertParsed("ONEAR(2) new york","new ONEAR york",Query.Type.ADVANCED); + } + + @Test + public void testNear3() { + tester.assertParsed("NEAR(3) new york","new NEAR(3) york",Query.Type.ADVANCED); + } + + @Test + public void testNear4() { + tester.assertParsed("ONEAR(3) new york","new ONEAR(3) york",Query.Type.ADVANCED); + } + + @Test + public void testNear5() { + tester.assertParsed("NEAR(3) map new york","map NEAR(3) new NEAR(3) york",Query.Type.ADVANCED); + } + + @Test + public void testNear6() { + tester.assertParsed("ONEAR(3) map new york","map ONEAR(3) new ONEAR(3) york",Query.Type.ADVANCED); + } + + @Test + public void testNear7() { + tester.assertParsed("NEAR(4) (NEAR(3) map new) york","map NEAR(3) new NEAR(4) york",Query.Type.ADVANCED); + } + + @Test + public void testNear8() { + tester.assertParsed("ONEAR(4) (ONEAR(3) map new) york","map ONEAR(3) new ONEAR(4) york",Query.Type.ADVANCED); + } + + @Test + public void testNearPrefix() { + tester.assertParsed("NEAR(2) a b*","a NEAR b*",Query.Type.ADVANCED); + } + + // bug 3672512 + // This is make sure these doesn't fail, not that they're parsed very nicely + @Test + public void testNestedBracesAndPhrases() { + String userQuery = "(\"Secondary Curriculum\" (\"Key Stage 3\" OR KS3) (\"Key Stage 4\" OR KS4)) "; + tester.assertParsed( + "OR (OR \"Secondary Curriculum\" (OR \"Key Stage 3\" OR KS3)) (OR \"Key Stage 4\" OR KS4)", + userQuery, Query.Type.ALL); + userQuery = "(\"Grande distribution\" (\"developpement durable\" OR \"commerce equitable\"))"; + tester.assertParsed( + "OR \"Grande distribution\" (OR \"developpement durable\" OR \"commerce equitable\")", + userQuery, Query.Type.ALL); + userQuery = "(\"road tunnel\" (\"tunnel management\" OR AID OR \"traffic systems\" OR supervision OR " + + "\"decision aid system\") (Spie OR Telegra OR Telvent OR Steria)) "; + tester.assertParsed( + "OR (OR \"road tunnel\" (OR \"tunnel management\" OR AID OR \"traffic systems\" OR supervision OR \"decision aid system\")) (OR Spie OR Telegra OR Telvent OR Steria)", + userQuery, Query.Type.ALL); + } + + // bug 3726354 + @Test + public void testYetAnotherCycleQuery() { + tester.assertParsed("+(OR (+d -f) b) -c", + "( b -c ( d -f )", + Query.Type.ALL); + } + + @Test + public void testSimpleEquivAdvanced() { + tester.assertParsed("EQUIV foo bar baz", "foo equiv bar equiv baz", Query.Type.ADVANCED); + } + + @Test + public void testEquivWordIntPhraseAdvanced() { + tester.assertParsed("EQUIV foo 5 \"a b\"", "foo equiv 5 equiv \"a b\"", Query.Type.ADVANCED); + } + + @Test + public void testEquivRejectCompositeAdvanced() { + try { + tester.assertParsed("this should not parse", "foo equiv (a or b)", Query.Type.ADVANCED); + } catch(Exception e) { + // Success + } + } + + @Test + public void testSimpleWandAdvanced() { + tester.assertParsed("WAND(100) foo bar baz", "foo wand bar wand baz", Query.Type.ADVANCED); + } + + @Test + public void testSimpleWandAdvancedWithNonDefaultN() { + tester.assertParsed("WAND(32) foo bar baz", "foo wand(32) bar wand(32) baz", Query.Type.ADVANCED); + } + + @Test + public void testSimpleWandAdvancedWithNonDefaultNAndWeights() { + tester.assertParsed("WAND(32) foo!32 bar!64 baz", "foo!32 wand(32) bar!64 wand(32) baz", Query.Type.ADVANCED); + } + + @Test + public void testTwoRanges() { + tester.assertParsed("AND score:[1.25;2.18] age:[25;30]","score:[1.25;2.18] AND age:[25;30]",Query.Type.ADVANCED); + } + + @Test + public void testTooLargeTermWeights() { + // This behavior is a bit silly: + tester.assertParsed("AND a 12345678901234567890", "a!12345678901234567890", Query.Type.ALL); + // but in light of + tester.assertParsed("AND a!150 b", "a!b", Query.Type.ALL); + // which was the behavior when adding handling of too large term + // weights, it is at least consistent. It should probably be implicit + // phrases instead. + } + + @Test + public void testSiteAndSegmentPhrases() { + tester.assertParsed("host.all:\"www abc com x y-z $\"", + "host.all:www.abc.com/x'y-z", "", + Query.Type.ALL, Language.ENGLISH); + } + + @Test + public void testSiteAndSegmentPhrasesFollowedByText() { + tester.assertParsed("AND host.all:\"www abc com x y-z $\" 'a b'", + "host.all:www.abc.com/x'y-z a'b", "", + Query.Type.ALL, Language.ENGLISH); + } + + @Test + public void testIntItemFollowedByDot() { + tester.assertParsed("AND Campion Ste 3 When To Her Lute Corinna Sings","Campion Ste: 3. When To Her Lute Corinna Sings", Query.Type.ALL); + } + + @Test + public void testNotIntItemIfPrecededByHyphen() { + tester.assertParsed("AND Horny Horny '98 Radio Edit","Horny [Horny '98 Radio Edit]]", Query.Type.ALL); + } + + @Test + public void testNonAsciiNumber() { + tester.assertParsed("title:\"199 119 201 149\"", "title:１９９．１１９．２ï¼ï¼‘．１４９", Query.Type.ALL); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java new file mode 100644 index 00000000000..e05f5c9db59 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser.test; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleDetector; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * A utility for writing parser tests + * + * @author bratseth + */ +public class ParsingTester { + + private static final Linguistics linguistics = new SimpleLinguistics(); + private IndexFacts indexFacts; + private SpecialTokenRegistry tokenRegistry; + + public ParsingTester() { + this(createIndexFacts(), createSpecialTokens()); + } + + public ParsingTester(SpecialTokens specialTokens) { + this(createIndexFacts(), specialTokens); + } + + public ParsingTester(IndexFacts indexFacts) { + this(indexFacts, createSpecialTokens()); + } + + public ParsingTester(IndexFacts indexFacts, SpecialTokens specialTokens) { + indexFacts.freeze(); + specialTokens.freeze(); + + this.indexFacts = indexFacts; + tokenRegistry = new SpecialTokenRegistry(); + tokenRegistry.addSpecialTokens(specialTokens); + } + + /** + * Returns an unfrozen version of the IndexFacts this will use. + * This can be used to add new indexes and passing the resulting IndexFacts to the constructor of this. + */ + public static IndexFacts createIndexFacts() { + String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg"; + ConfigGetter getter = new ConfigGetter<>(IndexInfoConfig.class); + IndexInfoConfig config = getter.getConfig(indexInfoConfigID); + return new IndexFacts(new IndexModel(config, (QrSearchersConfig)null)); + } + + /** + * Returns an unfrozen version of the special tokens this will use. + * This can be used to add new tokens and passing the resulting special tokens to the constructor of this. + */ + public static SpecialTokens createSpecialTokens() { + SpecialTokens tokens = new SpecialTokens("default"); + tokens.addSpecialToken("c++", null); + tokens.addSpecialToken(".net", "dotnet"); + tokens.addSpecialToken("tcp/ip", null); + tokens.addSpecialToken("c#", null); + tokens.addSpecialToken("special-token-fs","firstsecond"); + return tokens; + } + + /** + * Asserts that the canonical representation of the second string when parsed + * is the first string + * + * @return the produced root + */ + public Item assertParsed(String parsed, String toParse, Query.Type mode) { + return assertParsed(parsed, toParse, null, mode, new SimpleDetector().detect(toParse, null).getLanguage(), + new SimpleLinguistics()); + } + + /** + * Asserts that the canonical representation of the second string when parsed + * is the first string + * + * @return the produced root + */ + public Item assertParsed(String parsed, String toParse, String filter, Query.Type mode) { + return assertParsed(parsed, toParse, filter, mode, new SimpleDetector().detect(toParse,null).getLanguage()); + } + + public Item assertParsed(String parsed, String toParse, String filter, Query.Type mode, Language language) { + return assertParsed(parsed, toParse, filter, mode, language, linguistics); + } + + /** + * Asserts that the canonical representation of the second string when parsed + * is the first string + * + * @return the produced root + */ + public Item assertParsed(String parsed, String toParse, String filter, Query.Type mode, + Language language, Linguistics linguistics) { + Item root = parseQuery(toParse, filter, language, mode, linguistics); + if (parsed == null) { + assertTrue("root should be null, but was " + root, root == null); + } else { + assertNotNull("Got null from parsing " + toParse, root); + assertEquals("Parse of '" + toParse + "'", parsed, root.toString()); + } + return root; + } + + public Item parseQuery(String query, String filter, Language language, Query.Type type, Linguistics linguistics) { + Parser parser = ParserFactory.newInstance(type, new ParserEnvironment() + .setIndexFacts(indexFacts) + .setLinguistics(linguistics) + .setSpecialTokens(tokenRegistry.getSpecialTokens("default"))); + Item root = parser.parse(new Parsable().setQuery(query).setFilter(filter).setLanguage(language)).getRoot(); + if (root instanceof NullItem) { + return null; + } + return root; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/SubstringTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/SubstringTestCase.java new file mode 100644 index 00000000000..d2c5c84475d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/SubstringTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser.test; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +/** + * Check Substring in conjunction with query tokenization and parsing behaves properly. + * + * @author Steinar Knutsen + */ +public class SubstringTestCase { + + @Test + public final void testTokenLengthAndLowercasing() { + Query q = new Query("/?query=\u0130"); + WordItem root = (WordItem) q.getModel().getQueryTree().getRoot(); + assertEquals("\u0130", root.getRawWord()); + } + + + @Test + public final void testBug5968479() { + String first = "\u0130\u015EBANKASI"; + String second = "GAZ\u0130EM\u0130R"; + Query q = new Query("/?query=" + enc(first) + "%20" + enc(second)); + CompositeItem root = (CompositeItem) q.getModel().getQueryTree().getRoot(); + assertEquals(first, ((WordItem) root.getItem(0)).getRawWord()); + assertEquals(second, ((WordItem) root.getItem(1)).getRawWord()); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java new file mode 100644 index 00000000000..5df2572242e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -0,0 +1,765 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser.test; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.prelude.query.parser.Token; +import com.yahoo.prelude.query.parser.Tokenizer; + +import java.util.Collections; +import java.util.List; + +import static com.yahoo.prelude.query.parser.Token.Kind.*; + +/** + * Tests the tokenizer + * + * @author Jon S Bratseth + */ +public class TokenizerTestCase extends junit.framework.TestCase { + + private SpecialTokenRegistry defaultRegistry = new SpecialTokenRegistry("file:src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg"); + + public void testPlainTokenization() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(createSpecialTokens()); + List tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1"); + + assertEquals(new Token(WORD, "drive"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(LBRACE, "("), tokens.get(2)); + assertEquals(new Token(WORD, "to"), tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + assertEquals(new Token(WORD, "hwy88"), tokens.get(5)); + assertEquals(new Token(COMMA, ","), tokens.get(6)); + assertEquals(new Token(SPACE, " "), tokens.get(7)); + assertEquals(new Token(NUMBER, "88"), tokens.get(8)); + assertEquals(new Token(RBRACE, ")"), tokens.get(9)); + assertEquals(new Token(SPACE, " "), tokens.get(10)); + assertEquals(new Token(PLUS, "+"), tokens.get(11)); + assertEquals(new Token(WORD, "or"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "language"), tokens.get(14)); + assertEquals(new Token(COLON, ":"), tokens.get(15)); + assertEquals(new Token(WORD, "en"), tokens.get(16)); + assertEquals(new Token(SPACE, " "), tokens.get(17)); + assertEquals(new Token(WORD, "ugcapi"), tokens.get(18)); + assertEquals(new Token(UNDERSCORE, "_"), tokens.get(19)); + assertEquals(new Token(NUMBER, "1"), tokens.get(20)); + } + + public void testOutsideBMPCodepoints() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + List tokens = tokenizer.tokenize("\ud841\udd47"); + assertEquals(new Token(WORD, "\ud841\udd47"), tokens.get(0)); + } + + public void testOneSpecialToken() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(createSpecialTokens()); + List tokens = tokenizer.tokenize("c++ lovers, please apply"); + + assertEquals(new Token(WORD, "c++"), tokens.get(0)); + } + + public void testSpecialTokenCombination() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(createSpecialTokens()); + List tokens = tokenizer.tokenize("c#, c++ or .net know, not tcp/ip"); + + assertEquals(new Token(WORD, "c#"), tokens.get(0)); + assertEquals(new Token(COMMA, ","), tokens.get(1)); + assertEquals(new Token(SPACE, " "), tokens.get(2)); + assertEquals(new Token(WORD, "c++"), tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + assertEquals(new Token(WORD, "or"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, ".net"), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "know"), tokens.get(9)); + assertEquals(new Token(COMMA, ","), tokens.get(10)); + assertEquals(new Token(SPACE, " "), tokens.get(11)); + assertEquals(new Token(WORD, "not"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "tcp/ip"), tokens.get(14)); + } + + /** + * In cjk languages, special tokens must be recognized as substrings of strings not + * separated by space, as special token recognition happens before tokenization + */ + public void testSpecialTokenCJK() { + assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size()); + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + tokenizer.setSubstringSpecialTokens(true); + tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default")); + + List tokens = tokenizer.tokenize("fooc#bar,c++with spacebarknowknowknow,knowknownot know"); + assertEquals(new Token(WORD, "foo"), tokens.get(0)); + assertEquals(new Token(WORD, "c#"), tokens.get(1)); + assertEquals(new Token(WORD, "bar"), tokens.get(2)); + assertEquals(new Token(COMMA, ","), tokens.get(3)); + assertEquals(new Token(WORD, "cpp"), tokens.get(4)); + assertEquals(new Token(WORD, "with-space"), tokens.get(5)); + assertEquals(new Token(WORD, "bar"), tokens.get(6)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(7)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(8)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(9)); + assertEquals(new Token(COMMA, ","), tokens.get(10)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(11)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(12)); + assertEquals(new Token(WORD, "not"), tokens.get(13)); + assertEquals(new Token(SPACE, " "), tokens.get(14)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(15)); + } + + public void testSpecialTokenCaseInsensitive() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(createSpecialTokens()); + List tokens = tokenizer.tokenize("The AS/400 is great"); + + assertEquals(new Token(WORD, "The"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "as/400"), tokens.get(2)); + assertEquals(new Token(SPACE, " "), tokens.get(3)); + assertEquals(new Token(WORD, "is"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(WORD, "great"), tokens.get(6)); + } + + public void testSpecialTokenNonMatch() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(createSpecialTokens()); + List tokens = tokenizer.tokenize("c++ c+ aS/400 i/o .net i/ooo ap.net"); + + assertEquals(new Token(WORD, "c++"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "c+"), tokens.get(2)); + assertEquals(new Token(SPACE, " "), tokens.get(3)); + assertEquals(new Token(WORD, "as/400"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(WORD, "i/o"), tokens.get(6)); + assertEquals(new Token(SPACE, " "), tokens.get(7)); + assertEquals(new Token(WORD, ".net"), tokens.get(8)); + assertEquals(new Token(SPACE, " "), tokens.get(9)); + assertEquals(new Token(WORD, "i"), tokens.get(10)); + assertEquals(new Token(NOISE, ""), tokens.get(11)); + assertEquals(new Token(WORD, "ooo"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "ap"), tokens.get(14)); + assertEquals(new Token(WORD, ".net"), tokens.get(15)); + } + + // Re-add if underscore becomes some sort of word character again + // public void testUnderscores() { + // Tokenizer tokenizer = new Tokenizer(linguistics); + // List tokens = tokenizer.tokenize("_a __a ___a ____a"); + // assertEquals(": _a, \" \": , : __a, \" \": , : ___a, \" \": , : ____, : a, : ", + // Tokenizer.formatTokenList(tokens)); + // tokens = tokenizer.tokenize("a_b a__b a___b a____b"); + // assertEquals(": a_b, \" \": , : a__b, \" \": , : a___b, \" \": , : a, : ____, : b, : ", + // Tokenizer.formatTokenList(tokens)); + // tokens = tokenizer.tokenize("a_ a__ a___ a____"); + // assertEquals(": a_, \" \": , : a__, \" \": , : a___, \" \": , : a, : ____, : ", + // Tokenizer.formatTokenList(tokens)); + // tokens = tokenizer.tokenize("_a_ __a__ ___a___ ____a____"); + // assertEquals(": _a_, \" \": , : __a__, \" \": , : ___a___, \" \": , : ____, : a, : ____, : ", + // Tokenizer.formatTokenList(tokens)); + // tokens = tokenizer.tokenize("____a___ ___a____"); + // assertEquals(": ____, : a___, \" \": , : ___a, : ____, : ", + // Tokenizer.formatTokenList(tokens)); + // tokens = tokenizer.tokenize("_ __ ___ ____"); + // assertEquals(": _, \" \": , : __, \" \": , : ___, \" \": , : ____, : ", + // Tokenizer.formatTokenList(tokens)); + // tokens = tokenizer.tokenize("_a_ba__ba____ba____b"); + // assertEquals(": _a_ba__ba, : ____, : ba, : ____, : b, : ", + // Tokenizer.formatTokenList(tokens)); + // SpecialTokenRegistry.set(new SpecialTokenRegistry()); // Reset + // } + + public void testSpecialTokenConfigurationDefault() { + String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg"; + + SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); + assertEquals("Special tokens configured", 6, + r.getSpecialTokens("default").size()); + assertEquals("Special tokens configured", 4, + r.getSpecialTokens("other").size()); + + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens( + r.getSpecialTokens("default")); + List tokens = tokenizer.tokenize( + "with space, c++ or .... know, not b.s.d."); + + assertEquals(new Token(WORD, "with space"), tokens.get(0)); + assertEquals(new Token(COMMA, ","), tokens.get(1)); + assertEquals(new Token(SPACE, " "), tokens.get(2)); + assertEquals(new Token(WORD, "c++"), tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + assertEquals(new Token(WORD, "or"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "...."), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "know"), tokens.get(9)); + assertEquals(new Token(COMMA, ","), tokens.get(10)); + assertEquals(new Token(SPACE, " "), tokens.get(11)); + assertEquals(new Token(WORD, "not"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "b.s.d."), tokens.get(14)); + } + + public void testSpecialTokenConfigurationOther() { + String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg"; + + SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); + assertEquals("Special tokens configured", 6, + r.getSpecialTokens("default").size()); + assertEquals("Special tokens configured", 4, + r.getSpecialTokens("other").size()); + + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens( + r.getSpecialTokens("other")); + List tokens = tokenizer.tokenize( + "with space,!!!*** [huh] or ------ " + "know, &&&%%% b.s.d."); + + assertEquals(new Token(WORD, "with"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "space"), tokens.get(2)); + assertEquals(new Token(COMMA, ","), tokens.get(3)); + assertEquals(new Token(WORD, "!!!***"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(WORD, "[huh]"), tokens.get(6)); + assertEquals(new Token(SPACE, " "), tokens.get(7)); + assertEquals(new Token(WORD, "or"), tokens.get(8)); + assertEquals(new Token(SPACE, " "), tokens.get(9)); + assertEquals(new Token(WORD, "------"), tokens.get(10)); + assertEquals(new Token(SPACE, " "), tokens.get(11)); + assertEquals(new Token(WORD, "know"), tokens.get(12)); + assertEquals(new Token(COMMA, ","), tokens.get(13)); + assertEquals(new Token(SPACE, " "), tokens.get(14)); + assertEquals(new Token(WORD, "&&&%%%"), tokens.get(15)); + assertEquals(new Token(SPACE, " "), tokens.get(16)); + assertEquals(new Token(WORD, "b"), tokens.get(17)); + assertEquals(new Token(DOT, "."), tokens.get(18)); + assertEquals(new Token(WORD, "s"), tokens.get(19)); + assertEquals(new Token(DOT, "."), tokens.get(20)); + assertEquals(new Token(WORD, "d"), tokens.get(21)); + assertEquals(new Token(DOT, "."), tokens.get(22)); + + assertTrue(((Token) tokens.get(10)).isSpecial()); + } + + public void testSpecialTokenConfigurationMissing() { + String tokenFile = "file:source/bogus/specialtokens.cfg"; + + SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile); + + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(r.getSpecialTokens("other")); + List tokens = tokenizer.tokenize("c++"); + + assertEquals(new Token(WORD, "c"), tokens.get(0)); + assertEquals(new Token(PLUS, "+"), tokens.get(1)); + assertEquals(new Token(PLUS, "+"), tokens.get(2)); + } + + public void testTokenReplacing() { + assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size()); + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default")); + + List tokens = tokenizer.tokenize("with space, c++ or .... know, not b.s.d."); + assertEquals(new Token(WORD, "with-space"), tokens.get(0)); + assertEquals(new Token(COMMA, ","), tokens.get(1)); + assertEquals(new Token(SPACE, " "), tokens.get(2)); + assertEquals(new Token(WORD, "cpp"), tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + assertEquals(new Token(WORD, "or"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "...."), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "knuwww"), tokens.get(9)); + assertEquals(new Token(COMMA, ","), tokens.get(10)); + assertEquals(new Token(SPACE, " "), tokens.get(11)); + assertEquals(new Token(WORD, "not"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "b.s.d."), tokens.get(14)); + + assertTrue(((Token) tokens.get(9)).isSpecial()); + assertFalse(((Token) tokens.get(12)).isSpecial()); + } + + public void testExactMatchTokenization() { + Index index1=new Index("testexact1"); + index1.setExact(true,null); + Index index2=new Index("testexact2"); + index2.setExact(true,"()/aa*::*&"); + IndexFacts facts = new IndexFacts(); + facts.addIndex("testsd",index1); + facts.addIndex("testsd",index2); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + Tokenizer tokenizer=new Tokenizer(new SimpleLinguistics()); + List tokens=tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session); + // tokenizer.print(); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "a"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "b"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(LBRACE, "("), tokens.get(6)); + assertEquals(new Token(WORD, "normal"), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(9)); + assertEquals(new Token(COLON, ":"), tokens.get(10)); + assertEquals(new Token(WORD, "/,%#%&+-+"), tokens.get(11)); + assertEquals(new Token(SPACE, " "), tokens.get(12)); + assertEquals(new Token(RBRACE, ")"), tokens.get(13)); + assertEquals(new Token(SPACE, " "), tokens.get(14)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(15)); + assertEquals(new Token(COLON, ":"), tokens.get(16)); + assertEquals(new Token(WORD, "ho_/&%&/"), tokens.get(17)); + assertEquals(new Token(SPACE, " "), tokens.get(18)); + assertEquals(new Token(WORD, "b"), tokens.get(19)); + assertEquals(new Token(COLON, ":"), tokens.get(20)); + assertEquals(new Token(WORD, "c"), tokens.get(21)); + assertTrue(((Token) tokens.get(11)).isSpecial()); + assertFalse(((Token) tokens.get(15)).isSpecial()); + assertTrue(((Token) tokens.get(17)).isSpecial()); + } + + public void testExactMatchTokenizationTerminatorTerminatesQuery() { + Index index1=new Index("testexact1"); + index1.setExact(true,null); + Index index2=new Index("testexact2"); + index2.setExact(true,"()/aa*::*&"); + IndexFacts facts = new IndexFacts(); + facts.addIndex("testsd",index1); + facts.addIndex("testsd",index2); + Tokenizer tokenizer=new Tokenizer(new SimpleLinguistics()); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + List tokens=tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&", session); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "a"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "b"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(LBRACE, "("), tokens.get(6)); + assertEquals(new Token(WORD, "normal"), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(9)); + assertEquals(new Token(COLON, ":"), tokens.get(10)); + assertEquals(new Token(WORD, "/,%#%&+-+"), tokens.get(11)); + assertEquals(new Token(SPACE, " "), tokens.get(12)); + assertEquals(new Token(RBRACE, ")"), tokens.get(13)); + assertEquals(new Token(SPACE, " "), tokens.get(14)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(15)); + assertEquals(new Token(COLON, ":"), tokens.get(16)); + assertEquals(new Token(WORD, "ho_/&%&/"), tokens.get(17)); + assertTrue(((Token) tokens.get(17)).isSpecial()); + } + + public void testExactMatchTokenizationWithTerminatorTerminatedByEndOfString() { + Index index1=new Index("testexact1"); + index1.setExact(true,null); + Index index2=new Index("testexact2"); + index2.setExact(true,"()/aa*::*&"); + IndexFacts facts = new IndexFacts(); + facts.addIndex("testsd",index1); + facts.addIndex("testsd",index2); + Tokenizer tokenizer=new Tokenizer(new SimpleLinguistics()); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + List tokens=tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*", session); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "a"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "b"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(LBRACE, "("), tokens.get(6)); + assertEquals(new Token(WORD, "normal"), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(9)); + assertEquals(new Token(COLON, ":"), tokens.get(10)); + assertEquals(new Token(WORD, "/,%#%&+-+"), tokens.get(11)); + assertEquals(new Token(SPACE, " "), tokens.get(12)); + assertEquals(new Token(RBRACE, ")"), tokens.get(13)); + assertEquals(new Token(SPACE, " "), tokens.get(14)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(15)); + assertEquals(new Token(COLON, ":"), tokens.get(16)); + assertEquals(new Token(WORD, "ho_/&%&/()/aa*::*"), tokens.get(17)); + assertTrue(((Token) tokens.get(17)).isSpecial()); + } + + public void testExactMatchTokenizationEndsByColon() { + Index index1=new Index("testexact1"); + index1.setExact(true,null); + Index index2=new Index("testexact2"); + index2.setExact(true,"()/aa*::*&"); + IndexFacts facts = new IndexFacts(); + facts.addIndex("testsd",index1); + facts.addIndex("testsd",index2); + Tokenizer tokenizer=new Tokenizer(new SimpleLinguistics()); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + List tokens=tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "a"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "b"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(LBRACE, "("), tokens.get(6)); + assertEquals(new Token(WORD, "normal"), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(9)); + assertEquals(new Token(COLON, ":"), tokens.get(10)); + assertEquals(new Token(WORD, "!/%#%&+-+"), tokens.get(11)); + assertEquals(new Token(SPACE, " "), tokens.get(12)); + assertEquals(new Token(RBRACE, ")"), tokens.get(13)); + assertEquals(new Token(SPACE, " "), tokens.get(14)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(15)); + assertEquals(new Token(COLON, ":"), tokens.get(16)); + assertEquals(new Token(WORD, "ho_/&%&/"), tokens.get(17)); + assertEquals(new Token(WORD, "b"), tokens.get(18)); + assertEquals(new Token(COLON, ":"), tokens.get(19)); + } + + public void testExactMatchHeuristics() { + Index index1=new Index("testexact1"); + index1.setExact(true, null); + Index index2=new Index("testexact2"); + index2.setExact(true, "()/aa*::*&"); + IndexFacts indexFacts = new IndexFacts(); + indexFacts.addIndex("testsd", index1); + indexFacts.addIndex("testsd", index2); + IndexFacts.Session facts = indexFacts.newSession(Collections.emptySet(), Collections.emptySet()); + + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + List tokens = tokenizer.tokenize("normal a:b (normal testexact1:foo) testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "a"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "b"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(LBRACE, "("), tokens.get(6)); + assertEquals(new Token(WORD, "normal"), tokens.get(7)); + assertEquals(new Token(SPACE, " "), tokens.get(8)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(9)); + assertEquals(new Token(COLON, ":"), tokens.get(10)); + assertEquals(new Token(WORD, "foo"), tokens.get(11)); + assertEquals(new Token(RBRACE, ")"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(14)); + assertEquals(new Token(COLON, ":"), tokens.get(15)); + assertEquals(new Token(WORD, "bar"), tokens.get(16)); + + tokens = tokenizer.tokenize("testexact1:a*teens", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "a*teens"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1:foo\"bar", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo\"bar"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1:foo!bar", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo!bar"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1:foo! ", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + + tokens = tokenizer.tokenize("testexact1:foo!! ", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(3)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + + tokens = tokenizer.tokenize("testexact1:foo!100 ", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(3)); + assertEquals(new Token(NUMBER, "100"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + + tokens = tokenizer.tokenize("testexact1:foo*!100 ", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + assertEquals(new Token(STAR, "*"), tokens.get(3)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(4)); + assertEquals(new Token(NUMBER, "100"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + + tokens = tokenizer.tokenize("testexact1: *\"foo bar\"*!100 ", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(STAR, "*"), tokens.get(2)); + assertEquals(new Token(WORD, "foo bar"), tokens.get(3)); + assertEquals(new Token(STAR, "*"), tokens.get(4)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(5)); + assertEquals(new Token(NUMBER, "100"), tokens.get(6)); + assertEquals(new Token(SPACE, " "), tokens.get(7)); + + tokens = tokenizer.tokenize("testexact1: *\"foo bar\"*!100", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(STAR, "*"), tokens.get(2)); + assertEquals(new Token(WORD, "foo bar"), tokens.get(3)); + assertEquals(new Token(STAR, "*"), tokens.get(4)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(5)); + assertEquals(new Token(NUMBER, "100"), tokens.get(6)); + + tokens = tokenizer.tokenize("testexact1: *foobar*!100", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(STAR, "*"), tokens.get(2)); + assertEquals(new Token(WORD, "foobar"), tokens.get(3)); + assertEquals(new Token(STAR, "*"), tokens.get(4)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(5)); + assertEquals(new Token(NUMBER, "100"), tokens.get(6)); + + tokens = tokenizer.tokenize("testexact1: *foobar*!100!", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(STAR, "*"), tokens.get(2)); + assertEquals(new Token(WORD, "foobar*!100"),tokens.get(3)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(4)); + + tokens = tokenizer.tokenize("testexact1:foo(bar)", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo(bar)"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1:\"foo\"", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1: foo", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1: \"foo\"", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1: \"foo\"", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "foo"), tokens.get(2)); + + tokens = tokenizer.tokenize("testexact1:vespa testexact2:resolved", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "vespa"), tokens.get(2)); + assertEquals(new Token(SPACE, " "), tokens.get(3)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(4)); + assertEquals(new Token(COLON, ":"), tokens.get(5)); + assertEquals(new Token(WORD, "resolved"), tokens.get(6)); + + tokens = tokenizer.tokenize("testexact1:\"news search\" testexact2:resolved", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "news search"),tokens.get(2)); + assertEquals(new Token(SPACE, " "), tokens.get(3)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(4)); + assertEquals(new Token(COLON, ":"), tokens.get(5)); + assertEquals(new Token(WORD, "resolved"), tokens.get(6)); + + tokens = tokenizer.tokenize("(testexact1:\"news search\" testexact1:vespa)", facts); + assertEquals(new Token(LBRACE, "("), tokens.get(0)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(1)); + assertEquals(new Token(COLON, ":"), tokens.get(2)); + assertEquals(new Token(WORD, "news search"),tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(5)); + assertEquals(new Token(COLON, ":"), tokens.get(6)); + assertEquals(new Token(WORD, "vespa"), tokens.get(7)); + assertEquals(new Token(RBRACE, ")"), tokens.get(8)); + + tokens = tokenizer.tokenize("testexact1:news*", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "news"), tokens.get(2)); + assertEquals(new Token(STAR, "*"), tokens.get(3)); + + tokens = tokenizer.tokenize("testexact1:\"news\"*", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "news"), tokens.get(2)); + assertEquals(new Token(STAR, "*"), tokens.get(3)); + + tokens = tokenizer.tokenize("testexact1:\"news search\"!200", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "news search"),tokens.get(2)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(3)); + assertEquals(new Token(NUMBER, "200"), tokens.get(4)); + + tokens = tokenizer.tokenize("testexact1:vespa!200", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(WORD, "vespa"), tokens.get(2)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(3)); + assertEquals(new Token(NUMBER, "200"), tokens.get(4)); + + tokens = tokenizer.tokenize("testexact1:*\"news\"*", facts); + assertEquals(new Token(WORD, "testexact1"), tokens.get(0)); + assertEquals(new Token(COLON, ":"), tokens.get(1)); + assertEquals(new Token(STAR, "*"), tokens.get(2)); + assertEquals(new Token(WORD, "news"), tokens.get(3)); + assertEquals(new Token(STAR, "*"), tokens.get(4)); + + tokens = tokenizer.tokenize("normal(testexact1:foo) testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(LBRACE, "("), tokens.get(1)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "foo"), tokens.get(4)); + assertEquals(new Token(RBRACE, ")"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(7)); + assertEquals(new Token(COLON, ":"), tokens.get(8)); + assertEquals(new Token(WORD, "bar"), tokens.get(9)); + + tokens = tokenizer.tokenize("normal testexact1:(foo testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "(foo"), tokens.get(4)); + assertEquals(new Token(SPACE, " "), tokens.get(5)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(6)); + assertEquals(new Token(COLON, ":"), tokens.get(7)); + assertEquals(new Token(WORD, "bar"), tokens.get(8)); + + tokens = tokenizer.tokenize("normal testexact1:foo! testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "foo"), tokens.get(4)); + assertEquals(new Token(EXCLAMATION, "!"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(7)); + assertEquals(new Token(COLON, ":"), tokens.get(8)); + assertEquals(new Token(WORD, "bar"), tokens.get(9)); + + tokens = tokenizer.tokenize("normal testexact1:foo* testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "foo"), tokens.get(4)); + assertEquals(new Token(STAR, "*"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(7)); + assertEquals(new Token(COLON, ":"), tokens.get(8)); + assertEquals(new Token(WORD, "bar"), tokens.get(9)); + + tokens = tokenizer.tokenize("normal testexact1: foo* testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, "foo"), tokens.get(4)); + assertEquals(new Token(STAR, "*"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(7)); + assertEquals(new Token(COLON, ":"), tokens.get(8)); + assertEquals(new Token(WORD, "bar"), tokens.get(9)); + + tokens = tokenizer.tokenize("normal testexact1:\" foo\"* testexact2:bar", facts); + assertEquals(new Token(WORD, "normal"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(WORD, "testexact1"), tokens.get(2)); + assertEquals(new Token(COLON, ":"), tokens.get(3)); + assertEquals(new Token(WORD, " foo"), tokens.get(4)); + assertEquals(new Token(STAR, "*"), tokens.get(5)); + assertEquals(new Token(SPACE, " "), tokens.get(6)); + assertEquals(new Token(WORD, "testexact2"), tokens.get(7)); + assertEquals(new Token(COLON, ":"), tokens.get(8)); + assertEquals(new Token(WORD, "bar"), tokens.get(9)); + } + + public void testSingleQuoteAsWordCharacter() { + Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); + + tokenizer.setSpecialTokens(createSpecialTokens()); + List tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en nalle:a'a ugcapi_1 'a' 'a a'"); + + assertEquals(new Token(WORD, "drive"), tokens.get(0)); + assertEquals(new Token(SPACE, " "), tokens.get(1)); + assertEquals(new Token(LBRACE, "("), tokens.get(2)); + assertEquals(new Token(WORD, "to"), tokens.get(3)); + assertEquals(new Token(SPACE, " "), tokens.get(4)); + assertEquals(new Token(WORD, "hwy88"), tokens.get(5)); + assertEquals(new Token(COMMA, ","), tokens.get(6)); + assertEquals(new Token(SPACE, " "), tokens.get(7)); + assertEquals(new Token(NUMBER, "88"), tokens.get(8)); + assertEquals(new Token(RBRACE, ")"), tokens.get(9)); + assertEquals(new Token(SPACE, " "), tokens.get(10)); + assertEquals(new Token(PLUS, "+"), tokens.get(11)); + assertEquals(new Token(WORD, "or"), tokens.get(12)); + assertEquals(new Token(SPACE, " "), tokens.get(13)); + assertEquals(new Token(WORD, "language"), tokens.get(14)); + assertEquals(new Token(COLON, ":"), tokens.get(15)); + assertEquals(new Token(WORD, "en"), tokens.get(16)); + assertEquals(new Token(SPACE, " "), tokens.get(17)); + assertEquals(new Token(WORD, "nalle"), tokens.get(18)); + assertEquals(new Token(COLON, ":"), tokens.get(19)); + assertEquals(new Token(WORD, "a'a"), tokens.get(20)); + assertEquals(new Token(SPACE, " "), tokens.get(21)); + assertEquals(new Token(WORD, "ugcapi"), tokens.get(22)); + assertEquals(new Token(UNDERSCORE, "_"), tokens.get(23)); + assertEquals(new Token(NUMBER, "1"), tokens.get(24)); + assertEquals(new Token(SPACE, " "), tokens.get(25)); + assertEquals(new Token(WORD, "'a'"), tokens.get(26)); + assertEquals(new Token(SPACE, " "), tokens.get(27)); + assertEquals(new Token(WORD, "'a"), tokens.get(28)); + assertEquals(new Token(SPACE, " "), tokens.get(29)); + assertEquals(new Token(WORD, "a'"), tokens.get(30)); + } + + private SpecialTokens createSpecialTokens() { + SpecialTokens tokens = new SpecialTokens("default"); + + tokens.addSpecialToken("c+", null); + tokens.addSpecialToken("c++", null); + tokens.addSpecialToken(".net", null); + tokens.addSpecialToken("tcp/ip", null); + tokens.addSpecialToken("i/o", null); + tokens.addSpecialToken("c#", null); + tokens.addSpecialToken("AS/400", null); + return tokens; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/WashPhrasesTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/WashPhrasesTestCase.java new file mode 100644 index 00000000000..244b173dadc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/WashPhrasesTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.parser.test; + + +import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.parser.AbstractParser; +import com.yahoo.search.Query; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; + +/** + * Tests guards against in single item phrases. + * + * @author Steinar Knutsen + */ +public class WashPhrasesTestCase extends junit.framework.TestCase { + + public WashPhrasesTestCase(String name) { + super(name); + } + + public void testSimplePositive() { + PhraseItem root = new PhraseItem(); + + root.addItem(new WordItem("abc")); + assertEquals("abc", transformQuery(root)); + } + + public void testPositive1() { + AndItem root = new AndItem(); + + root.addItem(new WordItem("a")); + PhraseItem embedded = new PhraseItem(); + + embedded.addItem(new WordItem("bcd")); + root.addItem(embedded); + root.addItem(new WordItem("e")); + assertEquals("AND a bcd e", transformQuery(root)); + } + + public void testPositive2() { + AndItem root = new AndItem(); + + root.addItem(new WordItem("a")); + CompositeItem embedded = new AndItem(); + + embedded.addItem(new WordItem("bcd")); + CompositeItem phrase = new PhraseItem(); + + phrase.addItem(new WordItem("def")); + embedded.addItem(phrase); + root.addItem(embedded); + root.addItem(new WordItem("e")); + assertEquals("AND a (AND bcd def) e", transformQuery(root)); + } + + public void testNoTerms() { + assertNull(transformQuery("\"\"")); + } + + public void testNegative1() { + assertEquals("\"abc def\"", transformQuery("\"abc def\"")); + } + + public void testNegative2() { + assertEquals("AND a \"abc def\" b", transformQuery("a \"abc def\" b")); + } + + public void testNegative3() { + AndItem root = new AndItem(); + + root.addItem(new WordItem("a")); + CompositeItem embedded = new AndItem(); + + embedded.addItem(new WordItem("bcd")); + CompositeItem phrase = new PhraseItem(); + + phrase.addItem(new WordItem("def")); + phrase.addItem(new WordItem("ghi")); + embedded.addItem(phrase); + root.addItem(embedded); + root.addItem(new WordItem("e")); + assertEquals("AND a (AND bcd \"def ghi\") e", transformQuery(root)); + } + + private String transformQuery(String rawQuery) { + Parser parser = ParserFactory.newInstance(Query.Type.ALL, new ParserEnvironment()); + Item root = parser.parse(new Parsable().setQuery(rawQuery)).getRoot(); + if (root instanceof NullItem) { + return null; + } + return root.toString(); + } + + private String transformQuery(Item queryTree) { + return AbstractParser.simplifyPhrases(queryTree).toString(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg new file mode 100644 index 00000000000..0d264e04799 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg @@ -0,0 +1,111 @@ +indexinfo[3] +indexinfo[0].name one +indexinfo[0].command[44] +indexinfo[0].command[0].indexname url.all +indexinfo[0].command[0].command fullurl +indexinfo[0].command[1].indexname host.all +indexinfo[0].command[1].command urlhost +indexinfo[0].command[2].indexname site +indexinfo[0].command[2].command urlhost +indexinfo[0].command[3].indexname url.all +indexinfo[0].command[3].command index +indexinfo[0].command[4].indexname host.all +indexinfo[0].command[4].command index +indexinfo[0].command[5].indexname site +indexinfo[0].command[5].command index +indexinfo[0].command[6].indexname foo.bar +indexinfo[0].command[6].command index +indexinfo[0].command[7].indexname site +indexinfo[0].command[7].command index +indexinfo[0].command[8].indexname to +indexinfo[0].command[8].command index +indexinfo[0].command[9].indexname ull +indexinfo[0].command[9].command index +indexinfo[0].command[10].indexname s +indexinfo[0].command[10].command index +indexinfo[0].command[11].indexname mail +indexinfo[0].command[11].command index +indexinfo[0].command[12].indexname a +indexinfo[0].command[12].command index +indexinfo[0].command[13].indexname normal +indexinfo[0].command[13].command index +indexinfo[0].command[14].indexname url.domain +indexinfo[0].command[14].command index +indexinfo[0].command[15].indexname normal.title +indexinfo[0].command[15].command index +indexinfo[0].command[16].indexname url +indexinfo[0].command[16].command index +indexinfo[0].command[17].indexname r.s +indexinfo[0].command[17].command index +indexinfo[0].command[18].indexname title +indexinfo[0].command[18].command index +indexinfo[0].command[19].indexname domain +indexinfo[0].command[19].command index +indexinfo[0].command[20].indexname pagedepth +indexinfo[0].command[20].command index +indexinfo[0].command[21].indexname audio.audall +indexinfo[0].command[21].command index +indexinfo[0].command[22].indexname fast.type +indexinfo[0].command[22].command index +indexinfo[0].command[23].indexname www +indexinfo[0].command[23].command index +indexinfo[0].command[24].indexname date +indexinfo[0].command[24].command index +indexinfo[0].command[25].indexname document.size +indexinfo[0].command[25].command index +indexinfo[0].command[26].indexname date.all +indexinfo[0].command[26].command index +indexinfo[0].command[27].indexname size.all +indexinfo[0].command[27].command index +indexinfo[0].command[28].indexname name +indexinfo[0].command[28].command index +indexinfo[0].command[29].indexname newstype +indexinfo[0].command[29].command index +indexinfo[0].command[30].indexname language +indexinfo[0].command[30].command index +indexinfo[0].command[31].indexname mixedCase +indexinfo[0].command[31].command index +indexinfo[0].command[32].indexname performernameall +indexinfo[0].command[32].command index +indexinfo[0].command[33].indexname songnameall +indexinfo[0].command[33].command index +indexinfo[0].command[34].indexname metadata +indexinfo[0].command[34].command index +indexinfo[0].command[35].indexname SongConsumable +indexinfo[0].command[35].command index +indexinfo[0].command[36].indexname country +indexinfo[0].command[36].command index +indexinfo[0].command[37].indexname collapsedrecord +indexinfo[0].command[37].command index +indexinfo[0].command[38].indexname collapsecount +indexinfo[0].command[38].command index +indexinfo[0].command[39].indexname doctype +indexinfo[0].command[39].command index +indexinfo[0].command[40].indexname score_under +indexinfo[0].command[40].command index +indexinfo[0].command[41].indexname _under_score_ +indexinfo[0].command[41].command index +indexinfo[0].command[42].indexname exactindex +indexinfo[0].command[42].command index +indexinfo[0].command[43].indexname exactindex +indexinfo[0].command[43].command exact + +indexinfo[1].name twoRanges +indexinfo[1].command[2] +indexinfo[1].command[0].indexname score +indexinfo[1].command[0].command index +indexinfo[1].command[1].indexname age +indexinfo[1].command[1].command index + +indexinfo[2].name webIndices +indexinfo[2].command[5] +indexinfo[2].command[0].indexname intitle +indexinfo[2].command[0].command index +indexinfo[2].command[1].indexname inurl +indexinfo[2].command[1].command index +indexinfo[2].command[2].indexname hostname +indexinfo[2].command[2].command index +indexinfo[2].command[3].indexname link +indexinfo[2].command[3].command index +indexinfo[2].command[4].indexname url +indexinfo[2].command[4].command index diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg new file mode 100644 index 00000000000..6a189de0164 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg @@ -0,0 +1,12 @@ +tokenlist[1] +tokenlist[0].name default +tokenlist[0].tokens[6] +tokenlist[0].tokens[0].token .... +tokenlist[0].tokens[1].token c++ +tokenlist[0].tokens[1].replace cpp +tokenlist[0].tokens[2].token b.s.d. +tokenlist[0].tokens[3].token with space +tokenlist[0].tokens[3].replace with-space +tokenlist[0].tokens[4].token c# +tokenlist[0].tokens[5].token know +tokenlist[0].tokens[5].replace knuwww diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg new file mode 100644 index 00000000000..5f54d47353f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg @@ -0,0 +1,15 @@ +tokenlist[2] +tokenlist[0].name default +tokenlist[0].tokens[6] +tokenlist[0].tokens[0].token .... +tokenlist[0].tokens[1].token c++ +tokenlist[0].tokens[2].token b.s.d. +tokenlist[0].tokens[3].token with space +tokenlist[0].tokens[4].token c# +tokenlist[0].tokens[5].token dvd±r +tokenlist[1].name other +tokenlist[1].tokens[4] +tokenlist[1].tokens[0].token [huh] +tokenlist[1].tokens[1].token &&&%%% +tokenlist[1].tokens[2].token ------ +tokenlist[1].tokens[3].token !!!*** diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/DotProductItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/DotProductItemTestCase.java new file mode 100644 index 00000000000..1d2261081d1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/DotProductItemTestCase.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +import com.yahoo.prelude.query.*; + +/** + * @author havardpe + */ +public class DotProductItemTestCase { + + @Test + public void testDotProductItem() { + DotProductItem item = new DotProductItem("index_name"); + assertEquals("index_name", item.getIndexName()); + assertEquals(Item.ItemType.DOTPRODUCT, item.getItemType()); + } + + @Test + public void testDotProductClone() { + DotProductItem dpOrig = new DotProductItem("myDP"); + dpOrig.addToken("first",11); + dpOrig.getTokens(); + DotProductItem dpClone = (DotProductItem) dpOrig.clone(); + dpClone.addToken("second", 22); + assertEquals(2, dpClone.getNumTokens()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/IntItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/IntItemTestCase.java new file mode 100644 index 00000000000..28acb310472 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/IntItemTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.search.Query; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Jon Bratseth + */ +public class IntItemTestCase { + + @Test + public void testEquals() { + Query q1 = new Query("/?query=123%20456%20789"); + Query q2 = new Query("/?query=123%20456"); + + AndItem andItem = (AndItem) q2.getModel().getQueryTree().getRoot(); + andItem.addItem(new IntItem(789l, "")); + + assertEquals(q1, q2); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java new file mode 100644 index 00000000000..d51ad9bc6d5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java @@ -0,0 +1,319 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + + +import java.nio.ByteBuffer; + +import com.yahoo.prelude.query.*; + + +/** + * Item encoding tests + * + * @author bratseth + */ +public class ItemEncodingTestCase extends junit.framework.TestCase { + + public ItemEncodingTestCase(String name) { + super(name); + } + + private void assertType(ByteBuffer buffer, int etype, int features) { + byte type = buffer.get(); + assertEquals("Code", etype, type & 0x1f); + assertEquals("Features", features, (type & 0xe0) >> 5); + } + private void assertWeight(ByteBuffer buffer, int weight) { + int w = (weight > (1 << 5)) ? buffer.getShort() & 0x3fff: buffer.get(); + assertEquals("Weight", weight, w); + } + public void testWordItemEncoding() { + WordItem word = new WordItem("test"); + + word.setWeight(150); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = word.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 1, count); + + assertType(buffer, 4, 1); + assertWeight(buffer, 150); + + assertEquals("Index length", 0, buffer.get()); + assertEquals("Word length", 4, buffer.get()); + assertEquals("Word length", 4, buffer.remaining()); + assertEquals('t', buffer.get()); + assertEquals('e', buffer.get()); + assertEquals('s', buffer.get()); + assertEquals('t', buffer.get()); + } + + public void testStartHostMarkerEncoding() { + WordItem word = MarkerWordItem.createStartOfHost(); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = word.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 1, count); + + assertType(buffer, 4, 0); + + assertEquals("Index length", 0, buffer.get()); + assertEquals("Word length", 9, buffer.get()); + assertEquals("Word length", 9, buffer.remaining()); + assertEquals('S', buffer.get()); + assertEquals('t', buffer.get()); + assertEquals('A', buffer.get()); + assertEquals('r', buffer.get()); + assertEquals('T', buffer.get()); + assertEquals('h', buffer.get()); + assertEquals('O', buffer.get()); + assertEquals('s', buffer.get()); + assertEquals('T', buffer.get()); + } + + public void testEndHostMarkerEncoding() { + WordItem word = MarkerWordItem.createEndOfHost(); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = word.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 1, count); + + assertType(buffer, 4, 0); + + assertEquals("Index length", 0, buffer.get()); + assertEquals("Word length", 7, buffer.get()); + assertEquals("Word length", 7, buffer.remaining()); + assertEquals('E', buffer.get()); + assertEquals('n', buffer.get()); + assertEquals('D', buffer.get()); + assertEquals('h', buffer.get()); + assertEquals('O', buffer.get()); + assertEquals('s', buffer.get()); + assertEquals('T', buffer.get()); + } + + public void testFilterWordItemEncoding() { + WordItem word = new WordItem("test"); + + word.setFilter(true); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = word.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 1, count); + + assertType(buffer, 4, 4); + assertEquals(0x08, buffer.get()); + + assertEquals("Index length", 0, buffer.get()); + assertEquals("Word length", 4, buffer.get()); + assertEquals("Word length", 4, buffer.remaining()); + assertEquals('t', buffer.get()); + assertEquals('e', buffer.get()); + assertEquals('s', buffer.get()); + assertEquals('t', buffer.get()); + } + + public void testNoRankedNoPositionDataWordItemEncoding() { + WordItem word = new WordItem("test"); + word.setRanked(false); + word.setPositionData(false); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = word.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 1, count); + + assertType(buffer, 4, 4); + assertEquals(0x05, buffer.get()); + + assertEquals("Index length", 0, buffer.get()); + assertEquals("Word length", 4, buffer.get()); + assertEquals("Word length", 4, buffer.remaining()); + assertEquals('t', buffer.get()); + assertEquals('e', buffer.get()); + assertEquals('s', buffer.get()); + assertEquals('t', buffer.get()); + } + + public void testAndItemEncoding() { + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + AndItem and=new AndItem(); + and.addItem(a); + and.addItem(b); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = and.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 3, count); + + assertType(buffer, 1, 0); + + assertEquals("And arity", 2, buffer.get()); + + assertWord(buffer,"a"); + assertWord(buffer,"b"); + } + + public void testNearItemEncoding() { + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + NearItem near=new NearItem(7); + near.addItem(a); + near.addItem(b); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = near.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 3, count); + + assertType(buffer, 11, 0); + + assertEquals("Near arity", 2, buffer.get()); + assertEquals("Limit", 7, buffer.get()); + + assertWord(buffer,"a"); + assertWord(buffer,"b"); + } + + public void testONearItemEncoding() { + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + NearItem onear=new ONearItem(7); + onear.addItem(a); + onear.addItem(b); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = onear.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 3, count); + + assertType(buffer, 12, 0); + assertEquals("Near arity", 2, buffer.get()); + assertEquals("Limit", 7, buffer.get()); + + assertWord(buffer,"a"); + assertWord(buffer,"b"); + } + + public void testEquivItemEncoding() { + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + EquivItem equiv = new EquivItem(); + equiv.addItem(a); + equiv.addItem(b); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = equiv.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 3, count); + + assertType(buffer, 14, 0); + assertEquals("Equiv arity", 2, buffer.get()); + + assertWord(buffer, "a"); + assertWord(buffer, "b"); + } + + public void testWandItemEncoding() { + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + WeakAndItem wand = new WeakAndItem(); + wand.addItem(a); + wand.addItem(b); + + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = wand.encode(buffer); + + buffer.flip(); + + assertEquals("Serialization count", 3, count); + + assertType(buffer, 16, 0); + assertEquals("WeakAnd arity", 2, buffer.get()); + assertEquals("WeakAnd N", 100, buffer.getShort() & 0x3fff); + assertEquals(0, buffer.get()); + + assertWord(buffer, "a"); + assertWord(buffer, "b"); + } + + public void testPureWeightedStringEncoding() { + PureWeightedString a = new PureWeightedString("a"); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + assertEquals("Serialization size", 3, buffer.remaining()); + assertEquals("Serialization count", 1, count); + assertType(buffer, 19, 0); + assertString(buffer, a.getString()); + } + + public void testPureWeightedStringEncodingWithNonDefaultWeight() { + PureWeightedString a = new PureWeightedString("a", 7); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + assertEquals("Serialization size", 4, buffer.remaining()); + assertEquals("Serialization count", 1, count); + assertType(buffer, 19, 1); + assertWeight(buffer, 7); + assertString(buffer, a.getString()); + } + + public void testPureWeightedIntegerEncoding() { + PureWeightedInteger a = new PureWeightedInteger(23432568763534865l); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + assertEquals("Serialization size", 9, buffer.remaining()); + assertEquals("Serialization count", 1, count); + assertType(buffer, 20, 0); + assertEquals("Value", a.getValue(), buffer.getLong()); + } + + public void testPureWeightedLongEncodingWithNonDefaultWeight() { + PureWeightedInteger a = new PureWeightedInteger(23432568763534865l, 7); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + assertEquals("Serialization size", 10, buffer.remaining()); + assertEquals("Serialization count", 1, count); + assertType(buffer, 20, 1); + assertWeight(buffer, 7); + assertEquals("Value", a.getValue(), buffer.getLong());; + } + + private void assertString(ByteBuffer buffer, String word) { + assertEquals("Word length", word.length(), buffer.get()); + for (int i=0; iSteinar Knutsen + */ +public class PhraseItemTestCase extends junit.framework.TestCase { + + public PhraseItemTestCase(String name) { + super(name); + } + + public void testAddItem() { + PhraseItem p = new PhraseItem(); + PhraseSegmentItem pp = new PhraseSegmentItem("", false, false); + PhraseItem ppp = new PhraseItem(); + pp.addItem(new WordItem("b")); + pp.addItem(new WordItem("c")); + ppp.addItem(new WordItem("e")); + ppp.addItem(new WordItem("f")); + p.addItem(new WordItem("a")); + p.addItem(pp); + p.addItem(new WordItem("d")); + p.addItem(ppp); + assertEquals("\"a 'b c' d e f\"", p.toString()); + } + + public void testAddItemWithIndex() { + PhraseItem p = new PhraseItem(); + PhraseSegmentItem pp = new PhraseSegmentItem("", false, false); + PhraseItem ppp = new PhraseItem(); + pp.addItem(new WordItem("a")); + pp.addItem(new WordItem("b")); + ppp.addItem(new WordItem("c")); + ppp.addItem(new WordItem("d")); + p.addItem(0, new WordItem("e")); + p.addItem(0, pp); + p.addItem(2, new WordItem("f")); + p.addItem(1, ppp); + assertEquals("\"'a b' c d e f\"", p.toString()); + } + + public void testSetItem() { + PhraseItem backup = new PhraseItem(); + PhraseSegmentItem segment = new PhraseSegmentItem("", false, false); + PhraseItem innerPhrase = new PhraseItem(); + WordItem testWord = new WordItem("z"); + PhraseItem p; + segment.addItem(new WordItem("p")); + segment.addItem(new WordItem("q")); + innerPhrase.addItem(new WordItem("x")); + innerPhrase.addItem(new WordItem("y")); + backup.addItem(new WordItem("a")); + backup.addItem(new WordItem("b")); + backup.addItem(new WordItem("c")); + + p = (PhraseItem) backup.clone(); + p.setItem(0, segment); + assertEquals("\"'p q' b c\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(1, segment); + assertEquals("\"a 'p q' c\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(2, segment); + assertEquals("\"a b 'p q'\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(0, innerPhrase); + assertEquals("\"x y b c\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(1, innerPhrase); + assertEquals("\"a x y c\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(2, innerPhrase); + assertEquals("\"a b x y\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(0, testWord); + assertEquals("\"z b c\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(1, testWord); + assertEquals("\"a z c\"", p.toString()); + + p = (PhraseItem) backup.clone(); + p.setItem(2, testWord); + assertEquals("\"a b z\"", p.toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java new file mode 100644 index 00000000000..a5ae6b78d4b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PredicateQueryItem; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; + +import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; + +/** + * @author Magnar Nedland + */ +public class PredicateQueryItemTestCase { + @Test + public void requireThatItemConstantsAreSet() { + PredicateQueryItem item = new PredicateQueryItem(); + assertEquals(Item.ItemType.PREDICATE_QUERY, item.getItemType()); + assertEquals("PREDICATE_QUERY_ITEM", item.getName()); + assertEquals(1, item.getTermCount()); + assertEquals("predicate", item.getIndexName()); + item.setIndexName("foobar"); + assertEquals("foobar", item.getIndexName()); + } + + @Test + public void requireThatFeaturesCanBeAdded() { + PredicateQueryItem item = new PredicateQueryItem(); + assertEquals(0, item.getFeatures().size()); + item.addFeature("foo", "bar"); + item.addFeature("foo", "baz", 0xffff); + item.addFeature(new PredicateQueryItem.Entry("qux", "quux")); + item.addFeature(new PredicateQueryItem.Entry("corge", "grault", 0xf00ba)); + assertEquals(4, item.getFeatures().size()); + Iterator it = item.getFeatures().iterator(); + assertEquals(-1, it.next().getSubQueryBitmap()); + assertEquals(0xffffL, it.next().getSubQueryBitmap()); + assertEquals(-1, it.next().getSubQueryBitmap()); + assertEquals(0xf00baL, it.next().getSubQueryBitmap()); + } + + @Test + public void requireThatRangeFeaturesCanBeAdded() { + PredicateQueryItem item = new PredicateQueryItem(); + assertEquals(0, item.getRangeFeatures().size()); + item.addRangeFeature("foo", 23); + item.addRangeFeature("foo", 34, 0x12345678L); + item.addRangeFeature(new PredicateQueryItem.RangeEntry("qux", 43)); + item.addRangeFeature(new PredicateQueryItem.RangeEntry("corge", 54, 0xf00ba)); + assertEquals(4, item.getRangeFeatures().size()); + Iterator it = item.getRangeFeatures().iterator(); + assertEquals(-1, it.next().getSubQueryBitmap()); + assertEquals(0x12345678L, it.next().getSubQueryBitmap()); + assertEquals(-1, it.next().getSubQueryBitmap()); + assertEquals(0xf00baL, it.next().getSubQueryBitmap()); + } + + @Test + public void requireThatToStringWorks() { + PredicateQueryItem item = new PredicateQueryItem(); + assertEquals("PREDICATE_QUERY_ITEM ", item.toString()); + item.addFeature("foo", "bar"); + item.addFeature("foo", "baz", 0xffffL); + assertEquals("PREDICATE_QUERY_ITEM foo=bar, foo=baz[0xffff]", item.toString()); + item.addRangeFeature("foo", 23); + item.addRangeFeature("foo", 34, 0xfffffffffffffffeL); + assertEquals("PREDICATE_QUERY_ITEM foo=bar, foo=baz[0xffff], foo:23, foo:34[0xfffffffffffffffe]", item.toString()); + } + + @Test + public void requireThatPredicateQueryItemCanBeEncoded() { + PredicateQueryItem item = new PredicateQueryItem(); + assertEquals("PREDICATE_QUERY_ITEM ", item.toString()); + item.addFeature("foo", "bar"); + item.addFeature("foo", "baz", 0xffffL); + ByteBuffer buffer = ByteBuffer.allocate(1000); + item.encode(buffer); + buffer.flip(); + byte[] actual = new byte[buffer.remaining()]; + buffer.get(actual); + assertArrayEquals(new byte[]{ + 23, // PREDICATE_QUERY code 23 + 9, 'p', 'r', 'e', 'd', 'i', 'c', 'a', 't', 'e', + 2, // 2 features + 3, 'f', 'o', 'o', 3, 'b', 'a', 'r', -1, -1, -1, -1, -1, -1, -1, -1, // key, value, subquery + 3, 'f', 'o', 'o', 3, 'b', 'a', 'z', 0, 0, 0, 0, 0, 0, -1, -1, // key, value, subquery + 0}, // no range features + actual); + + item.addRangeFeature("foo", 23); + item.addRangeFeature("foo", 34, 0xfffffffffffffffeL); + buffer.clear(); + item.encode(buffer); + buffer.flip(); + actual = new byte[buffer.remaining()]; + buffer.get(actual); + assertArrayEquals(new byte[]{ + 23, // PREDICATE_QUERY code 23 + 9, 'p', 'r', 'e', 'd', 'i', 'c', 'a', 't', 'e', + 2, // 2 features + 3, 'f', 'o', 'o', 3, 'b', 'a', 'r', -1, -1, -1, -1, -1, -1, -1, -1, // key, value, subquery + 3, 'f', 'o', 'o', 3, 'b', 'a', 'z', 0, 0, 0, 0, 0, 0, -1, -1, // key, value, subquery + 2, // 2 range features + 3, 'f', 'o', 'o', 0, 0, 0, 0, 0, 0, 0, 23, -1, -1, -1, -1, -1, -1, -1, -1, // key, value, subquery + 3, 'f', 'o', 'o', 0, 0, 0, 0, 0, 0, 0, 34, -1, -1, -1, -1, -1, -1, -1, -2}, // key, value, subquery + actual); + } + + @Test + public void requireThatPredicateQueryItemWithManyAttributesCanBeEncoded() { + PredicateQueryItem item = new PredicateQueryItem(); + assertEquals("PREDICATE_QUERY_ITEM ", item.toString()); + for (int i = 0; i < 200; ++i) { + item.addFeature("foo", "bar"); + } + ByteBuffer buffer = ByteBuffer.allocate(10000); + item.encode(buffer); + buffer.flip(); + byte[] actual = new byte[buffer.remaining()]; + buffer.get(actual); + byte [] expectedPrefix = new byte[]{ + 23, // PREDICATE_QUERY code 23 + 9, 'p', 'r', 'e', 'd', 'i', 'c', 'a', 't', 'e', + (byte)0x80, (byte)0xc8, // 200 features (0x80c8 => 0xc8 == 200) + 3, 'f', 'o', 'o', 3, 'b', 'a', 'r', -1, -1, -1, -1, -1, -1, -1, -1, // key, value, subquery + 3, 'f', 'o', 'o', 3, 'b', 'a', 'r', -1, -1, -1, -1, -1, -1, -1, -1, // key, value, subquery + 3, 'f', 'o', 'o', 3, 'b', 'a', 'r', -1, -1, -1, -1, -1, -1, -1, -1, // key, value, subquery + }; // ... + assertArrayEquals(expectedPrefix, Arrays.copyOfRange(actual, 0, expectedPrefix.length)); + + } +} \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerMicroBenchmark.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerMicroBenchmark.java new file mode 100644 index 00000000000..8938ef9cf87 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerMicroBenchmark.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.QueryCanonicalizer; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.query.QueryTree; + +/** + * @author bratseth + */ +public class QueryCanonicalizerMicroBenchmark { + + public void run() { + System.out.println("Running ..."); + for (int i = 0; i < 10*1000; i++) + canonicalize(); + long startTime = System.currentTimeMillis(); + int repetitions = 10 * 1000 * 1000; + for (int i = 0; i < repetitions; i++) + canonicalize(); + long totalTime = System.currentTimeMillis() - startTime; + System.out.println("Total time: " + totalTime + " ms\nTime per canonicalization: " + + 1000*1000*totalTime/(float)repetitions + " ns"); + } + + private void canonicalize() { + AndItem and = new AndItem(); + and.addItem(new WordItem("shoe", "prod")); + and.addItem(new WordItem("apparel & accessories", "tcnm")); + RankItem rank = new RankItem(); + rank.addItem(and); + for (int i = 0; i < 25; i++) + rank.addItem(new WordItem("word" + i, "normbrnd")); + QueryTree tree = new QueryTree(rank); + QueryCanonicalizer.canonicalize(tree); + } + + public static void main(String[] args) { + new QueryCanonicalizerMicroBenchmark().run(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java new file mode 100644 index 00000000000..4185065b33c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -0,0 +1,329 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.query.QueryTree; + +/** + * @author Jon S Bratseth + */ +public class QueryCanonicalizerTestCase extends junit.framework.TestCase { + + public QueryCanonicalizerTestCase(String name) { + super(name); + } + + public void testSingleLevelSingleItemComposite() { + CompositeItem root = new AndItem(); + + root.addItem(new WordItem("word")); + assertCanonicalized("word", null, root); + } + + public void testSingleLevelSingleItemNonReducibleComposite() { + CompositeItem root = new WeakAndItem(); + + root.addItem(new WordItem("word")); + assertCanonicalized("WAND(100) word", null, root); + } + + public void testMultilevelSingleItemComposite() { + CompositeItem root = new AndItem(); + CompositeItem and1 = new AndItem(); + CompositeItem and2 = new AndItem(); + + root.addItem(and1); + and1.addItem(and2); + and2.addItem(new WordItem("word")); + assertCanonicalized("word", null, root); + } + + public void testMultilevelComposite() { + // AND (RANK (AND a b c)) WAND(25,0.0,1.0) + AndItem and = new AndItem(); + RankItem rank = new RankItem(); + and.addItem(rank); + AndItem nestedAnd = new AndItem(); + nestedAnd.addItem(new WordItem("a")); + nestedAnd.addItem(new WordItem("b")); + nestedAnd.addItem(new WordItem("c")); + rank.addItem(nestedAnd); + WandItem wand = new WandItem("default", 100); + and.addItem(wand); + + assertCanonicalized("AND (AND a b c) WAND(100,0.0,1.0) default}", null, and); + } + + public void testMultilevelEmptyComposite() { + CompositeItem root = new AndItem(); + CompositeItem and1 = new AndItem(); + CompositeItem and2 = new AndItem(); + + root.addItem(and1); + and1.addItem(and2); + assertCanonicalized("NULL", "No query", new Query()); + } + + public void testMultilevelMultiBranchEmptyComposite() { + CompositeItem root = new AndItem(); + CompositeItem and1 = new AndItem(); + CompositeItem and21 = new AndItem(); + CompositeItem and22 = new AndItem(); + CompositeItem and31 = new AndItem(); + CompositeItem and32 = new AndItem(); + + root.addItem(and1); + and1.addItem(and21); + and1.addItem(and22); + and22.addItem(and31); + and22.addItem(and32); + assertCanonicalized("NULL", "No query", new Query()); + } + + public void testMultilevelMultiBranchSingleItemComposite() { + CompositeItem root = new AndItem(); + CompositeItem and1 = new AndItem(); + CompositeItem and21 = new AndItem(); + CompositeItem and22 = new AndItem(); + CompositeItem and31 = new AndItem(); + CompositeItem and32 = new AndItem(); + + root.addItem(and1); + and1.addItem(and21); + and1.addItem(and22); + and22.addItem(and31); + and22.addItem(and32); + and22.addItem(new WordItem("word")); + assertCanonicalized("word", null, new Query("?query=word")); + } + + public void testNullRoot() { + assertCanonicalized("NULL", "No query", new Query()); + } + + public void testNestedNull() { + CompositeItem root = new AndItem(); + CompositeItem or = new AndItem(); + CompositeItem and = new AndItem(); + + root.addItem(or); + or.addItem(and); + Query query = new Query(); + + query.getModel().getQueryTree().setRoot(root); + + assertCanonicalized("NULL", "No query: Contained an empty AND only", root); + } + + public void testNestedNullItem() { + CompositeItem root = new AndItem(); + CompositeItem or = new AndItem(); + CompositeItem and = new AndItem(); + and.addItem(new NullItem()); + and.addItem(new NullItem()); + + root.addItem(or); + or.addItem(and); + Query query = new Query(); + + query.getModel().getQueryTree().setRoot(root); + + assertCanonicalized("NULL", "No query: Contained an empty AND only", root); + } + + public void testNestedNullAndSingle() { + CompositeItem root = new AndItem(); + CompositeItem or = new OrItem(); + + root.addItem(or); + CompositeItem and = new AndItem(); + + or.addItem(and); + or.addItem(new WordItem("word")); + assertCanonicalized("word", null, root); + } + + public void testRemovalOfUnnecessaryComposites() { + CompositeItem root = new AndItem(); + CompositeItem or = new OrItem(); + + root.addItem(or); + CompositeItem and = new AndItem(); + + or.addItem(new WordItem("word1")); + or.addItem(and); + or.addItem(new WordItem("word2")); + or.addItem(new WordItem("word3")); + assertCanonicalized("OR word1 word2 word3", null, root); + } + + public void testNegativeMustHaveNegatives() { + CompositeItem root = new NotItem(); + + root.addItem(new WordItem("positive")); + assertCanonicalized("positive", null, root); + } + + public void testNegativeMustHavePositive() { + NotItem root = new NotItem(); + + root.addNegativeItem(new WordItem("negative")); + assertCanonicalized("+(null) -negative", + "Can not search for only negative items", root); + } + + public void testNegativeMustHavePositiveNested() { + CompositeItem root = new AndItem(); + NotItem not = new NotItem(); + + root.addItem(not); + root.addItem(new WordItem("word")); + not.addNegativeItem(new WordItem("negative")); + assertCanonicalized("AND (+(null) -negative) word", + "Can not search for only negative items", root); + } + + /** + * Tests that connexity is preserved by cloning and transferred to rank properties by preparing the query + * (which strictly is an implementation detail which we should rather hide). + */ + public void testConnexityAndCloning() { + Query q = new Query("?query=a%20b"); + CompositeItem root = (CompositeItem) q.getModel().getQueryTree().getRoot(); + ((WordItem) root.getItem(0)).setConnectivity(root.getItem(1), java.lang.Math.E); + q = q.clone(); + + assertNull("Not prepared yet", q.getRanking().getProperties().get("vespa.term.1.connexity")); + q.prepare(); + assertEquals("2", q.getRanking().getProperties().get("vespa.term.1.connexity").get(0)); + assertEquals("2.718281828459045", q.getRanking().getProperties().get("vespa.term.1.connexity").get(1)); + q = q.clone(); // The clone stays prepared + assertEquals("2", q.getRanking().getProperties().get("vespa.term.1.connexity").get(0)); + assertEquals("2.718281828459045", q.getRanking().getProperties().get("vespa.term.1.connexity").get(1)); + } + + /** + * Tests that significance is transferred to rank properties by preparing the query + * (which strictly is an implementation detail which we should rather hide). + */ + public void testSignificance() { + Query q = new Query("?query=a%20b"); + CompositeItem root = (CompositeItem) q.getModel().getQueryTree().getRoot(); + ((WordItem) root.getItem(0)).setSignificance(0.5); + ((WordItem) root.getItem(1)).setSignificance(0.95); + q.prepare(); + assertEquals("0.5", q.getRanking().getProperties().get("vespa.term.1.significance").get(0)); + assertEquals("0.95", q.getRanking().getProperties().get("vespa.term.2.significance").get(0)); + } + + public void testPhraseWeight() { + PhraseItem root = new PhraseItem(); + root.setWeight(200); + root.addItem(new WordItem("a")); + assertCanonicalized("a!200", null, root); + } + + public void testEquivDuplicateRemoval() { + { + EquivItem root = new EquivItem(); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + assertCanonicalized("EQUIV a b", null, root); + } + { + EquivItem root = new EquivItem(); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + assertCanonicalized("EQUIV a b", null, root); + } + { + EquivItem root = new EquivItem(); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("a")); + assertCanonicalized("a", null, root); + } + { + EquivItem root = new EquivItem(); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("a")); + assertCanonicalized("a", null, root); + } + { + EquivItem root = new EquivItem(); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("a")); + assertCanonicalized("EQUIV a b", null, root); + } + { + EquivItem root = new EquivItem(); + PhraseItem one = new PhraseItem(); + PhraseItem theOther = new PhraseItem(); + WordItem first = new WordItem("a"); + WordItem second = new WordItem("b"); + one.addItem(first); + one.addItem(second); + theOther.addItem(first.clone()); + theOther.addItem(second.clone()); + root.addItem(one); + root.addItem(theOther); + assertCanonicalized("\"a b\"", null, root); + } + { + EquivItem root = new EquivItem(); + PhraseSegmentItem one = new PhraseSegmentItem("a b", "a b", true, false); + PhraseSegmentItem theOther = new PhraseSegmentItem("a b", "a b", true, false); + WordItem first = new WordItem("a"); + WordItem second = new WordItem("b"); + one.addItem(first); + one.addItem(second); + theOther.addItem(first.clone()); + theOther.addItem(second.clone()); + root.addItem(one); + root.addItem(theOther); + assertCanonicalized("'a b'", null, root); + } + } + + public void testRankDuplicateCheapification() { + AndItem and = new AndItem(); + WordItem shoe = new WordItem("shoe", "prod"); + and.addItem(shoe); + and.addItem(new WordItem("apparel & accessories", "tcnm")); + RankItem rank = new RankItem(); + rank.addItem(and); + + rank.addItem(new WordItem("shoe", "prod")); // rank item which also ossurs in first argument + for (int i = 0; i < 25; i++) + rank.addItem(new WordItem("word" + i, "normbrnd")); + QueryTree tree = new QueryTree(rank); + + assertTrue(shoe.isRanked()); + assertTrue(shoe.usePositionData()); + QueryCanonicalizer.canonicalize(tree); + assertFalse(shoe.isRanked()); + assertFalse(shoe.usePositionData()); + } + + private void assertCanonicalized(String canonicalForm, String expectedError, Item root) { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(root); + assertCanonicalized(canonicalForm, expectedError, query); + } + + private void assertCanonicalized(String canonicalForm, String expectedError, Query query) { + String error = QueryCanonicalizer.canonicalize(query); + + assertEquals(expectedError, error); + if (canonicalForm == null) { + assertNull(null, query.getModel().getQueryTree().getRoot()); + } else { + assertEquals(canonicalForm, query.getModel().getQueryTree().getRoot().toString()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java new file mode 100644 index 00000000000..cd3f127e8cc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryLanguageTestCase.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.language.Language; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; + +/** + *

Tests that the correct query language strings are generated for various + * query trees.

+ * + *

The opposite direction is tested by + * {@link com.yahoo.prelude.query.parser.test.ParseTestCase}. + * Note that the query language statements produced by a query tree is a + * subset of the statements accepted by the parser.

+ * + * @author bratseth + */ +public class QueryLanguageTestCase extends junit.framework.TestCase { + + public QueryLanguageTestCase(String name) { + super(name); + } + + public void testWord() { + WordItem w = new WordItem("test"); + + assertEquals("test", w.toString()); + } + + public void testWordWithIndex() { + WordItem w = new WordItem("test"); + + w.setIndexName("test.index"); + assertEquals("test.index:test", w.toString()); + } + + public void testPhrase() { + PhraseItem p = new PhraseItem(); + + p.addItem(new WordItem("part")); + p.addItem(new WordItem("of")); + p.addItem(new WordItem("phrase")); + assertEquals("\"part of phrase\"", p.toString()); + } + + public void testPhraseWithIndex() { + PhraseItem p = new PhraseItem(); + + p.addItem(new WordItem("part")); + p.addItem(new WordItem("of")); + p.addItem(new WordItem("phrase")); + p.setIndexName("some.index"); + assertEquals("some.index:\"part of phrase\"", p.toString()); + } + + public void testNotItem() { + NotItem n = new NotItem(); + + n.addNegativeItem(new WordItem("notthis")); + n.addNegativeItem(new WordItem("andnotthis")); + n.addPositiveItem(new WordItem("butthis")); + assertEquals("+butthis -notthis -andnotthis", n.toString()); + } + + public void testLanguagesInQueryParameter() { + // Right parameter is the parameter given in the query, as language= + // Left parameter is the language sent to linguistics + + // Ancient + assertLanguage(Language.CHINESE_SIMPLIFIED,"zh-cn"); + assertLanguage(Language.CHINESE_SIMPLIFIED,"zh-Hans"); + assertLanguage(Language.CHINESE_SIMPLIFIED,"zh-hans"); + assertLanguage(Language.CHINESE_TRADITIONAL,"zh-tw"); + assertLanguage(Language.CHINESE_TRADITIONAL,"zh-Hant"); + assertLanguage(Language.CHINESE_TRADITIONAL,"zh-hant"); + assertLanguage(Language.CHINESE_TRADITIONAL,"zh"); + assertLanguage(Language.ENGLISH, "en"); + assertLanguage(Language.GERMAN, "de"); + assertLanguage(Language.JAPANESE, "ja"); + assertLanguage(Language.fromLanguageTag("jp") ,"jp"); + assertLanguage(Language.KOREAN, "ko"); + + // Since 2.0 + assertLanguage(Language.FRENCH, "fr"); + assertLanguage(Language.SPANISH, "es"); + assertLanguage(Language.ITALIAN, "it"); + assertLanguage(Language.PORTUGUESE, "pt"); + + //Since 2.2 + assertLanguage(Language.THAI, "th"); + } + + private void assertLanguage(Language expectedLanguage, String languageParameter) { + Query query = new Query("?query=test&language=" + languageParameter); + assertEquals(expectedLanguage, query.getModel().getParsingLanguage()); + + /* + This should also work and give something else than und/unknown + assertEquals("en", new Query("?query=test&language=en_US").getParsingLanguage().languageCode()); + assertEquals("nb_NO", new Query("?query=test&language=nb_NO").getParsingLanguage().languageCode()); + */ + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryTestCase.java new file mode 100644 index 00000000000..a7d50e9e6c1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryTestCase.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + + +import com.yahoo.search.Query; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; + +import java.util.Iterator; + + +/** + * Tests query trees + * + * @author bratseth + */ +public class QueryTestCase extends junit.framework.TestCase { + + public QueryTestCase(String name) { + super(name); + } + + /** Tests that query hash and equality is value dependent only */ + public void testQueryEquality() { + String query = "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy"; + String filter = "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar"; + + Item root1 = parseQuery(query, filter, Query.Type.ANY); + Item root2 = parseQuery(query, filter, Query.Type.ANY); + + assertEquals(root1.hashCode(), root2.hashCode()); + assertEquals(root1, root2); + } + + /** Check copy of query trees is a deep copy */ + public void testDeepCopy() { + Item root1 = parseQuery("a and b and (c or d) and e rank f andnot g", null, Query.Type.ADVANCED); + Item root2 = root1.clone(); + + assertTrue("Item.clone() should be a deep copy.",nonIdenticalTrees(root1, root2)); + } + + private static Item parseQuery(String query, String filter, Query.Type type) { + Parser parser = ParserFactory.newInstance(type, new ParserEnvironment()); + return parser.parse(new Parsable().setQuery(query).setFilter(filter)); + } + + // Control two equal trees does not have a "is" relationship for + // any element + private boolean nonIdenticalTrees(Item root1, Item root2) { + if (root1 instanceof CompositeItem) { + boolean nonID = root1 != root2; + Iterator i1 = ((CompositeItem) root1).getItemIterator(); + Iterator i2 = ((CompositeItem) root2).getItemIterator(); + + while (i1.hasNext() && nonID) { + nonID &= nonIdenticalTrees((Item) i1.next(), (Item) i2.next()); + } + return nonID; + + } else { + return root1 != root2; + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/RangeItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/RangeItemTestCase.java new file mode 100644 index 00000000000..6e7a47f681e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/RangeItemTestCase.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.RangeItem; +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; + +public class RangeItemTestCase { + + @Test + public void testRangeConstruction() { + verifyRange(new RangeItem(5, 7, 9, "a", true), 9, true); + verifyRange(new RangeItem(5,7, "a", true), 0, true); + verifyRange(new RangeItem(5,7, "a"), 0, false); + } + + private void verifyRange(RangeItem range, int limit, boolean isFromQuery) { + assertEquals(5, range.getFrom()); + assertEquals(7, range.getTo()); + assertEquals(limit, range.getHitLimit()); + assertEquals("a", range.getIndexName()); + if (range.getHitLimit() != 0) { + assertEquals("[5;7;9]", range.getNumber()); + } else { + assertEquals("[5;7]", range.getNumber()); + } + assertEquals(isFromQuery, range.isFromQuery()); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = range.encode(buffer); + ByteBuffer buffer2 = ByteBuffer.allocate(128); + int count2 = new IntItem(range.getNumber(), range.getIndexName(), range.isFromQuery()).encode(buffer2); + assertEquals(buffer, buffer2); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/SegmentItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/SegmentItemTestCase.java new file mode 100644 index 00000000000..ce4d0bff2ab --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/SegmentItemTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.WordItem; + +/** + * Functional test for the logic in items made up from a single block of text. + * + * @author steinar + */ +public class SegmentItemTestCase { + + @Test + public final void test() { + PhraseSegmentItem item = new PhraseSegmentItem("a b c", false, true); + item.addItem(new WordItem("a")); + item.addItem(new WordItem("b")); + item.addItem(new WordItem("c")); + assertEquals(100, item.getItem(0).getWeight()); + item.setWeight(150); + assertEquals(150, item.getItem(0).getWeight()); + assertEquals(item.getItem(0).getWeight(), item.getItem(1).getWeight()); + assertEquals(item.getItem(0).getWeight(), item.getItem(2).getWeight()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/WandItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/WandItemTestCase.java new file mode 100644 index 00000000000..294166de665 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/WandItemTestCase.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.io.HexDump; +import com.yahoo.prelude.query.textualrepresentation.Discloser; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.yahoo.prelude.query.*; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for WandItem. + */ +public class WandItemTestCase { + + private static double DELTA = 0.0000001; + + private static WandItem createSimpleItem() { + WandItem item = new WandItem("myfield", 10); + item.addToken("foo", 30); + item.setScoreThreshold(20); + item.setThresholdBoostFactor(2.0); + return item; + } + + @Test + public void requireThatWandItemCanBeConstructed() { + WandItem item = new WandItem("myfield", 10); + assertEquals("myfield", item.getIndexName()); + assertEquals(10, item.getTargetNumHits()); + assertEquals(0.0, item.getScoreThreshold(), DELTA); + assertEquals(1.0, item.getThresholdBoostFactor(), DELTA); + assertEquals(Item.ItemType.WAND, item.getItemType()); + } + + @Test + public void requireThatEncodeIsWorking() { + WandItem item = createSimpleItem(); + + ByteBuffer actual = ByteBuffer.allocate(128); + ByteBuffer expect = ByteBuffer.allocate(128); + expect.put((byte) 22).put((byte) 1); + Item.putString("myfield", expect); + expect.put((byte)10); // targetNumHits + expect.putDouble(20); // scoreThreshold + expect.putDouble(2.0); // thresholdBoostFactor + new PureWeightedString("foo", 30).encode(expect); + + assertEquals(2, item.encode(actual)); + + actual.flip(); + expect.flip(); + + assertTrue(actual.equals(expect)); + } + + @Test + public void requireThatToStringIsWorking() { + assertEquals("WAND(10,20.0,2.0) myfield{[30]:\"foo\"}", createSimpleItem().toString()); + } + + @Test + public void requireThatDiscloseIsWorking() { + class TestDiscloser implements Discloser { + public Map props = new HashMap<>(); + public void addProperty(String key, Object value) { + props.put(key, value); + } + public void setValue(Object value) {} + public void addChild(Item item) {} + } + TestDiscloser discloser = new TestDiscloser(); + createSimpleItem().disclose(discloser); + assertEquals(10, discloser.props.get("targetNumHits")); + assertEquals(20.0, discloser.props.get("scoreThreshold")); + assertEquals(2.0, discloser.props.get("thresholdBoostFactor")); + assertEquals("myfield", discloser.props.get("index")); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/WeightedSetItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/WeightedSetItemTestCase.java new file mode 100644 index 00000000000..90293e7bfe6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/WeightedSetItemTestCase.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.test; + +import com.yahoo.prelude.query.*; + +import java.nio.ByteBuffer; + +import com.yahoo.io.HexDump; + +public class WeightedSetItemTestCase extends junit.framework.TestCase { + + public WeightedSetItemTestCase(String name) { + super(name); + } + + public void testTokenAPI() { + WeightedSetItem ws = new WeightedSetItem("index"); + assertEquals(0, ws.getNumTokens()); + assertNull(ws.getTokenWeight("bogus")); + + // insert tokens + assertEquals(new Integer(1), ws.addToken("foo")); + assertEquals(new Integer(2), ws.addToken("bar", 2)); + assertEquals(new Integer(3), ws.addToken("baz", 3)); + + // check state + assertEquals(3, ws.getNumTokens()); + assertEquals(new Integer(1), ws.getTokenWeight("foo")); + assertEquals(new Integer(2), ws.getTokenWeight("bar")); + assertEquals(new Integer(3), ws.getTokenWeight("baz")); + + // add duplicate tokens + assertEquals(new Integer(2), ws.addToken("foo", 2)); + assertEquals(new Integer(3), ws.addToken("baz", 2)); + + // check state + assertEquals(3, ws.getNumTokens()); + assertEquals(new Integer(2), ws.getTokenWeight("foo")); + assertEquals(new Integer(2), ws.getTokenWeight("bar")); + assertEquals(new Integer(3), ws.getTokenWeight("baz")); + + // remove token + assertEquals(new Integer(2), ws.removeToken("bar")); + assertEquals(2, ws.getNumTokens()); + assertNull(ws.getTokenWeight("bar")); + + // remove non-existing token + assertNull(ws.removeToken("bogus")); + assertEquals(2, ws.getNumTokens()); + } + + public void testNegativeWeight() { + WeightedSetItem ws = new WeightedSetItem("index"); + assertEquals(new Integer(-10), ws.addToken("bad", -10)); + assertEquals(1, ws.getNumTokens()); + assertEquals(new Integer(-10), ws.getTokenWeight("bad")); + } + + static class FakeWSItem extends CompositeIndexedItem { + public FakeWSItem() { setIndexName("index"); } + public ItemType getItemType() { return ItemType.WEIGHTEDSET; } + public String getName() { return "WEIGHTEDSET"; } + public int getNumWords() { return 1; } + public String getIndexedString() { return ""; } + + public void add(String token, int weight) { + WordItem w = new WordItem(token, getIndexName()); + w.setWeight(weight); + super.addItem(w); + } + } + + public void testEncoding() { + WeightedSetItem item = new WeightedSetItem("index"); + // need 2 alternative reference encoding, as the encoding + // order is kept undefined to improve performance. + FakeWSItem ref1 = new FakeWSItem(); + FakeWSItem ref2 = new FakeWSItem(); + + item.addToken("foo", 10); + item.addToken("bar", 20); + ref1.add("foo", 10); + ref1.add("bar", 20); + ref2.add("bar", 20); + ref2.add("foo", 10); + + ByteBuffer actual = ByteBuffer.allocate(128); + ByteBuffer expect1 = ByteBuffer.allocate(128); + ByteBuffer expect2 = ByteBuffer.allocate(128); + expect1.put((byte)15).put((byte)2); + Item.putString("index", expect1); + new PureWeightedString("foo", 10).encode(expect1); + new PureWeightedString("bar", 20).encode(expect1); + expect2.put((byte)15).put((byte)2); + Item.putString("index", expect2); + new PureWeightedString("bar", 20).encode(expect2); + new PureWeightedString("foo", 10).encode(expect2); + + assertEquals(3, item.encode(actual)); + + actual.flip(); + expect1.flip(); + expect2.flip(); + + if (actual.equals(expect1)) { + assertFalse(actual.equals(expect2)); + } else { + assertTrue(actual.equals(expect2)); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java new file mode 100644 index 00000000000..377cb5a05b4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query.textualrepresentation.test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import junit.framework.TestCase; + +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.textualrepresentation.Discloser; +import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; + +/** + * Test of TextualQueryRepresentation. + * + * @author tonytv + */ +@SuppressWarnings({"rawtypes", "unchecked"}) +public class TextualQueryRepresentationTestCase extends TestCase { + private enum ExampleEnum { + example; + } + + private class MockItem extends Item { + private final String name; + + @Override + public void setIndexName(String index) { + } + + @Override + public ItemType getItemType() { + return null; + } + + @Override + public String getName() { + return name; + } + + @Override + public int encode(ByteBuffer buffer) { + return 0; + } + + @Override + public int getTermCount() { + return 0; + } + + @Override + protected void appendBodyString(StringBuilder buffer) { + } + + MockItem(String name) { + this.name = name; + } + } + + private final Item basic = new MockItem("basic") { + @Override + public void disclose(Discloser discloser) { + Map exampleMap = new HashMap<>(); + exampleMap.put(1, "one"); + exampleMap.put(2, "two"); + exampleMap.put(3, Arrays.asList('x', 'y', 'z')); + + discloser.addProperty("01", null); + discloser.addProperty("02", "a string."); + discloser.addProperty("03", 1234); + discloser.addProperty("04", true); + discloser.addProperty("05", ExampleEnum.example); + discloser.addProperty("06", new int[]{1, 2, 3}); + discloser.addProperty("07", Arrays.asList('x', 'y', 'z')); + discloser.addProperty("08", new ArrayList()); + discloser.addProperty("09", new HashSet(Arrays.asList(1, 2, 3))); + discloser.addProperty("10", exampleMap); + + discloser.setValue("example-value: \"12\""); + } + }; + + private final Item composite = new MockItem("composite") { + @Override + public void disclose(Discloser discloser) { + discloser.addProperty("reference", basic); + discloser.addChild(basic); + discloser.addChild(basic.clone()); + } + }; + + private String getTextualQueryRepresentation(Item item) { + return new TextualQueryRepresentation(item).toString(); + } + + public void testBasic() throws Exception { + String basicText = getTextualQueryRepresentation(basic); + assertEquals(getCorrect("basic.txt"), basicText); + + } + + public void testComposite() throws Exception { + String compositeText = getTextualQueryRepresentation(composite); + assertEquals(getCorrect("composite.txt"), compositeText); + } + + private String getCorrect(String filename) throws Exception { + BufferedReader reader = new BufferedReader(new FileReader( + "src/test/java/com/yahoo/prelude/query/textualrepresentation/test/" + filename)); + StringBuilder result = new StringBuilder(); + for (String line; (line = reader.readLine()) != null;) + result.append(line).append('\n'); + return result.toString(); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/basic.txt b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/basic.txt new file mode 100644 index 00000000000..998bfeb43a0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/basic.txt @@ -0,0 +1,3 @@ +basic[01=null 02="a string." 03=1234 04=true 05=example 06=(1 2 3) 07=("x" "y" "z") 08=() 09=(1 2 3) 10=map(1=>"one" 2=>"two" 3=>("x" "y" "z"))]{ + "example-value: \"12\"" +} diff --git a/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/composite.txt b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/composite.txt new file mode 100644 index 00000000000..9065fd6d5b5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/composite.txt @@ -0,0 +1,8 @@ +composite[reference=0]{ + basic[%id=0 01=null 02="a string." 03=1234 04=true 05=example 06=(1 2 3) 07=("x" "y" "z") 08=() 09=(1 2 3) 10=map(1=>"one" 2=>"two" 3=>("x" "y" "z"))]{ + "example-value: \"12\"" + } + basic[01=null 02="a string." 03=1234 04=true 05=example 06=(1 2 3) 07=("x" "y" "z") 08=() 09=(1 2 3) 10=map(1=>"one" 2=>"two" 3=>("x" "y" "z"))]{ + "example-value: \"12\"" + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java new file mode 100644 index 00000000000..497e1ca36ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexFactsFactory; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.parser.TestLinguistics; +import com.yahoo.prelude.querytransform.CJKSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.searchchain.Execution; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author Steinar Knutsen + */ +public class CJKSearcherTestCase { + + private final IndexFacts indexFacts = IndexFactsFactory.newInstance("file:src/test/java/com/yahoo/prelude/" + + "querytransform/test/cjk-index-info.cfg", null); + + @Test + public void testTermWeight() { + assertTransformed("efg!10", "SAND e!10 fg!10", + Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, TestLinguistics.INSTANCE); + } + + /** + * Overlapping tokens splits some sequences of "bcd" into "bc" "cd" instead of e.g. "b", + * "cd". This improves recall in some cases. Vespa + * must combine overlapping tokens as PHRASE, not AND to avoid a too high recall because of the token overlap. + */ + @Test + public void testCjkQueryWithOverlappingTokens() { + // The test language segmenter will segment "bcd" into the overlapping tokens "bc" "cd" + assertTransformed("bcd", "'bc cd'", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, + TestLinguistics.INSTANCE); + + // While "efg" will be segmented into one of the standard options, "e" "fg" + assertTransformed("efg", "SAND e fg", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, + TestLinguistics.INSTANCE); + } + + private void assertTransformed(String queryString, String expected, Query.Type mode, Language actualLanguage, + Language queryLanguage, Linguistics linguistics) { + Parser parser = ParserFactory.newInstance(mode, new ParserEnvironment() + .setIndexFacts(indexFacts) + .setLinguistics(linguistics)); + Item root = parser.parse(new Parsable().setQuery(queryString).setLanguage(actualLanguage)).getRoot(); + assertFalse(root instanceof NullItem); + + Query query = new Query("?language=" + queryLanguage.languageCode()); + query.getModel().getQueryTree().setRoot(root); + + new Execution(new Chain(new CJKSearcher()), + new Execution.Context(null, indexFacts, null, null, linguistics)).search(query); + assertEquals(expected, query.getModel().getQueryTree().getRoot().toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CollapsePhraseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CollapsePhraseSearcherTestCase.java new file mode 100644 index 00000000000..b1c9bebba73 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CollapsePhraseSearcherTestCase.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.CollapsePhraseSearcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +/** + * Check CollapsePhraseSearcher works and only is triggered when it + * should. + * + * @author Steinar Knutsen + */ +public class CollapsePhraseSearcherTestCase extends junit.framework.TestCase { + + public CollapsePhraseSearcherTestCase(String name) { + super(name); + } + + public void testSimplePositive() { + PhraseItem root = new PhraseItem(); + root.addItem(new WordItem("abc")); + assertEquals("abc", + transformQuery(root)); + } + + public void testPositive1() { + AndItem root = new AndItem(); + root.addItem(new WordItem("a")); + PhraseItem embedded = new PhraseItem(); + embedded.addItem(new WordItem("bcd")); + root.addItem(embedded); + root.addItem(new WordItem("e")); + assertEquals("AND a bcd e", + transformQuery(root)); + } + + public void testPositive2() { + AndItem root = new AndItem(); + root.addItem(new WordItem("a")); + CompositeItem embedded = new AndItem(); + embedded.addItem(new WordItem("bcd")); + CompositeItem phrase = new PhraseItem(); + phrase.addItem(new WordItem("def")); + embedded.addItem(phrase); + root.addItem(embedded); + root.addItem(new WordItem("e")); + assertEquals("AND a (AND bcd def) e", + transformQuery(root)); + } + public void testNoTerms() { + assertEquals("NULL", transformQuery("?query=" + enc("\"\""))); + } + + public void testNegative1() { + assertEquals("\"abc def\"", transformQuery("?query=" + enc("\"abc def\""))); + } + + public void testNegative2() { + assertEquals("AND a \"abc def\" b", transformQuery("?query=" + enc("a \"abc def\" b"))); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + public void testNegative3() { + AndItem root = new AndItem(); + root.addItem(new WordItem("a")); + CompositeItem embedded = new AndItem(); + embedded.addItem(new WordItem("bcd")); + CompositeItem phrase = new PhraseItem(); + phrase.addItem(new WordItem("def")); + phrase.addItem(new WordItem("ghi")); + embedded.addItem(phrase); + root.addItem(embedded); + root.addItem(new WordItem("e")); + assertEquals("AND a (AND bcd \"def ghi\") e", + transformQuery(root)); + } + private String transformQuery(String rawQuery) { + CollapsePhraseSearcher searcher = new CollapsePhraseSearcher(); + Query query = new Query(rawQuery); + new Execution(searcher, Execution.Context.createContextStub()).search(query); + Item newRoot = query.getModel().getQueryTree().getRoot(); + if (newRoot != null) + return newRoot.toString(); + else + return null; + } + + private String transformQuery(Item queryTree) { + CollapsePhraseSearcher searcher = new CollapsePhraseSearcher(); + Query query = new Query(); + query.getModel().getQueryTree().setRoot(queryTree); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + Item newRoot = query.getModel().getQueryTree().getRoot(); + if (newRoot != null) + return newRoot.toString(); + else + return null; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java new file mode 100644 index 00000000000..3e0597483eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/IndexCombinatorTestCase.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.search.Query; +import com.yahoo.prelude.querytransform.IndexCombinatorSearcher; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; +import junit.framework.TestCase; + +/** + * Control query transformations when doing index name expansion in QRS. + * + * @author Steinar Knutsen + */ +public class IndexCombinatorTestCase extends TestCase { + + private Searcher transformer; + private IndexFacts f; + + public IndexCombinatorTestCase(String arg0) { + super(arg0); + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + transformer = new IndexCombinatorSearcher(); + f = new IndexFacts(); + f.addIndex("one", "z"); + Index i = new Index("default"); + i.addCommand("match-group a i"); + f.addIndex("one", i); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testDoNothing() { + Result r = search("?query=z:y"); + assertEquals("z:y", r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + private Result search(String query) { + return new Execution(transformer, Execution.Context.createContextStub(f)).search(new Query(QueryTestCase.httpEncode(query))); + } + + public void testBasic() { + Result r = search("?query=y"); + assertEquals("OR a:y i:y", r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicPair() { + Result r = search("?query=x y"); + assertEquals( + "OR (AND a:x a:y) (AND a:x i:y) (AND i:x a:y) (AND i:x i:y)", r + .getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicTriplet() { + Result r = search("?query=x y z"); + assertEquals("AND (OR a:x i:x) (OR a:y i:y) (OR a:z i:z)", r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicMixedSinglet() { + Result r = search("?query=x z:q"); + assertEquals("OR (AND a:x z:q) (AND i:x z:q)", r.getQuery().getModel().getQueryTree().getRoot() + .toString()); + } + + public void testBasicMixedPair() { + Result r = search("?query=x y z:q"); + assertEquals( + "OR (AND a:x a:y z:q) (AND a:x i:y z:q) (AND i:x a:y z:q) (AND i:x i:y z:q)", + r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicMixedTriplet() { + Result r = search("?query=x y z:q r"); + assertEquals("AND (OR a:x i:x) (OR a:y i:y) z:q (OR a:r i:r)", r + .getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicOr() { + Result r = search("?query=x y&type=any"); + assertEquals("OR a:y i:y a:x i:x", r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicPhrase() { + Result r = search("?query=\"x y\""); + assertEquals("OR a:x y i:x y", r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testPhraseAndTerm() { + Result r = search("?query=\"x y\" z"); + assertEquals( + "OR (AND a:x y a:z) (AND a:x y i:z) (AND i:x y a:z) (AND i:x y i:z)", + r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testBasicNot() { + Result r = search("?query=+x -y"); + assertEquals("+(OR a:x i:x) -(OR a:y i:y)", r.getQuery().getModel().getQueryTree().getRoot() + .toString()); + } + + public void testLessBasicNot() { + Result r = search("?query=a and b andnot c&type=adv"); + assertEquals( + "+(OR (AND a:a a:b) (AND a:a i:b) (AND i:a a:b) (AND i:a i:b)) -(OR a:c i:c)", + r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testLongerAndInPositive() { + Result r = search("?query=a and b and c andnot d&type=adv"); + assertEquals( + "+(AND (OR a:a i:a) (OR a:b i:b) (OR a:c i:c)) -(OR a:d i:d)", r + .getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testTreeInNegativeBranch() { + Result r = search("?query=a andnot (b and c)&type=adv"); + assertEquals("+(OR a:a i:a) -(AND (OR a:b i:b) (OR a:c i:c))", r + .getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testSomeTerms() { + Result r = search("?query=a b -c +d g.h \"abc def\" z:q"); + assertEquals( + "+(AND (OR a:a i:a) (OR a:b i:b) (OR a:d i:d) (OR a:g h i:g h) (OR a:abc def i:abc def) z:q) -(OR a:c i:c)", + r.getQuery().getModel().getQueryTree().getRoot().toString()); + } + + public void testMixedIndicesAndAttributes() { + String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg"; + ConfigGetter getter = new ConfigGetter<>(IndexInfoConfig.class); + IndexInfoConfig config = getter.getConfig(indexInfoConfigID); + IndexFacts facts = new IndexFacts(new IndexModel(config, (QrSearchersConfig)null)); + + Result r = new Execution(transformer, Execution.Context.createContextStub(facts)).search(new Query(QueryTestCase.httpEncode("?query=\"a b\""))); + assertEquals("OR default:\"a b\" attribute1:a b attribute2:a b", r + .getQuery().getModel().getQueryTree().getRoot().toString()); + r = new Execution(transformer, Execution.Context.createContextStub(facts)).search(new Query(QueryTestCase.httpEncode("?query=\"a b\" \"c d\""))); + assertEquals( + "OR (AND default:\"a b\" default:\"c d\")" + + " (AND default:\"a b\" attribute1:c d)" + + " (AND default:\"a b\" attribute2:c d)" + + " (AND attribute1:a b default:\"c d\")" + + " (AND attribute1:a b attribute1:c d)" + + " (AND attribute1:a b attribute2:c d)" + + " (AND attribute2:a b default:\"c d\")" + + " (AND attribute2:a b attribute1:c d)" + + " (AND attribute2:a b attribute2:c d)", + r.getQuery().getModel().getQueryTree().getRoot().toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java new file mode 100644 index 00000000000..666e89cd324 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.search.Query; +import com.yahoo.prelude.querytransform.LiteralBoostSearcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * Tests the complete field match query transformer + * + * @author Steinar Knutsen + */ +public class LiteralBoostSearcherTestCase { + + @Test + public void testSimpleQueryWithBoost() { + assertEquals("RANK abc default_literal:abc", + transformQuery("?query=abc&source=cluster1&restrict=type1")); + } + + @Test + public void testSimpleQueryNoBoost() { + assertEquals("abc", + transformQuery("?query=abc&source=cluster1&restrict=type2")); + } + + @Test + public void testQueryWithExplicitIndex() { + assertEquals("RANK absolute:abc absolute_literal:abc", + transformQuery("?query=absolute:abc&source=cluster1&restrict=type1")); + } + + @Test + public void testQueryWithExplicitIndexNoBoost() { + assertEquals("absolute:abc", + transformQuery("?query=absolute:abc&source=cluster1&restrict=type2")); + } + + @Test + public void testQueryWithNegativeBranch() { + assertEquals("RANK (+(AND abc def) -ghi) "+ + "default_literal:abc default_literal:def", + transformQuery("?query=abc and def andnot ghi&type=adv&source=cluster1&restrict=type1")); + } + + @Test + public void testJumbledQuery() { + assertEquals + ("RANK (OR (+(OR abc def) -ghi) jkl) " + + "default_literal:abc default_literal:def default_literal:jkl", + transformQuery("?query=abc or def andnot ghi or jkl&type=adv&source=cluster1&restrict=type1")); + } + + @Test + public void testTermindexQuery() { + assertEquals("RANK (+(AND a b d) -c) default_literal:a "+ + "default_literal:b default_literal:d", + transformQuery("?query=a b -c d&source=cluster1&restrict=type1")); + } + + @Test + public void testQueryWithoutBoost() { + assertEquals("RANK (AND \"nonexistant a\" \"nonexistant b\") default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", + transformQuery("?query=nonexistant:a nonexistant:b&source=cluster1&restrict=type1")); + } + + private String transformQuery(String rawQuery) { + Query query = new Query(QueryTestCase.httpEncode(rawQuery)); + new Execution(new LiteralBoostSearcher(), Execution.Context.createContextStub(createIndexFacts())).search(query); + return query.getModel().getQueryTree().getRoot().toString(); + } + + private IndexFacts createIndexFacts() { + Map> clusters = new LinkedHashMap<>(); + clusters.put("cluster1", Arrays.asList("type1", "type2", "type3")); + clusters.put("cluster2", Arrays.asList("type4", "type5")); + Map searchDefs = new LinkedHashMap<>(); + searchDefs.put("type1", createSearchDefinitionWithFields("type1", true)); + searchDefs.put("type2", createSearchDefinitionWithFields("type2", false)); + searchDefs.put("type3", new SearchDefinition("type3")); + searchDefs.put("type3", new SearchDefinition("type3")); + searchDefs.put("type4", new SearchDefinition("type4")); + searchDefs.put("type5", new SearchDefinition("type5")); + SearchDefinition union = new SearchDefinition("union"); + return new IndexFacts(new IndexModel(clusters, searchDefs, union)); + } + + private SearchDefinition createSearchDefinitionWithFields(String name, boolean literalBoost) { + SearchDefinition type = new SearchDefinition(name); + Index defaultIndex = new Index("default"); + defaultIndex.setLiteralBoost(literalBoost); + type.addIndex(defaultIndex); + Index absoluteIndex = new Index("absolute"); + absoluteIndex.setLiteralBoost(literalBoost); + type.addIndex(absoluteIndex); + return type; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NoRankingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NoRankingSearcherTestCase.java new file mode 100644 index 00000000000..c1c0371ab81 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NoRankingSearcherTestCase.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import junit.framework.TestCase; + +import com.yahoo.search.Query; +import com.yahoo.prelude.querytransform.NoRankingSearcher; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +public class NoRankingSearcherTestCase extends TestCase { + + Searcher s = new NoRankingSearcher(); + + public void testDoSearch() { + Query q = new Query("?query=a&sorting=%2ba%20-b&ranking=hello"); + assertEquals("hello", q.getRanking().getProfile()); + new Execution(s, Execution.Context.createContextStub()).search(q); + assertEquals("unranked", q.getRanking().getProfile()); + } + + public void testSortOnRelevanceAscending() { + Query q = new Query("?query=a&sorting=%2ba%20-b%20-[rank]&ranking=hello"); + new Execution(s, Execution.Context.createContextStub()).search(q); + assertEquals("hello", q.getRanking().getProfile()); + } + + public void testSortOnRelevanceDescending() { + Query q = new Query("?query=a&sorting=%2ba%20-b%20-[rank]&ranking=hello"); + new Execution(s, Execution.Context.createContextStub()).search(q); + assertEquals("hello", q.getRanking().getProfile()); + } + + public void testNoSorting() { + Query q = new Query("?query=a"); + new Execution(s, Execution.Context.createContextStub()).search(q); + assertEquals("default", q.getRanking().getProfile()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NonPhrasingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NonPhrasingSearcherTestCase.java new file mode 100644 index 00000000000..27345d0b6f6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NonPhrasingSearcherTestCase.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.NonPhrasingSearcher; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests non-phrasing + * + * @author bratseth + */ +public class NonPhrasingSearcherTestCase extends junit.framework.TestCase { + + private Searcher searcher; + + public NonPhrasingSearcherTestCase(String name) { + super(name); + } + + public void testSingleWordNonPhrasing() { + searcher= + new NonPhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query("?query=void+aword+kanoo"); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertEquals("AND void kanoo", query.getModel().getQueryTree().getRoot().toString()); + } + + public void testMultipleWordNonPhrasing() { + searcher= + new NonPhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query("?query=void+tudor+vidor+kanoo"); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + CompositeItem item=(CompositeItem) query.getModel().getQueryTree().getRoot(); + assertEquals(2,item.getItemCount()); + assertEquals("void",((WordItem)item.getItem(0)).getWord()); + assertEquals("kanoo",((WordItem)item.getItem(1)).getWord()); + } + + public void testNoNonPhrasingIfNoOtherPhrases() { + searcher= + new NonPhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query("?query=tudor+vidor"); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + CompositeItem item=(CompositeItem) query.getModel().getQueryTree().getRoot(); + assertEquals(2,item.getItemCount()); + assertEquals("tudor",((WordItem)item.getItem(0)).getWord()); + assertEquals("vidor",((WordItem)item.getItem(1)).getWord()); + } + + public void testNoNonPhrasingIfSuggestOnly() { + searcher= + new NonPhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query("?query=void+tudor+vidor+kanoo&suggestonly=true"); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + CompositeItem item=(CompositeItem) query.getModel().getQueryTree().getRoot(); + assertEquals(4,item.getItemCount()); + assertEquals("void",((WordItem)item.getItem(0)).getWord()); + assertEquals("tudor",((WordItem)item.getItem(1)).getWord()); + assertEquals("vidor",((WordItem)item.getItem(2)).getWord()); + assertEquals("kanoo",((WordItem)item.getItem(3)).getWord()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java new file mode 100644 index 00000000000..caf99bb369e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/NormalizingSearcherTestCase.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.querytransform.NormalizingSearcher; +import com.yahoo.search.searchchain.Execution; + +import org.junit.Test; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class NormalizingSearcherTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + + @Test + public void testNoNormalizingNecssary() { + Query query = new Query("/search?query=bilen&search=cluster1&restrict=type1"); + createExecution().search(query); + assertEquals("bilen", query.getModel().getQueryTree().getRoot().toString()); + } + + @Test + public void testAttributeQuery() { + Query query = new Query("/search?query=attribute:" + enc("b\u00e9yonc\u00e8 b\u00e9yonc\u00e8") + "&search=cluster1&restrict=type1"); + createExecution().search(query); + assertEquals("AND attribute:b\u00e9yonc\u00e8 beyonce", query.getModel().getQueryTree().getRoot().toString()); + } + + @Test + public void testOneTermNormalizing() { + Query query = new Query("/search?query=b\u00e9yonc\u00e8&search=cluster1&restrict=type1"); + createExecution().search(query); + assertEquals("beyonce", query.getModel().getQueryTree().getRoot().toString()); + } + + @Test + public void testOneTermNoNormalizingDifferentSearchDef() { + Query query = new Query("/search?query=b\u00e9yonc\u00e8&search=cluster1&restrict=type2"); + createExecution().search(query); + assertEquals("béyoncè", query.getModel().getQueryTree().getRoot().toString()); + } + + @Test + public void testTwoTermQuery() throws UnsupportedEncodingException { + Query query = new Query("/search?query=" + enc("b\u00e9yonc\u00e8 beyonc\u00e9") + "&search=cluster1&restrict=type1"); + createExecution().search(query); + assertEquals("AND beyonce beyonce", query.getModel().getQueryTree().getRoot().toString()); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testPhraseQuery() { + Query query = new Query("/search?query=" + enc("\"b\u00e9yonc\u00e8 beyonc\u00e9\"") + "&search=cluster1&restrict=type1"); + query.setTraceLevel(2); + createExecution().search(query); + assertEquals("\"beyonce beyonce\"", query.getModel().getQueryTree().getRoot().toString()); + } + + @Test + public void testLiteralBoost() { + Query query = new Query("/search?query=nop&search=cluster1&restrict=type1"); + List terms = new ArrayList<>(); + Substring origin = new Substring(0, 5, "h\u00F4tels"); + terms.add(new WordAlternativesItem.Alternative("h\u00F4tels", 1.0d)); + terms.add(new WordAlternativesItem.Alternative("h\u00F4tel", 0.7d)); + query.getModel().getQueryTree().setRoot(new WordAlternativesItem("default", true, origin, terms)); + createExecution().search(query); + WordAlternativesItem w = (WordAlternativesItem) query.getModel().getQueryTree().getRoot(); + assertEquals(4, w.getAlternatives().size()); + boolean foundHotel = false; + for (WordAlternativesItem.Alternative a : w.getAlternatives()) { + if ("hotel".equals(a.word)) { + foundHotel = true; + assertEquals(.7d * .7d, a.exactness, 1e-15); + } + } + assertTrue("Did not find the expected normalized form \"hotel\".", foundHotel); + } + + + @Test + public void testPhraseSegmentNormalization() { + Query query = new Query("/search?query=&search=cluster1&restrict=type1"); + PhraseSegmentItem phraseSegment = new PhraseSegmentItem("default", false, false); + phraseSegment.addItem(new WordItem("net")); + query.getModel().getQueryTree().setRoot(phraseSegment); + assertEquals("'net'", query.getModel().getQueryTree().getRoot().toString()); + createExecution().search(query); + assertEquals("'net'", query.getModel().getQueryTree().getRoot().toString()); + } + + private Execution createExecution() { + return new Execution(new NormalizingSearcher(linguistics), + Execution.Context.createContextStub(null, createIndexFacts(), linguistics)); + } + + private IndexFacts createIndexFacts() { + Map> clusters = new LinkedHashMap<>(); + clusters.put("cluster1", Arrays.asList("type1", "type2", "type3")); + clusters.put("cluster2", Arrays.asList("type4", "type5")); + Map searchDefs = new LinkedHashMap<>(); + searchDefs.put("type1", createSearchDefinitionWithFields("type1", true)); + searchDefs.put("type2", createSearchDefinitionWithFields("type2", false)); + searchDefs.put("type3", new SearchDefinition("type3")); + searchDefs.put("type3", new SearchDefinition("type3")); + searchDefs.put("type4", new SearchDefinition("type4")); + searchDefs.put("type5", new SearchDefinition("type5")); + SearchDefinition union = new SearchDefinition("union"); + return new IndexFacts(new IndexModel(clusters, searchDefs, union)); + } + + private SearchDefinition createSearchDefinitionWithFields(String name, boolean normalize) { + SearchDefinition type = new SearchDefinition(name); + + Index defaultIndex = new Index("default"); + defaultIndex.setNormalize(normalize); + type.addIndex(defaultIndex); + + Index absoluteIndex = new Index("absolute"); + absoluteIndex.setNormalize(normalize); + type.addIndex(absoluteIndex); + + Index normalizercheckIndex = new Index("normalizercheck"); + normalizercheckIndex.setNormalize(normalize); + type.addIndex(normalizercheckIndex); + + Index attributeIndex = new Index("attribute"); + attributeIndex.setAttribute(true); + type.addIndex(attributeIndex); + + return type; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhraseMatcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhraseMatcherTestCase.java new file mode 100644 index 00000000000..0173a42f592 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhraseMatcherTestCase.java @@ -0,0 +1,252 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.PhraseMatcher; + +import java.util.List; + +/** + * @author Jon Bratseth + */ +public class PhraseMatcherTestCase extends junit.framework.TestCase { + + public PhraseMatcherTestCase(String name) { + super(name); + } + + public void testSingleItemMatching() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + matcher.setMatchSingleItems(true); + List matches=matcher.matchPhrases(new WordItem("aword")); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(1,match.getLength()); + assertEquals("",match.getData()); + assertEquals(null,match.getOwner()); + assertEquals(0,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem("aword"),i.next()); + assertNull(i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testSingleItemMatchingCaseInsensitive() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + matcher.setMatchSingleItems(true); + final String mixedCase = "aWoRD"; + List matches=matcher.matchPhrases(new WordItem(mixedCase)); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(1,match.getLength()); + assertEquals("",match.getData()); + assertEquals(null,match.getOwner()); + assertEquals(0,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem(mixedCase),i.next()); + assertNull(i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testSingleItemMatchingWithPluralIgnore() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + matcher.setMatchSingleItems(true); + List matches=matcher.matchPhrases(new WordItem("awords")); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(1,match.getLength()); + assertEquals("",match.getData()); + assertEquals(null,match.getOwner()); + assertEquals(0,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem("awords"),i.next()); + assertEquals("aword",i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testSingleItemMatchingCaseInsensitiveWithPluralIgnore() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + matcher.setMatchSingleItems(true); + final String mixedCase = "aWoRDS"; + List matches=matcher.matchPhrases(new WordItem(mixedCase)); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(1,match.getLength()); + assertEquals("",match.getData()); + assertEquals(null,match.getOwner()); + assertEquals(0,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem(mixedCase),i.next()); + assertEquals("aword",i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testPhraseMatching() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + AndItem and=new AndItem(); + and.addItem(new WordItem("noisebefore")); + and.addItem(new WordItem("this")); + and.addItem(new WordItem("is")); + and.addItem(new WordItem("a")); + and.addItem(new WordItem("test")); + and.addItem(new WordItem("noiseafter")); + List matches=matcher.matchPhrases(and); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(4,match.getLength()); + assertEquals("",match.getData()); + assertEquals(and,match.getOwner()); + assertEquals(1,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem("this"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("is"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("a"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("test"),i.next()); + assertEquals(null,i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testPhraseMatchingCaseInsensitive() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + AndItem and=new AndItem(); + and.addItem(new WordItem("noisebefore")); + final String firstWord = "thIs"; + and.addItem(new WordItem(firstWord)); + final String secondWord = "Is"; + and.addItem(new WordItem(secondWord)); + final String thirdWord = "A"; + and.addItem(new WordItem(thirdWord)); + final String fourthWord = "tEst"; + and.addItem(new WordItem(fourthWord)); + and.addItem(new WordItem("noiseafter")); + List matches=matcher.matchPhrases(and); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(4,match.getLength()); + assertEquals("",match.getData()); + assertEquals(and,match.getOwner()); + assertEquals(1,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem(firstWord),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem(secondWord),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem(thirdWord),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem(fourthWord),i.next()); + assertEquals(null,i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testPhraseMatchingWithNumber() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + AndItem and=new AndItem(); + and.addItem(new WordItem("noisebefore")); + and.addItem(new WordItem("this")); + and.addItem(new WordItem("is")); + and.addItem(new IntItem("3")); + and.addItem(new WordItem("tests")); + and.addItem(new WordItem("noiseafter")); + List matches=matcher.matchPhrases(and); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(4,match.getLength()); + assertEquals("",match.getData()); + assertEquals(and,match.getOwner()); + assertEquals(1,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem("this"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("is"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new IntItem("3"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("tests"),i.next()); + assertEquals(null,i.getReplace()); + assertFalse(i.hasNext()); + } + + public void testPhraseMatchingWithPluralIgnore() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + AndItem and=new AndItem(); + and.addItem(new WordItem("noisebefore")); + and.addItem(new WordItem("thi")); + and.addItem(new WordItem("is")); + and.addItem(new WordItem("a")); + and.addItem(new WordItem("tests")); + and.addItem(new WordItem("noiseafter")); + List matches=matcher.matchPhrases(and); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(4,match.getLength()); + assertEquals("",match.getData()); + assertEquals(and,match.getOwner()); + assertEquals(1,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem("thi"),i.next()); + assertEquals("this",i.getReplace()); + assertEquals(new WordItem("is"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("a"),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem("tests"),i.next()); + assertEquals("test",i.getReplace()); + assertFalse(i.hasNext()); + } + + + public void testPhraseMatchingCaseInsensitiveWithPluralIgnore() { + PhraseMatcher matcher=new PhraseMatcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa",true); + AndItem and=new AndItem(); + and.addItem(new WordItem("noisebefore")); + final String firstWord = "thI"; + and.addItem(new WordItem(firstWord)); + final String secondWord = "Is"; + and.addItem(new WordItem(secondWord)); + final String thirdWord = "A"; + and.addItem(new WordItem(thirdWord)); + final String fourthWord = "tEsts"; + and.addItem(new WordItem(fourthWord)); + and.addItem(new WordItem("noiseafter")); + List matches=matcher.matchPhrases(and); + + assertNotNull(matches); + assertEquals(1,matches.size()); + PhraseMatcher.Phrase match=(PhraseMatcher.Phrase)matches.get(0); + assertEquals(4,match.getLength()); + assertEquals("",match.getData()); + assertEquals(and,match.getOwner()); + assertEquals(1,match.getStartIndex()); + PhraseMatcher.Phrase.MatchIterator i=match.itemIterator(); + assertEquals(new WordItem(firstWord),i.next()); + assertEquals("this",i.getReplace()); + assertEquals(new WordItem(secondWord),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem(thirdWord),i.next()); + assertEquals(null,i.getReplace()); + assertEquals(new WordItem(fourthWord),i.next()); + assertEquals("test",i.getReplace()); + assertFalse(i.hasNext()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhrasingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhrasingSearcherTestCase.java new file mode 100644 index 00000000000..e94784eb39c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/PhrasingSearcherTestCase.java @@ -0,0 +1,178 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.PhrasingSearcher; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests phrasing stuff + * + * @author bratseth + * @author Einar M R Rosenvinge + */ +public class PhrasingSearcherTestCase extends junit.framework.TestCase { + + private Searcher searcher; + + public PhrasingSearcherTestCase(String name) { + super(name); + } + + @SuppressWarnings("deprecation") + public void testTotalPhrasing() { + + searcher= + new PhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query(); + AndItem andItem=new AndItem(); + andItem.addItem(new WordItem("tudor","someindex")); + andItem.addItem(new WordItem("vidor","someindex")); + query.getModel().getQueryTree().setRoot(andItem); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + Item item=((CompositeItem) query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue(item instanceof PhraseItem); + PhraseItem phrase=(PhraseItem)item; + assertEquals(2,phrase.getItemCount()); + assertEquals("tudor",phrase.getWordItem(0).getWord()); + assertEquals("vidor",phrase.getWordItem(1).getWord()); + assertEquals("someindex",phrase.getIndexName()); + } + + @SuppressWarnings("deprecation") + public void testPartialPhrasing() { + + searcher= + new PhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query("?query=void%20tudor%20vidor%20kanoo"); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + CompositeItem item=(CompositeItem) query.getModel().getQueryTree().getRoot(); + assertEquals("void",((WordItem)item.getItem(0)).getWord()); + assertEquals("kanoo",((WordItem)item.getItem(2)).getWord()); + + PhraseItem phrase=(PhraseItem)item.getItem(1); + assertEquals(2,phrase.getItemCount()); + assertEquals("tudor",phrase.getWordItem(0).getWord()); + assertEquals("vidor",phrase.getWordItem(1).getWord()); + } + + public void testPartialPhrasingSuggestOnly() { + + searcher= + new PhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query("?query=void%20tudor%20vidor%20kanoo&suggestonly=true"); + new Execution(searcher, Execution.Context.createContextStub()).search(query); + CompositeItem item=(CompositeItem) query.getModel().getQueryTree().getRoot(); + assertEquals("void", ((WordItem)item.getItem(0)).getWord()); + assertEquals("tudor",((WordItem)item.getItem(1)).getWord()); + assertEquals("vidor",((WordItem)item.getItem(2)).getWord()); + assertEquals("kanoo",((WordItem)item.getItem(3)).getWord()); + } + + public void testNoPhrasingIfDifferentIndices() { + + searcher= + new PhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query(); + AndItem andItem=new AndItem(); + andItem.addItem(new WordItem("tudor","someindex")); + andItem.addItem(new WordItem("vidor","anotherindex")); + query.getModel().getQueryTree().setRoot(andItem); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + CompositeItem item=(CompositeItem) query.getModel().getQueryTree().getRoot(); + + assertTrue(item.getItem(0) instanceof WordItem); + WordItem word=(WordItem)item.getItem(0); + assertEquals("tudor",word.getWord()); + + assertTrue(item.getItem(1) instanceof WordItem); + word=(WordItem)item.getItem(1); + assertEquals("vidor",word.getWord()); + } + + public void testMultiplePhrases() { + + searcher= + new PhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query(); + AndItem andItem=new AndItem(); + andItem.addItem(new WordItem("tudor","someindex")); + andItem.addItem(new WordItem("tudor","someindex")); + andItem.addItem(new WordItem("vidor","someindex")); + andItem.addItem(new WordItem("vidor","someindex")); + + OrItem orItem=new OrItem(); + andItem.addItem(orItem); + + orItem.addItem(new WordItem("tudor")); + AndItem andItem2=new AndItem(); + andItem2.addItem(new WordItem("this","anotherindex")); + andItem2.addItem(new WordItem("is","anotherindex")); + andItem2.addItem(new WordItem("a","anotherindex")); + andItem2.addItem(new WordItem("test","anotherindex")); + andItem2.addItem(new WordItem("tudor","anotherindex")); + andItem2.addItem(new WordItem("vidor","anotherindex")); + orItem.addItem(andItem2); + orItem.addItem(new WordItem("vidor")); + + + query.getModel().getQueryTree().setRoot(andItem); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertEquals("AND someindex:tudor someindex:\"tudor vidor\" someindex:vidor (OR tudor (AND anotherindex:\"this is a test\" anotherindex:\"tudor vidor\") vidor)", query.getModel().getQueryTree().getRoot().toString()); + } + + public void testNoDetection() { + + searcher= + new PhrasingSearcher("src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa"); + + Query query=new Query(); + AndItem andItem=new AndItem(); + andItem.addItem(new WordItem("no")); + andItem.addItem(new WordItem("such")); + andItem.addItem(new WordItem("phrase")); + query.getModel().getQueryTree().setRoot(andItem); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + + assertEquals("AND no such phrase", query.getModel().getQueryTree().getRoot().toString()); + + } + + public void testNoFileNoChange() { + searcher = new PhrasingSearcher(""); + + Query query=new Query(); + AndItem andItem=new AndItem(); + andItem.addItem(new WordItem("no", "anindex")); + andItem.addItem(new WordItem("such", "anindex")); + andItem.addItem(new WordItem("phrase", "indexo")); + OrItem orItem = new OrItem(); + orItem.addItem(new WordItem("habla")); + orItem.addItem(new WordItem("babla")); + orItem.addItem(new WordItem("habla")); + andItem.addItem(orItem); + query.getModel().getQueryTree().setRoot(andItem); + + new Execution(searcher, Execution.Context.createContextStub()).search(query); + + assertEquals("AND anindex:no anindex:such indexo:phrase (OR habla babla habla)", query.getModel().getQueryTree().getRoot().toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java new file mode 100644 index 00000000000..58b46662e8b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.search.Query; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author balder + */ +public class QueryRewriteTestCase { + + @Test + public void requireThatOptimizeByRestrictSimplifiesORItemsThatHaveFullRecall() { + assertRewritten("sddocname:foo OR sddocname:bar OR sddocname:baz", "foo", "sddocname:foo"); + assertRewritten("sddocname:foo OR sddocname:bar OR sddocname:baz", "bar", "sddocname:bar"); + assertRewritten("sddocname:foo OR sddocname:bar OR sddocname:baz", "baz", "sddocname:baz"); + + assertRewritten("lhs OR (sddocname:foo OR sddocname:bar OR sddocname:baz)", "foo", "sddocname:foo"); + assertRewritten("lhs OR (sddocname:foo OR sddocname:bar OR sddocname:baz)", "bar", "sddocname:bar"); + assertRewritten("lhs OR (sddocname:foo OR sddocname:bar OR sddocname:baz)", "baz", "sddocname:baz"); + + assertRewritten("lhs AND (sddocname:foo OR sddocname:bar OR sddocname:baz)", "foo", "lhs"); + assertRewritten("lhs AND (sddocname:foo OR sddocname:bar OR sddocname:baz)", "bar", "lhs"); + assertRewritten("lhs AND (sddocname:foo OR sddocname:bar OR sddocname:baz)", "baz", "lhs"); + } + + @Test + public void requireThatOptimizeByRestrictSimplifiesANDItemsThatHaveZeroRecall() { + assertRewritten("sddocname:foo AND bar AND baz", "cox", "NULL"); + assertRewritten("foo AND sddocname:bar AND baz", "cox", "NULL"); + assertRewritten("foo AND bar AND sddocname:baz", "cox", "NULL"); + + assertRewritten("lhs AND (sddocname:foo AND bar AND baz)", "cox", "NULL"); + assertRewritten("lhs AND (foo AND sddocname:bar AND baz)", "cox", "NULL"); + assertRewritten("lhs AND (foo AND bar AND sddocname:baz)", "cox", "NULL"); + + assertRewritten("lhs OR (sddocname:foo AND bar AND baz)", "cox", "lhs"); + assertRewritten("lhs OR (foo AND sddocname:bar AND baz)", "cox", "lhs"); + assertRewritten("lhs OR (foo AND bar AND sddocname:baz)", "cox", "lhs"); + } + + @Test + public void testRestrictRewrite() { + assertRewritten("a AND b", "per", "AND a b"); + assertRewritten("a OR b", "per", "OR a b"); + assertRewritten("sddocname:per", "per", "sddocname:per"); + assertRewritten("sddocname:per", "espen", "NULL"); + assertRewritten("sddocname:per OR sddocname:peder", "per", "sddocname:per"); + assertRewritten("sddocname:per AND sddocname:peder", "per", "NULL"); + assertRewritten("(sddocname:per AND a) OR (sddocname:peder AND b)", "per", "a"); + assertRewritten("sddocname:per ANDNOT b", "per", "+sddocname:per -b"); + assertRewritten("sddocname:perder ANDNOT b", "per", "NULL"); + assertRewritten("a ANDNOT sddocname:per a b", "per", "NULL"); + } + + @Test + public void testRestrictRank() { + assertRewritten("sddocname:per&filter=abc", "espen", "|abc"); + assertRewritten("sddocname:per&filter=abc", "per", "RANK sddocname:per |abc"); + } + + private static void assertRewritten(String queryParam, String restrictParam, String expectedOptimizedQuery) { + Query query = new Query("?type=adv&query=" + queryParam.replace(" ", "%20") + "&restrict=" + restrictParam); + QueryRewrite.optimizeByRestrict(query); + QueryRewrite.collapseSingleComposites(query); + assertEquals(expectedOptimizedQuery, query.getModel().getQueryTree().toString()); + } + + @Test + public void assertAndNotMovedUp() { + Query query = new Query(); + NotItem not = new NotItem(); + not.addPositiveItem(new WordItem("a")); + not.addNegativeItem(new WordItem("na")); + AndItem and = new AndItem(); + and.addItem(not); + query.getModel().getQueryTree().setRoot(and); + QueryRewrite.optimizeAndNot(query); + assertTrue(query.getModel().getQueryTree().getRoot() instanceof NotItem); + NotItem n = (NotItem) query.getModel().getQueryTree().getRoot(); + assertEquals(2, n.getItemCount()); + assertTrue(n.getPositiveItem() instanceof AndItem); + AndItem a = (AndItem) n.getPositiveItem(); + assertEquals(1, a.getItemCount()); + assertEquals("a", a.getItem(0).toString()); + assertEquals("na", n.getItem(1).toString()); + } + + @Test + public void assertMultipleAndNotIsCollapsed() { + Query query = new Query(); + NotItem not1 = new NotItem(); + not1.addPositiveItem(new WordItem("a")); + not1.addNegativeItem(new WordItem("na1")); + not1.addNegativeItem(new WordItem("na2")); + NotItem not2 = new NotItem(); + not2.addPositiveItem(new WordItem("b")); + not2.addNegativeItem(new WordItem("nb")); + AndItem and = new AndItem(); + and.addItem(new WordItem("1")); + and.addItem(not1); + and.addItem(new WordItem("2")); + and.addItem(not2); + and.addItem(new WordItem("3")); + query.getModel().getQueryTree().setRoot(and); + + QueryRewrite.optimizeAndNot(query); + + assertTrue(query.getModel().getQueryTree().getRoot() instanceof NotItem); + NotItem n = (NotItem) query.getModel().getQueryTree().getRoot(); + assertTrue(n.getPositiveItem() instanceof AndItem); + assertEquals(4, n.getItemCount()); + AndItem a = (AndItem) n.getPositiveItem(); + assertEquals(5, a.getItemCount()); + assertEquals("na1", n.getItem(1).toString()); + assertEquals("na2",n.getItem(2).toString()); + assertEquals("nb", n.getItem(3).toString()); + assertEquals("1", a.getItem(0).toString()); + assertEquals("a", a.getItem(1).toString()); + assertEquals("2", a.getItem(2).toString()); + assertEquals("b", a.getItem(3).toString()); + assertEquals("3", a.getItem(4).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java new file mode 100755 index 00000000000..55ce63a9789 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; + +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.RecallSearcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author Simon Thoresen + */ +public class RecallSearcherTestCase extends junit.framework.TestCase { + + public void testIgnoreEmptyProperty() { + RecallSearcher searcher = new RecallSearcher(); + Query query = new Query(); + Result result = new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertNull(result.hits().getError()); + assertTrue(query.getModel().getQueryTree().getRoot() instanceof NullItem); + } + + public void testDenyRankItems() { + RecallSearcher searcher = new RecallSearcher(); + Query query = new Query("?recall=foo"); + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertNotNull(result.hits().getError()); + } + + public void testParse() { + List empty = new ArrayList<>(); + assertQueryTree("?query=foo", Arrays.asList("foo"), empty); + assertQueryTree("?recall=%2bfoo", empty, Arrays.asList("foo")); + assertQueryTree("?query=foo&filter=bar&recall=%2bbaz", Arrays.asList("foo", "bar"), Arrays.asList("baz")); + assertQueryTree("?query=foo+bar&filter=baz&recall=%2bcox", Arrays.asList("foo", "bar", "baz"), Arrays.asList("cox")); + assertQueryTree("?query=foo&filter=bar+baz&recall=%2bcox", Arrays.asList("foo", "bar", "baz"), Arrays.asList("cox")); + assertQueryTree("?query=foo&filter=bar&recall=-baz+%2bcox", Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox")); + assertQueryTree("?query=foo%20bar&recall=%2bbaz%20-cox", Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox")); + } + + private static void assertQueryTree(String query, List ranked, List unranked) { + RecallSearcher searcher = new RecallSearcher(); + Query obj = new Query(query); + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(obj); + if (result.hits().getError() != null) { + fail(result.hits().getError().toString()); + } + + List myRanked = new ArrayList<>(ranked); + List myUnranked = new ArrayList<>(unranked); + + Stack stack = new Stack<>(); + stack.push(obj.getModel().getQueryTree().getRoot()); + while (!stack.isEmpty()) { + Item item = stack.pop(); + if (item instanceof WordItem) { + String word = ((WordItem)item).getWord(); + if (item.isRanked()) { + int idx = myRanked.indexOf(word); + if (idx < 0) { + fail("Term '" + word + "' not expected as ranked term."); + } + myRanked.remove(idx); + } else { + int idx = myUnranked.indexOf(word); + if (idx < 0) { + fail("Term '" + word + "' not expected as unranked term."); + } + myUnranked.remove(idx); + } + } + if (item instanceof CompositeItem) { + CompositeItem lst = (CompositeItem)item; + for (Iterator it = lst.getItemIterator(); it.hasNext();) { + stack.push((Item)it.next()); + } + } + } + + if (!myRanked.isEmpty()) { + fail("Ranked terms " + myRanked + " not found."); + } + if (!myUnranked.isEmpty()) { + fail("Unranked terms " + myUnranked + " not found."); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java new file mode 100644 index 00000000000..213a87bd6b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/StemmingSearcherTestCase.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.querytransform.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexFactsFactory; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.query.*; +import com.yahoo.prelude.querytransform.StemmingSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.searchchain.Execution; + +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Mathias M. Lidal + */ +public class StemmingSearcherTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + private final IndexFacts indexFacts = IndexFactsFactory.newInstance("dir:src/test/java/com/yahoo/prelude/" + + "querytransform/test/", null); + + @Test + public void testStemOnlySomeTerms() { + assertStem("/search?query=Holes in CVS and Subversion nostem:Found", + "AND hole in cvs and subversion nostem:Found"); + } + + @Test + public void testPhraseSegmentTransforms() { + Query q1 = buildQueryWithSegmentPhrase(); + executeStemming(q1); + assertEquals("AND a 'd e'", q1.getModel().getQueryTree().getRoot().toString()); + } + + private Query buildQueryWithSegmentPhrase() { + Query q1 = new Query("/search?query=placeholder&language=de"); + q1.getModel().setExecution(newExecution()); + AndItem root = new AndItem(); + root.addItem(new WordItem("a", true)); + // this is a trick, note the string to stem contains space + PhraseSegmentItem p = new PhraseSegmentItem("d e", true, false); + p.addItem(new WordItem("b", true)); + p.addItem(new WordItem("c", true)); + p.lock(); + root.addItem(p); + q1.getModel().getQueryTree().setRoot(root); + assertEquals("AND a 'b c'", q1.getModel().getQueryTree().getRoot().toString()); + return q1; + } + + @Test + public void testPreserveConnectivityToPhrase() { + Query q1 = buildQueryWithSegmentPhrase(); + CompositeItem r = (CompositeItem)q1.getModel().getQueryTree().getRoot(); + WordItem first = (WordItem)r.getItem(0); + PhraseSegmentItem second = (PhraseSegmentItem)r.getItem(1); + first.setConnectivity(second, 1.0d); + executeStemming(q1); + assertEquals("AND a 'd e'", q1.getModel().getQueryTree().getRoot().toString()); + r = (CompositeItem)q1.getModel().getQueryTree().getRoot(); + first = (WordItem)r.getItem(0); + second = (PhraseSegmentItem)r.getItem(1); + assertEquals("Connectivity incorrect.", + second, first.getConnectedItem()); + } + + @Test + public void testDontStemPrefixes() { + assertStem("/search?query=ist*&language=de", "ist*"); + } + + @Test + public void testStemming() { + Query query = new Query("/search?query="); + executeStemming(query); + assertTrue(query.getModel().getQueryTree().getRoot() instanceof NullItem); + } + + @Test + public void testNounStemming() { + assertStem("/search?query=noun:towers noun:tower noun:tow", + "AND noun:tower noun:tower noun:tow"); + assertStem("/search?query=notnoun:towers notnoun:tower notnoun:tow", + "AND notnoun:tower notnoun:tower notnoun:tow"); + } + + @Test + public void testEmptyIndexInfo() { + String indexInfoConfigID = "file:src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg"; + ConfigGetter getter = new ConfigGetter<>(IndexInfoConfig.class); + IndexInfoConfig config = getter.getConfig(indexInfoConfigID); + + IndexFacts indexFacts = new IndexFacts(new IndexModel(config, (QrSearchersConfig)null)); + + Query q = new Query(QueryTestCase.httpEncode("?query=cars")); + new Execution(new Chain(new StemmingSearcher(linguistics)), + new Execution.Context(null, indexFacts, null, null, linguistics)).search(q); + assertEquals("cars", q.getModel().getQueryTree().getRoot().toString()); + } + + @Test + public void testLiteralBoost() { + Query q = new Query(QueryTestCase.httpEncode("/search?language=en&search=three")); + WordItem scratch = new WordItem("trees", true); + scratch.setStemmed(false); + q.getModel().getQueryTree().setRoot(scratch); + executeStemming(q); + assertTrue("Expected a set of word alternatives as root.", + q.getModel().getQueryTree().getRoot() instanceof WordAlternativesItem); + WordAlternativesItem w = (WordAlternativesItem) q.getModel().getQueryTree().getRoot(); + boolean foundExpectedBaseForm = false; + for (WordAlternativesItem.Alternative a : w.getAlternatives()) { + if ("trees".equals(a.word)) { + assertEquals(1.0d, a.exactness, 1e-15); + foundExpectedBaseForm = true; + } + } + assertTrue("Did not find original word form in query.", foundExpectedBaseForm); + } + + private Execution.Context newExecutionContext() { + return new Execution.Context(null, indexFacts, null, null, linguistics); + } + + private Execution newExecution() { + return new Execution(newExecutionContext()); + } + + private void executeStemming(Query query) { + new Execution(new Chain(new StemmingSearcher(linguistics)), + newExecutionContext()).search(query); + } + + private void assertStem(String queryString, String expectedQueryTree) { + assertStemEncoded(QueryTestCase.httpEncode(queryString), expectedQueryTree); + } + + private void assertStemEncoded(String encodedQueryString, String expectedQueryTree) { + Query query = new Query(encodedQueryString); + executeStemming(query); + assertEquals(expectedQueryTree, query.getModel().getQueryTree().getRoot().toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/accent-removal-index-info.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/accent-removal-index-info.cfg new file mode 100644 index 00000000000..d3f3e35bd62 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/accent-removal-index-info.cfg @@ -0,0 +1,47 @@ +indexinfo[3] +indexinfo[0].name one +indexinfo[0].command[12] +indexinfo[0].command[0].indexname exactemento +indexinfo[0].command[0].command compact-to-term +indexinfo[0].command[1].indexname default +indexinfo[0].command[1].command stem +indexinfo[0].command[2].indexname default +indexinfo[0].command[2].command normalize +indexinfo[0].command[3].indexname default +indexinfo[0].command[3].command "complete-boost 42" +indexinfo[0].command[4].indexname full +indexinfo[0].command[4].command "complete-boost 17" +indexinfo[0].command[5].indexname default +indexinfo[0].command[5].command "literal-boost 20" +indexinfo[0].command[6].indexname noun +indexinfo[0].command[6].command "stem DEFAULT" +indexinfo[0].command[7].indexname notnoun +indexinfo[0].command[7].command stem +indexinfo[0].command[8].indexname nostem +indexinfo[0].command[8].command index +indexinfo[0].command[9].indexname other +indexinfo[0].command[9].command index +indexinfo[0].command[10].indexname noun +indexinfo[0].command[10].command index +indexinfo[0].command[11].indexname notnoun +indexinfo[0].command[11].command index +indexinfo[0].command[12].indexname normalizercheck +indexinfo[0].command[12].command normalize +indexinfo[0].command[13].indexname normalizercheck +indexinfo[0].command[13].command index +indexinfo[1].name two +indexinfo[1].command[4] +indexinfo[1].command[0].indexname _default +indexinfo[1].command[0].command compact-to-term +indexinfo[1].command[1].indexname b +indexinfo[1].command[1].command compact-to-term +indexinfo[1].command[2].indexname absolute +indexinfo[1].command[2].command "complete-boost 23" +indexinfo[1].command[3].indexname absolute +indexinfo[1].command[3].command literal-boost +indexinfo[2].name five +indexinfo[2].command[2] +indexinfo[2].command[0].indexname noaccentremoval +indexinfo[2].command[0].command attribute +indexinfo[2].command[1].indexname noaccentremoval +indexinfo[2].command[1].command normalize diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/cjk-index-info.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/cjk-index-info.cfg new file mode 100644 index 00000000000..06e6500cc2b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/cjk-index-info.cfg @@ -0,0 +1,19 @@ +indexinfo[1] +indexinfo[0].name urltests +indexinfo[0].command[8] +indexinfo[0].command[0].indexname url.all +indexinfo[0].command[0].command fullurl +indexinfo[0].command[1].indexname host.all +indexinfo[0].command[1].command urlhost +indexinfo[0].command[2].indexname site +indexinfo[0].command[2].command urlhost +indexinfo[0].command[3].indexname url.all +indexinfo[0].command[3].command index +indexinfo[0].command[4].indexname host.all +indexinfo[0].command[4].command index +indexinfo[0].command[5].indexname site +indexinfo[0].command[5].command index +indexinfo[0].command[6].indexname basic +indexinfo[0].command[6].command index +indexinfo[0].command[7].indexname basic +indexinfo[0].command[7].command word diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/container-http.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/container-http.cfg new file mode 100644 index 00000000000..80c122636cf --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/container-http.cfg @@ -0,0 +1,3 @@ +enabled true +port.search 18081 +port.stats 18085 diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg new file mode 100644 index 00000000000..8da0231d933 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/emptyindexinfo.cfg @@ -0,0 +1,2 @@ +indexinfo[0] + diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/index-info.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/index-info.cfg new file mode 100644 index 00000000000..0c34dade1da --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/index-info.cfg @@ -0,0 +1,47 @@ +indexinfo[3] +indexinfo[0].name one +indexinfo[0].command[12] +indexinfo[0].command[0].indexname exactemento +indexinfo[0].command[0].command compact-to-term +indexinfo[0].command[1].indexname default +indexinfo[0].command[1].command stem +indexinfo[0].command[2].indexname default +indexinfo[0].command[2].command normalize +indexinfo[0].command[3].indexname default +indexinfo[0].command[3].command "complete-boost 42" +indexinfo[0].command[4].indexname full +indexinfo[0].command[4].command "complete-boost 17" +indexinfo[0].command[5].indexname default +indexinfo[0].command[5].command "literal-boost 20" +indexinfo[0].command[6].indexname noun +indexinfo[0].command[6].command "stem DEFAULT" +indexinfo[0].command[7].indexname notnoun +indexinfo[0].command[7].command stem +indexinfo[0].command[8].indexname nostem +indexinfo[0].command[8].command index +indexinfo[0].command[9].indexname other +indexinfo[0].command[9].command index +indexinfo[0].command[10].indexname noun +indexinfo[0].command[10].command index +indexinfo[0].command[11].indexname notnoun +indexinfo[0].command[11].command index +indexinfo[0].command[12].indexname normalizercheck +indexinfo[0].command[12].command normalize +indexinfo[0].command[13].indexname normalizercheck +indexinfo[0].command[13].command index +indexinfo[1].name two +indexinfo[1].command[4] +indexinfo[1].command[0].indexname _default +indexinfo[1].command[0].command compact-to-term +indexinfo[1].command[1].indexname b +indexinfo[1].command[1].command compact-to-term +indexinfo[1].command[2].indexname absolute +indexinfo[1].command[2].command "complete-boost 23" +indexinfo[1].command[3].indexname absolute +indexinfo[1].command[3].command literal-boost +indexinfo[2].name three +indexinfo[2].command[2] +indexinfo[2].command[0].indexname default +indexinfo[2].command[0].command stem +indexinfo[2].command[1].indexname default +indexinfo[2].command[1].command literal-boost diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg new file mode 100644 index 00000000000..a8790cc4051 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg @@ -0,0 +1,29 @@ +indexinfo[1] +indexinfo[0].name combinedattributeandindexsearch +indexinfo[0].command[12] +indexinfo[0].command[0].indexname sddocname +indexinfo[0].command[0].command index +indexinfo[0].command[1].indexname index1 +indexinfo[0].command[1].command index +indexinfo[0].command[2].indexname default +indexinfo[0].command[2].command index +indexinfo[0].command[3].indexname index1 +indexinfo[0].command[3].command stem +indexinfo[0].command[4].indexname default +indexinfo[0].command[4].command stem +indexinfo[0].command[5].indexname index1 +indexinfo[0].command[5].command index1 +indexinfo[0].command[5].command normalize +indexinfo[0].command[6].indexname default +indexinfo[0].command[6].command normalize +indexinfo[0].command[7].indexname attribute1 +indexinfo[0].command[7].command index +indexinfo[0].command[8].indexname attribute1 +indexinfo[0].command[8].command attribute +indexinfo[0].command[9].indexname attribute2 +indexinfo[0].command[9].command attribute +indexinfo[0].command[10].indexname default +indexinfo[0].command[10].command "match-group default attribute1 attribute2" +indexinfo[0].command[11].indexname ranklog +indexinfo[0].command[11].command xmlstring + diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases-input.txt b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases-input.txt new file mode 100644 index 00000000000..00b5e4e5c6a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases-input.txt @@ -0,0 +1,5 @@ +arne treholt +britney spears +nick cave +this is a test +woody allen diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases.fsa b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases.fsa new file mode 100644 index 00000000000..91a3e5f62a5 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-phrases.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments-input.txt b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments-input.txt new file mode 100644 index 00000000000..c6626bc81e1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments-input.txt @@ -0,0 +1,6 @@ +david bowie 13 +i am a man 60 +lord of the rings 36 +new york 11 +web search 13 +with david bowie 19 diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments.fsa b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments.fsa new file mode 100644 index 00000000000..3b96a567422 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-segments.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words-input.txt b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words-input.txt new file mode 100644 index 00000000000..b8b0c5b3e60 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words-input.txt @@ -0,0 +1,9 @@ +airlines +banana +before +bike +eat +scandal +stop +you +your diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words.fsa b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words.fsa new file mode 100644 index 00000000000..534aa58159c Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/proximity-stop-words.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa-input.txt b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa-input.txt new file mode 100644 index 00000000000..06014eb0bea --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa-input.txt @@ -0,0 +1,4 @@ +aword +this is 3 tests +this is a test +tudor vidor diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa new file mode 100644 index 00000000000..c13093443b6 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/test-fsa.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/testindexinfonoboost.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/testindexinfonoboost.cfg new file mode 100644 index 00000000000..4c943b73aeb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/testindexinfonoboost.cfg @@ -0,0 +1,15 @@ +indexinfo[2] +indexinfo[0].name one +indexinfo[0].command[3] +indexinfo[0].command[0].indexname exactemento +indexinfo[0].command[0].command compact-to-term +indexinfo[0].command[1].indexname default +indexinfo[0].command[1].command stem +indexinfo[0].command[2].indexname default +indexinfo[0].command[2].command normalize +indexinfo[1].name two +indexinfo[1].command[2] +indexinfo[1].command[0].indexname default +indexinfo[1].command[0].command compact-to-term +indexinfo[1].command[1].indexname b +indexinfo[1].command[1].command compact-to-term diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java new file mode 100644 index 00000000000..db7009cd2c4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -0,0 +1,480 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.searcher.BlendingSearcher; +import com.yahoo.prelude.searcher.DocumentSourceSearcher; +import com.yahoo.prelude.searcher.FillSearcher; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; + +/** + * Tests the BlendingSearcher class + * + * @author Bob Travis + * @author bratseth + */ +// The SuppressWarnings is to shut up the compiler about using +// deprecated FastHit constructor in the tests. +@SuppressWarnings({ "unchecked", "rawtypes" }) +public class BlendingSearcherTestCase extends junit.framework.TestCase { + + public BlendingSearcherTestCase(String name) { + super(name); + } + + public static class BlendingSearcherWrapper extends Searcher { + + private SearchChain blendingChain; + private final FederationConfig.Builder builder = new FederationConfig.Builder(); + private final Map searchers + = new HashMap<>(); + private SearchChainRegistry chainRegistry; + + private final String blendingDocumentId; + + public BlendingSearcherWrapper() { + blendingDocumentId = null; + } + + public BlendingSearcherWrapper(String blendingDocumentId) { + this.blendingDocumentId = blendingDocumentId; + } + + @SuppressWarnings("serial") + public BlendingSearcherWrapper(QrSearchersConfig cfg) { + QrSearchersConfig.Com.Yahoo.Prelude.Searcher.BlendingSearcher s = cfg.com().yahoo().prelude().searcher().BlendingSearcher(); + blendingDocumentId = s.docid().length() > 0 ? s.docid() : null; + } + + public boolean addChained(Searcher searcher, String sourceName) { + builder.target(new FederationConfig.Target.Builder(). + id(sourceName). + searchChain(new FederationConfig.Target.SearchChain.Builder(). + searchChainId(sourceName). + timeoutMillis(10000). + useByDefault(true)) + ); + searchers.put(sourceName, searcher); + return true; + } + + @Override + public com.yahoo.search.Result search(com.yahoo.search.Query query, Execution execution) { + query.setTimeout(10000); + query.setOffset(query.getOffset()); + query.setHits(query.getHits()); + Execution exec = new Execution(blendingChain, Execution.Context.createContextStub(chainRegistry, null)); + exec.context().populateFrom(execution.context()); + return exec.search(query); + } + + @Override + public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { + new Execution(blendingChain, Execution.Context.createContextStub(chainRegistry, null)).fill(result, summaryClass); + } + + public boolean initialize() { + chainRegistry = new SearchChainRegistry(); + + //First add all the current searchers as searchchains + for(Map.Entry entry : searchers.entrySet()) { + chainRegistry.register( + createSearchChain( + new ComponentId(entry.getKey()), + entry.getValue())); + } + + StrictContractsConfig contracts = new StrictContractsConfig(new StrictContractsConfig.Builder()); + + FederationSearcher fedSearcher = + new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry()); + BlendingSearcher blendingSearcher = new BlendingSearcher(blendingDocumentId); + blendingChain = new SearchChain(ComponentId.createAnonymousComponentId("blendingChain"), blendingSearcher, fedSearcher); + return true; + } + + private SearchChain createSearchChain(ComponentId chainId, Searcher searcher) { + return new SearchChain(chainId, searcher); + } + } + + @SuppressWarnings("serial") + public void testitTwoPhase() { + + DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); + DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); + DocumentSourceSearcher chain3 = new DocumentSourceSearcher(); + + Query q = new Query("/search?query=hannibal"); + + Result r1 = new Result(q); + Result r2 = new Result(q); + Result r3 = new Result(q); + + r1.setTotalHitCount(13); + r1.hits().add(new Hit("http://host1.com", 101){{setSource("one");}}); + r1.hits().add(new Hit("http://host2.com", 102){{setSource("one");}}); + r1.hits().add(new Hit("http://host3.com", 103){{setSource("one");}}); + chain1.addResultSet(q, r1); + + r2.setTotalHitCount(17); + r2.hits().add(new Hit("http://host1.com", 101){{setSource("two");}}); + r2.hits().add(new Hit("http://host2.com", 102){{setSource("two");}}); + r2.hits().add(new Hit("http://host4.com", 104){{setSource("two");}}); + chain2.addResultSet(q, r2); + + r3.setTotalHitCount(37); + r3.hits().add(new Hit("http://host5.com", 100){{setSource("three");}}); + r3.hits().add(new Hit("http://host6.com", 106){{setSource("three");}}); + r3.hits().add(new Hit("http://host7.com", 105){{setSource("three");}}); + chain3.addResultSet(q, r3); + + BlendingSearcherWrapper blender1 = new BlendingSearcherWrapper(); + blender1.addChained(chain1, "one"); + blender1.initialize(); + q.setWindow( 0, 10); + Result br1 = new Execution(blender1, Execution.Context.createContextStub()).search(q); + assertEquals(3, br1.getHitCount()); + assertEquals(13, br1.getTotalHitCount()); + assertEquals("http://host3.com/", br1.hits().get(0).getId().toString()); + + BlendingSearcherWrapper blender2 = new BlendingSearcherWrapper(); + blender2.addChained(chain1, "two"); + blender2.addChained(chain2, "three"); + blender2.initialize(); + q.setWindow( 0, 10); + Result br2 = new Execution(blender2, Execution.Context.createContextStub()).search(q); + assertEquals(6, br2.getHitCount()); + assertEquals(30, br2.getTotalHitCount()); + assertEquals("http://host4.com/", br2.hits().get(0).getId().toString()); + + BlendingSearcherWrapper blender3 = new BlendingSearcherWrapper(); + blender3.addChained(chain1, "four"); + blender3.addChained(chain2, "five"); + blender3.addChained(chain3, "six"); + blender3.initialize(); + q.setWindow( 0, 10); + Result br3 = new Execution(blender3, Execution.Context.createContextStub()).search(q); + assertEquals(9, br3.getHitCount()); + assertEquals(67, br3.getTotalHitCount()); + assertEquals("http://host6.com/", br3.hits().get(0).getId().toString()); + + q.setWindow( 0, 10); + Result br4 = new Execution(blender3, Execution.Context.createContextStub()).search(q); + assertEquals(9, br4.getHitCount()); + assertEquals("http://host6.com/", br4.hits().get(0).getId().toString()); + + q.setWindow( 3, 10); + Result br5 = new Execution(blender3, Execution.Context.createContextStub()).search(q); + assertEquals(6, br5.getHitCount()); + assertEquals("http://host3.com/", br5.hits().get(0).getId().toString()); + + q.setWindow( 3, 10); + br5 = new Execution(blender3, Execution.Context.createContextStub()).search(q); + assertEquals(6, br5.getHitCount()); + assertEquals("http://host3.com/", br5.hits().get(0).getId().toString()); + + q.setWindow( 3, 10); + br5 = new Execution(blender3, Execution.Context.createContextStub()).search(q); + assertEquals(6, br5.getHitCount()); + assertEquals("http://host3.com/", br5.hits().get(0).getId().toString()); + + } + + public void testMultipleBackendsWithDuplicateRemoval() { + DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); + DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); + Query q = new Query("/search?query=hannibal&search=a,b"); + Result r1 = new Result(q); + Result r2 = new Result(q); + + r1.setTotalHitCount(1); + r1.hits().add(new FastHit("http://host1.com/", 101)); + chain1.addResultSet(q, r1); + r2.hits().add(new FastHit("http://host1.com/", 102)); + r2.setTotalHitCount(1); + chain2.addResultSet(q, r2); + + BlendingSearcherWrapper blender = new BlendingSearcherWrapper("uri"); + blender.addChained(new FillSearcher(chain1), "a"); + blender.addChained(new FillSearcher(chain2), "b"); + blender.initialize(); + q.setWindow( 0, 10); + Result cr = new Execution(blender, Execution.Context.createContextStub()).search(q); + assertEquals(1, cr.getHitCount()); + assertEquals(101, ((int) cr.hits().get(0).getRelevance().getScore())); + } + + public void testMultipleBackendsWithErrorMerging() { + DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); + DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); + Query q = new Query("/search?query=hannibal&search=a,b"); + Result r1 = new Result(q, ErrorMessage.createNoBackendsInService(null)); + Result r2 = new Result(q, ErrorMessage.createRequestTooLarge(null)); + + r1.setTotalHitCount(0); + chain1.addResultSet(q, r1); + r2.hits().add(new FastHit("http://host1.com/", 102)); + r2.setTotalHitCount(1); + chain2.addResultSet(q, r2); + + BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); + blender.addChained(new FillSearcher(chain1), "a"); + blender.addChained(new FillSearcher(chain2), "b"); + blender.initialize(); + q.setWindow( 0, 10); + Result cr = new Execution(blender, Execution.Context.createContextStub()).search(q); + assertEquals(2, cr.getHitCount()); + assertEquals(1, cr.getConcreteHitCount()); + com.yahoo.search.result.ErrorHit errorHit = cr.hits().getErrorHit(); + Iterator errorIterator = errorHit.errorIterator(); + List errorList = Arrays.asList("Source 'a': No backends in service. Try later", + "Source 'b': 2: Request too large"); + String a = errorIterator.next().toString(); + assertTrue(a, errorList.contains(a)); + String b = errorIterator.next().toString(); + assertTrue(a, errorList.contains(b)); + assertFalse(errorIterator.hasNext()); + assertEquals(102, ((int) cr.hits().get(1).getRelevance().getScore())); + assertEquals(com.yahoo.container.protect.Error.NO_BACKENDS_IN_SERVICE.code, cr.hits().getError().getCode()); + } + + public void testBlendingWithSortSpec() { + DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); + DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); + + Query q = new Query("/search?query=banana+&sorting=%2Bfoobar"); + + Result r1 = new Result(q); + Result r2 = new Result(q); + + r1.setTotalHitCount(3); + Hit r1h1 = new Hit("http://host1.com/relevancy101", 101); + r1h1.setField("foobar", "3"); + r1h1.setQuery(q); + Hit r1h2 = new Hit("http://host2.com/relevancy102", 102); + r1h2.setField("foobar", "6"); + r1h2.setQuery(q); + Hit r1h3 = new Hit("http://host3.com/relevancy103", 103); + r1h3.setField("foobar", "2"); + r1h3.setQuery(q); + r1.hits().add(r1h1); + r1.hits().add(r1h2); + r1.hits().add(r1h3); + chain1.addResultSet(q, r1); + + r2.setTotalHitCount(3); + Hit r2h1 = new Hit("http://host1.com/relevancy201", 201); + r2h1.setField("foobar", "5"); + r2h1.setQuery(q); + Hit r2h2 = new Hit("http://host2.com/relevancy202", 202); + r2h2.setField("foobar", "1"); + r2h2.setQuery(q); + Hit r2h3 = new Hit("http://host3.com/relevancy203", 203); + r2h3.setField("foobar", "4"); + r2h3.setQuery(q); + r2.hits().add(r2h1); + r2.hits().add(r2h2); + r2.hits().add(r2h3); + chain2.addResultSet(q, r2); + + BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); + blender.addChained(new FillSearcher(chain1), "chainedone"); + blender.addChained(new FillSearcher(chain2), "chainedtwo"); + blender.initialize(); + q.setWindow( 0, 10); + Result br = new Execution(blender, Execution.Context.createContextStub()).search(q); + assertEquals(202, ((int) br.hits().get(0).getRelevance().getScore())); + assertEquals(103, ((int) br.hits().get(1).getRelevance().getScore())); + assertEquals(101, ((int) br.hits().get(2).getRelevance().getScore())); + assertEquals(203, ((int) br.hits().get(3).getRelevance().getScore())); + assertEquals(201, ((int) br.hits().get(4).getRelevance().getScore())); + assertEquals(102, ((int) br.hits().get(5).getRelevance().getScore())); + } + + /** + * Disabled because the document source searcher does not handle being asked for + * document sumaries for hits it did not create (it will insert the wrong values). + * But are we sure fsearch handles this case correctly? + */ + public void testBlendingWithSortSpecAnd2Phase() { + DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); + DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); + + Query q = new Query("/search?query=banana+&sorting=%2Battributefoobar"); + Result r1 = new Result(q); + Result r2 = new Result(q); + + r1.setTotalHitCount(3); + Hit r1h1 = new Hit("http://host1.com/relevancy101", 101); + r1h1.setField("attributefoobar", "3"); + Hit r1h2 = new Hit("http://host2.com/relevancy102", 102); + r1h2.setField("attributefoobar", "6"); + Hit r1h3 = new Hit("http://host3.com/relevancy103", 103); + r1h3.setField("attributefoobar", "2"); + r1.hits().add(r1h1); + r1.hits().add(r1h2); + r1.hits().add(r1h3); + chain1.addResultSet(q, r1); + + r2.setTotalHitCount(3); + Hit r2h1 = new Hit("http://host1.com/relevancy201", 201); + r2h1.setField("attributefoobar", "5"); + Hit r2h2 = new Hit("http://host2.com/relevancy202", 202); + r2h2.setField("attributefoobar", "1"); + Hit r2h3 = new Hit("http://host3.com/relevancy203", 203); + r2h3.setField("attributefoobar", "4"); + r2.hits().add(r2h1); + r2.hits().add(r2h2); + r2.hits().add(r2h3); + chain2.addResultSet(q, r2); + + BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); + blender.addChained(chain1, "chainedone"); + blender.addChained(chain2, "chainedtwo"); + blender.initialize(); + q.setWindow( 0, 10); + Result br = new Execution(blender, Execution.Context.createContextStub()).search(q); + assertEquals(202, ((int) br.hits().get(0).getRelevance().getScore())); + assertEquals(103, ((int) br.hits().get(1).getRelevance().getScore())); + assertEquals(101, ((int) br.hits().get(2).getRelevance().getScore())); + assertEquals(203, ((int) br.hits().get(3).getRelevance().getScore())); + assertEquals(201, ((int) br.hits().get(4).getRelevance().getScore())); + assertEquals(102, ((int) br.hits().get(5).getRelevance().getScore())); + } + + private BlendingSearcherWrapper setupFirstAndSecond() { + DocumentSourceSearcher first = new DocumentSourceSearcher(); + DocumentSourceSearcher second = new DocumentSourceSearcher(); + + Query query = new Query("?query=banana"); + + Result r1 = new Result(query); + r1.setTotalHitCount(1); + Hit r1h1 = new Hit("http://first/relevancy100", 200); + r1.hits().add(r1h1); + first.addResultSet(query, r1); + + Result r2 = new Result(query); + r2.setTotalHitCount(2); + Hit r2h1 = new Hit("http://second/relevancy300", 300); + Hit r2h2 = new Hit("http://second/relevancy100", 100); + r2.hits().add(r2h1); + r2.hits().add(r2h2); + second.addResultSet(query, r2); + + BlendingSearcherWrapper blender = new BlendingSearcherWrapper(); + blender.addChained(new FillSearcher(first), "first"); + blender.addChained(new FillSearcher(second), "second"); + blender.initialize(); + return blender; + } + + public void testOnlyFirstBackend() { + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana&search=first"); + + Result result = new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertEquals(1, result.getHitCount()); + assertEquals(200.0, result.hits().get(0).getRelevance().getScore()); + } + + public void testOnlySecondBackend() { + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana&search=second"); + + Result result = new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertEquals(2, result.getHitCount()); + assertEquals(300.0, result.hits().get(0).getRelevance().getScore()); + assertEquals(100.0, result.hits().get(1).getRelevance().getScore()); + } + + public void testBothBackendsExplicitly() { + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana&search=first,second"); + + Result result = new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertEquals(3, result.getHitCount()); + assertEquals(300.0, result.hits().get(0).getRelevance().getScore()); + assertEquals(200.0, result.hits().get(1).getRelevance().getScore()); + assertEquals(100.0, result.hits().get(2).getRelevance().getScore()); + } + + public void testBothBackendsImplicitly() { + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana"); + + Result result = new Execution(searcher, Execution.Context.createContextStub()).search(query); + assertEquals(3, result.getHitCount()); + assertEquals(300.0, result.hits().get(0).getRelevance().getScore()); + assertEquals(200.0, result.hits().get(1).getRelevance().getScore()); + assertEquals(100.0, result.hits().get(2).getRelevance().getScore()); + } + + public void testNonexistingBackendCausesError() { + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana&search=nonesuch"); + + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertEquals(0, result.getConcreteHitCount()); + assertNotNull(result.hits().getError()); + ErrorMessage e = result.hits().getError(); + assertEquals("Invalid query parameter", e.getMessage()); + //assertEquals("No source named 'nonesuch' to search. Valid sources are [first, second]", + // e.getDetailedMessage()); + } + + public void testNonexistingBackendsCausesErrorOnFirst() { + // Feel free to change to include all in the detail message... + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana&search=nonesuch,orsuch"); + + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertEquals(0, result.getConcreteHitCount()); + assertNotNull(result.hits().getError()); + ErrorMessage e = result.hits().getError(); + assertEquals("Invalid query parameter", e.getMessage()); + //TODO: Do not depend on sources order + assertEquals("4: Invalid query parameter: Could not resolve source ref 'nonesuch'. Could not resolve source ref 'orsuch'. Valid source refs are first, second.", + e.toString()); + } + + public void testExistingAndNonExistingBackendCausesBothErrorAndResult() { + BlendingSearcherWrapper searcher = setupFirstAndSecond(); + Query query = new Query("/search?query=banana&search=first,nonesuch,second"); + + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertEquals(3, result.getConcreteHitCount()); + assertEquals(300.0, result.hits().get(1).getRelevance().getScore()); + assertEquals(200.0, result.hits().get(2).getRelevance().getScore()); + assertEquals(100.0, result.hits().get(3).getRelevance().getScore()); + assertNotNull(result.hits().getError()); + ErrorMessage e = result.hits().getError(); + //TODO: Do not depend on sources order + assertEquals("Could not resolve source ref 'nonesuch'. Valid source refs are first, second.", + e.getDetailedMessage()); + + + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/CachingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/CachingSearcherTestCase.java new file mode 100644 index 00000000000..34e810448da --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/CachingSearcherTestCase.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import static org.junit.Assert.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.searcher.CachingSearcher; +import com.yahoo.prelude.searcher.DocumentSourceSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; + +/** + * Check CachingSearcher basically works. + * + * @author Steinar Knutsen + */ +public class CachingSearcherTestCase { + + private static final String QUERY_A_NOCACHEWRITE_TRUE = "/?query=a&nocachewrite=true"; + private static final String QUERY_A = "/?query=a"; + private Chain searchChain; + private DocumentSourceSearcher hits; + + @Before + public void setUp() throws Exception { + hits = new DocumentSourceSearcher(); + QrSearchersConfig config = new QrSearchersConfig( + new QrSearchersConfig.Builder() + .com(new QrSearchersConfig.Com.Builder() + .yahoo(new QrSearchersConfig.Com.Yahoo.Builder() + .prelude(new QrSearchersConfig.Com.Yahoo.Prelude.Builder() + .searcher(new QrSearchersConfig.Com.Yahoo.Prelude.Searcher.Builder() + .CachingSearcher( + new QrSearchersConfig.Com.Yahoo.Prelude.Searcher.CachingSearcher.Builder() + .cachesizemegabytes(10) + .maxentrysizebytes(5 * 1024 * 1024) + .timetoliveseconds(86400))))))); + CachingSearcher cache = new CachingSearcher(config, Statistics.nullImplementation); + searchChain = new Chain<>(cache, hits); + } + + public void readyResult(String q) { + Query query = new Query(q); + Result r = new Result(query); + for (int i = 0; i < 10; ++i) { + FastHit h = new FastHit("http://127.0.0.1/" + i, + 1.0 - ((double) i) / 10.0); + r.hits().add(h); + } + hits.addResultSet(query, r); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void test() { + readyResult(QUERY_A); + Execution e = new Execution(searchChain, Execution.Context.createContextStub()); + Result r = e.search(new Query(QUERY_A)); + assertEquals(10, r.hits().getConcreteSize()); + Query query = new Query(QUERY_A); + Result expected = new Result(query); + hits.addResultSet(query, expected); + e = new Execution(searchChain, Execution.Context.createContextStub()); + r = e.search(new Query(QUERY_A)); + assertEquals(10, r.hits().getConcreteSize()); + assertEquals(1, hits.getQueryCount()); + } + + @Test + public final void testNoCacheWrite() { + readyResult(QUERY_A_NOCACHEWRITE_TRUE); + Execution e = new Execution(searchChain, Execution.Context.createContextStub()); + Result r = e.search(new Query(QUERY_A_NOCACHEWRITE_TRUE)); + assertEquals(10, r.hits().getConcreteSize()); + Query query = new Query(QUERY_A_NOCACHEWRITE_TRUE); + Result expected = new Result(query); + hits.addResultSet(query, expected); + e = new Execution(searchChain, Execution.Context.createContextStub()); + r = e.search(new Query(QUERY_A_NOCACHEWRITE_TRUE)); + assertEquals(0, r.hits().getConcreteSize()); + assertEquals(2, hits.getQueryCount()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ErrorHitRenderTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ErrorHitRenderTestCase.java new file mode 100644 index 00000000000..4bf173489f9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ErrorHitRenderTestCase.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.search.result.DefaultErrorHit; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.text.XMLWriter; + +import java.io.StringWriter; + +/** + * Tests marking hit properties as XML + * + * @author Steinar Knutsen + */ +public class ErrorHitRenderTestCase extends junit.framework.TestCase { + + public ErrorHitRenderTestCase(String name) { + super(name); + } + + public void testXMLEscaping() throws java.io.IOException { + ErrorHit h = new DefaultErrorHit("testcase", + ErrorMessage.createUnspecifiedError("<>\"&")); + + StringWriter writer = new StringWriter(); + SearchRendererAdaptor.renderMessageDefaultErrorHit(new XMLWriter(writer), h.errors().iterator().next()); + assertEquals("<>\"&\n", + writer.toString()); + + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java new file mode 100644 index 00000000000..8642adfd2d4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -0,0 +1,479 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.searcher.FieldCollapsingSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.LongId; +import com.yahoo.search.grouping.result.RootId; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests the FieldCollapsingSearcher class + * + * @author Steinar Knutsen + */ +@SuppressWarnings("deprecation") +public class FieldCollapsingSearcherTestCase extends junit.framework.TestCase { + + public FieldCollapsingSearcherTestCase (String name) { + super(name); + } + + private FastHit createHit(String uri,int relevancy,int mid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(mid)); + return hit; + } + + private void assertHit(String uri,int relevancy,int mid,Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); + } + + private static class ZeroHitsControl extends com.yahoo.search.Searcher { + public int queryCount = 0; + public com.yahoo.search.Result search(com.yahoo.search.Query query, + com.yahoo.search.searchchain.Execution execution) { + ++queryCount; + if (query.getHits() == 0) { + return new Result(query); + } else { + return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); + } + } + } + + public void testFieldCollapsingWithoutHits() { + // Set up + Map chained = new HashMap<>(); + + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + ZeroHitsControl checker = new ZeroHitsControl(); + chained.put(collapse, checker); + + Query q = new Query("?query=test_collapse&collapsefield=amid"); + Result r = doSearch(collapse, q, 0, 0, chained); + + assertEquals(0, r.getHitCount()); + assertNull(r.hits().getError()); + assertEquals(1, checker.queryCount); + } + + public void testFieldCollapsingWithoutHitsHugeOffset() { + Map chained = new HashMap<>(); + + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + ZeroHitsControl checker = new ZeroHitsControl(); + chained.put(collapse, checker); + + Query q = new Query("?query=test_collapse&collapsefield=amid"); + Result r = doSearch(collapse, q, 1000, 0, chained); + + assertEquals(0, r.getHitCount()); + assertNull(r.hits().getError()); + assertEquals(1, checker.queryCount); + } + + public void testFieldCollapsing() { + Map chained = new HashMap<>(); + + // Set up + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + // Caveat: Collapse is set to false, because that's what the + // collapser asks for + Query q = new Query("?query=test_collapse&collapsefield=amid"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.setTotalHitCount(8); + docsource.addResult(q, r); + + // Test basic collapsing on mid + q = new Query("?query=test_collapse&collapsefield=amid"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + } + + public void testFieldCollapsingTwoPhase() { + // Set up + Map chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + // Caveat: Collapse is set to false, because that's what the + // collapser asks for + Query q = new Query("?query=test_collapse&collapsefield=amid"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.setTotalHitCount(8); + docsource.addResult(q, r); + + // Test basic collapsing on mid + q = new Query("?query=test_collapse&collapsefield=amid"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + } + + public void testNoCollapsingIfNotAskedTo() { + // Set up + Map chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.setTotalHitCount(8); + docsource.addResult(q, r); + + // Test that no collapsing occured + q = new Query("?query=test_collapse"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(8, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + } + + /** + * Tests that collapsing many hits from one site works, and without + * an excessive number of backend requests + */ + public void testCollapsingLargeCollection() { + // Set up + Map chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(4,2.0,"amid"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse&collapsesize=1&collapsefield=amid"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,0)); + r.hits().add(createHit("http://acme.org/d.html", 8,0)); + r.hits().add(createHit("http://acme.org/e.html", 8,0)); + r.hits().add(createHit("http://acme.org/f.html", 7,0)); + r.hits().add(createHit("http://acme.org/g.html", 7,0)); + r.hits().add(createHit("http://acme.org/h.html", 6,0)); + r.hits().add(createHit("http://acme.org/i.html", 5,1)); + r.hits().add(createHit("http://acme.org/j.html", 4,2)); + r.setTotalHitCount(10); + docsource.addResult(q, r); + + // Test collapsing + q = new Query("?query=test_collapse&collapsesize=1&collapsefield=amid"); + r = doSearch(collapse, q, 0, 2, chained); + + assertEquals(2, r.getHitCount()); + assertEquals(2, docsource.getQueryCount()); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/i.html", 5,1,r.hits().get(1)); + + // Next results + docsource.resetQueryCount(); + r = doSearch(collapse, q, 2, 2, chained); + assertEquals(1, r.getHitCount()); + assertEquals(2, docsource.getQueryCount()); + assertHit("http://acme.org/j.html",4,2,r.hits().get(0)); + } + + /** + * Tests collapsing of "messy" data + */ + public void testCollapsingDispersedCollection() { + // Set up + Map chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher(1,2.0,"amid"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + + Query q = new Query("?query=test_collapse&collapse=true&collapsefield=amid"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,1)); + r.hits().add(createHit("http://acme.org/b.html",10,1)); + r.hits().add(createHit("http://acme.org/c.html",10,0)); + r.hits().add(createHit("http://acme.org/d.html",10,0)); + r.hits().add(createHit("http://acme.org/e.html",10,0)); + r.hits().add(createHit("http://acme.org/f.html",10,0)); + r.hits().add(createHit("http://acme.org/g.html",10,0)); + r.hits().add(createHit("http://acme.org/h.html",10,0)); + r.hits().add(createHit("http://acme.org/i.html",10,0)); + r.hits().add(createHit("http://acme.org/j.html",10,1)); + r.setTotalHitCount(10); + docsource.addResult(q, r); + + // Test collapsing + q = new Query("?query=test_collapse&collapse=true&collapsefield=amid"); + r = doSearch(collapse, q, 0, 3, chained); + + assertEquals(2, r.getHitCount()); + assertHit("http://acme.org/a.html",10,1,r.hits().get(0)); + assertHit("http://acme.org/c.html",10,0,r.hits().get(1)); + } + + public static class QueryMessupSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + AndItem a = new AndItem(); + a.addItem(query.getModel().getQueryTree().getRoot()); + a.addItem(new WordItem("b")); + query.getModel().getQueryTree().setRoot(a); + + return execution.search(query); + } + } + + public void testQueryTransformAndCollapsing() { + // Set up + Map chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + Searcher messUp = new QueryMessupSearcher(); + + chained.put(collapse, messUp); + chained.put(messUp, docsource); + + // Caveat: Collapse is set to false, because that's what the + // collapser asks for + Query q = new Query("?query=test_collapse+b&collapsefield=amid"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,0)); + r.hits().add(createHit("http://acme.org/d.html", 8,0)); + r.hits().add(createHit("http://acme.org/e.html", 8,0)); + r.hits().add(createHit("http://acme.org/f.html", 7,0)); + r.hits().add(createHit("http://acme.org/g.html", 7,0)); + r.hits().add(createHit("http://acme.org/h.html", 6,1)); + r.setTotalHitCount(8); + docsource.addResult(q, r); + + // Test basic collapsing on mid + q = new Query("?query=test_collapse&collapsefield=amid"); + r = doSearch(collapse, q, 0, 2, chained); + + assertEquals(2, docsource.getQueryCount()); + assertEquals(2, r.getHitCount()); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); + } + + // This test depends on DocumentSourceSearcher filling the hits + // with whatever data it got, ignoring actual summary arguments + // in the fill call, then saying the hits are filled for the + // ignored argument. Rewrite to contain different summaries if + // DocumentSourceSearcher gets extended. + public void testFieldCollapsingTwoPhaseSelectSummary() { + // Set up + Map chained = new HashMap<>(); + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(collapse, docsource); + // Caveat: Collapse is set to false, because that's what the + // collapser asks for + Query q = new Query("?query=test_collapse&collapsefield=amid&summary=placeholder"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.setTotalHitCount(8); + docsource.addResult(q, r); + + // Test basic collapsing on mid + q = new Query("?query=test_collapse&collapsefield=amid&summary=placeholder"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertTrue(r.isFilled("placeholder")); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + + docsource.resetQueryCount(); + // Test basic collapsing on mid + q = new Query("?collapse.summary=short&query=test_collapse&collapsefield=amid&summary=placeholder"); + r = doSearch(collapse, q, 0, 10, chained); + + assertEquals(4, r.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertFalse(r.isFilled("placeholder")); + assertTrue(r.isFilled("short")); + assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + } + + public void testFieldCollapsingWithGrouping() { + // Set up + FieldCollapsingSearcher collapse = new FieldCollapsingSearcher("other"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + Chain chain=new Chain<>(collapse,new AddAggregationStyleGroupingResultSearcher(),docsource); + + // Caveat: Collapse is set to false, because that's what the + // collapser asks for + Query q = new Query("?query=test_collapse&collapsefield=amid"); + // The searcher turns off collapsing further on in the chain + q.properties().set("collapse", "0"); + Result r = new Result(q); + r.hits().add(createHit("http://acme.org/a.html",10,0)); + r.hits().add(createHit("http://acme.org/b.html", 9,0)); + r.hits().add(createHit("http://acme.org/c.html", 9,1)); + r.hits().add(createHit("http://acme.org/d.html", 8,1)); + r.hits().add(createHit("http://acme.org/e.html", 8,2)); + r.hits().add(createHit("http://acme.org/f.html", 7,2)); + r.hits().add(createHit("http://acme.org/g.html", 7,3)); + r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.setTotalHitCount(8); + docsource.addResult(q, r); + + // Test basic collapsing on mid + Query query = new Query("?query=test_collapse&collapsefield=amid"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + + // Assert that the regular hits are collapsed + assertEquals(4+1, result.getHitCount()); + assertEquals(1, docsource.getQueryCount()); + assertHit("http://acme.org/a.html",10,0,result.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1,result.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2,result.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3,result.hits().get(3)); + + // Assert that the aggregation group hierarchy is left intact + HitGroup root= getFirstGroupIn(result.hits()); + assertNotNull(root); + assertEquals("group:root:",root.getId().stringValue().substring(0,11)); // The id ends by a global counter currently + assertEquals(1,root.size()); + HitGroup groupList= (GroupList)root.get("grouplist:g1"); + assertNotNull(groupList); + assertEquals(1,groupList.size()); + HitGroup group= (HitGroup)groupList.get("group:long:37"); + assertNotNull(group); + } + + private Group getFirstGroupIn(HitGroup hits) { + for (Hit h : hits) { + if (h instanceof Group) return (Group)h; + } + return null; + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits, Map chained) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher, chained).search(query); + } + + private Chain chainedAsSearchChain(Searcher topOfChain, Map chained) { + List searchers = new ArrayList<>(); + for (Searcher current = topOfChain; current != null; current = chained.get(current)) { + searchers.add(current); + } + return new Chain<>(searchers); + } + + private Execution createExecution(Searcher searcher, Map chained) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher, chained), context); + } + + /** + * Simulates the return when grouping is used for aggregation purposes and there is a plain hit list in addition: + * The returned result contains both regular hits at the top level (from non-grouping) + * and groups contained aggregation information. + */ + private static class AddAggregationStyleGroupingResultSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result r=execution.search(query); + r.hits().add(createAggregationGroup("g1")); + return r; + } + + private HitGroup createAggregationGroup(String label) { + Group root = new Group(new RootId(0), new Relevance(1)); + GroupList groupList = new GroupList(label); + root.add(groupList); + Group value=new Group(new LongId(37l),new Relevance(2.11)); + groupList.add(value); + return root; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java new file mode 100644 index 00000000000..d868a0f0329 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JSONDebugSearcherTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.searcher.JSONDebugSearcher; +import com.yahoo.processing.execution.Execution.Trace; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Visit the trace and check JSON payload is stored there when requested. + * + * @author Steinar Knutsen + */ +public class JSONDebugSearcherTestCase { + + private static final String NODUMPJSON = "?query=1&tracelevel=6"; + private static final String DUMPJSON = "?query=1&dumpjson=jsonfield&tracelevel=6"; + private Chain searchChain; + + private static class LookForJson extends TraceVisitor { + private static final String JSON_PAYLOAD = "{1: 2}"; + public boolean gotJson = false; + + @Override + public void visit(TraceNode node) { + if (node.payload() == null || node.payload().getClass() != String.class) { + return; + } + if (node.payload().toString().equals(JSONDebugSearcher.JSON_FIELD + JSON_PAYLOAD)) { + gotJson = true; + } + } + } + + private Chain makeSearchChain(String content, Searcher dumper) { + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + addResult(new Query(DUMPJSON), content, docsource); + addResult(new Query(NODUMPJSON), content, docsource); + return new Chain<>(dumper, docsource); + } + + private void addResult(Query q, String content, DocumentSourceSearcher docsource) { + Result r = new Result(q); + FastHit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField("jsonfield", new JSONString(content)); + r.hits().add(hit); + docsource.addResult(q, r); + } + + + @Before + public void setUp() throws Exception { + searchChain = makeSearchChain("{1: 2}", new JSONDebugSearcher()); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void test() { + Execution e = new Execution(searchChain, Execution.Context.createContextStub()); + e.search(new Query(NODUMPJSON)); + Trace t = e.trace(); + LookForJson visitor = new LookForJson(); + t.accept(visitor); + assertEquals(false, visitor.gotJson); + e = new Execution(searchChain, Execution.Context.createContextStub()); + e.search(new Query(DUMPJSON)); + t = e.trace(); + t.accept(visitor); + assertEquals(true, visitor.gotJson); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java new file mode 100644 index 00000000000..a2a149a694a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -0,0 +1,263 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; +import com.yahoo.prelude.searcher.JuniperSearcher; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests conversion of juniper highlighting to XML + * + * @author Steinar Knutsen + */ +public class JuniperSearcherTestCase { + + /** + * Creates a search chain which always returns a result with one hit containing information given in this + * + * @param sdName the search definition type of the returned hit + * @param content the content of the "dynteaser" field of the returned hit + */ + private Chain createSearchChain(String sdName, String content) { + JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), + new QrSearchersConfig(new QrSearchersConfig.Builder())); + + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + addResult(new Query("?query=12"), sdName, content, docsource); + addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); + return new Chain(searcher, docsource); + } + + private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + Result r = new Result(query); + FastHit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField(Hit.SDDOCNAME_FIELD, sdName); + hit.setField("dynteaser", content); + r.hits().add(hit); + docsource.addResult(query, r); + } + + /** Creates a result of the search definiton "one" */ + private Result createResult(String content) { + return createResult("one", content, true); + } + + private Result createResult(String sdName, String content) { + return createResult(sdName, content, true); + } + + private Result createResult(String sdName, String content, boolean bolding) { + Chain chain = createSearchChain(sdName, content); + Query query = new Query("?query=12"); + if ( ! bolding) + query = new Query("?query=12&bolding=false"); + Execution execution = createExecution(chain); + Result result = execution.search(query); + execution.fill(result); + return result; + } + + private Execution createExecution(Chain chain) { + Map> clusters = new LinkedHashMap<>(); + Map searchDefs = new LinkedHashMap<>(); + searchDefs.put("one", createSearchDefinitionOne()); + searchDefs.put("two", createSearchDefinitionTwo()); + SearchDefinition union = new SearchDefinition("union"); + IndexModel indexModel = new IndexModel(clusters, searchDefs, union); + return new Execution(chain, Execution.Context.createContextStub(new IndexFacts(indexModel))); + } + + private SearchDefinition createSearchDefinitionOne() { + SearchDefinition one = new SearchDefinition("one"); + + Index dynteaser = new Index("dynteaser"); + dynteaser.setDynamicSummary(true); + one.addIndex(dynteaser); + + Index bigteaser = new Index("bigteaser"); + dynteaser.setHighlightSummary(true); + one.addIndex(bigteaser); + + Index otherteaser = new Index("otherteaser"); + otherteaser.setDynamicSummary(true); + one.addIndex(otherteaser); + + return one; + } + + private SearchDefinition createSearchDefinitionTwo() { + SearchDefinition two = new SearchDefinition("two"); + return two; + } + + @Test + public void testFieldRewriting() { + Result check = createResult("\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&"); + assertEquals(1, check.getHitCount()); + assertEquals("XYZQWEJKLASD&", + check.hits().get(0).getField("dynteaser").toString()); + check = createResult("a&b&c"); + assertEquals(1, check.getHitCount()); + assertEquals("a&b&c", + check.hits().get(0).getField("dynteaser").toString()); + } + + @Test + public void testNoRewritingDueToSearchDefinition() { + Result check = createResult("two", "\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&"); + assertEquals(1, check.getHitCount()); + assertEquals("\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&", + check.hits().get(0).getField("dynteaser").toString()); + check = createResult("a&b&c"); + assertEquals(1, check.getHitCount()); + assertEquals("a&b&c", + check.hits().get(0).getField("dynteaser").toString()); + } + + @Test + public void testBoldingEquals() { + assertFalse(new Query("?query=12").equals(new Query("?query=12&bolding=false"))); + } + + @Test + public void testUnboldedRewriting() { + Result check = createResult("one", "\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&", false); + assertEquals(1, check.getHitCount()); + assertEquals("XYZ...QWEJKLASD&", + check.hits().get(0).getField("dynteaser").toString()); + } + + @Test + public void testAnnotatedSummaryFields() { + Result check = createResult("\uFFF9Feeding\uFFFAfeed\uFFFB \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F into Vespa \uFFF9is\uFFFAbe\u001Eincrement of a set of \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F fed into Vespa \uFFF9is\u001Efloat in XML when \u001Fdocument\u001F attribute \uFFF9is\uFFFAbe\uFFFB int\u001E"); + assertEquals(1, check.getHitCount()); + assertEquals("Feeding documents into Vespa isincrement of a set of documents fed into Vespa float in XML when document attribute is int", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("one", "\uFFF9Feeding\uFFFAfeed\uFFFB \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F into Vespa \uFFF9is\uFFFAbe\u001Eincrement of a set of \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F fed into Vespa \uFFF9is\u001Efloat in XML when \u001Fdocument\u001F attribute \uFFF9is\uFFFAbe\uFFFB int\u001E", false); + assertEquals(1, check.getHitCount()); + assertEquals("Feeding documents into Vespa is...increment of a set of documents fed into Vespa ...float in XML when document attribute is int...", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("\u001ecommon the term \uFFF9is\uFFFAbe\uFFFB within the set of \u001f\uFFF9documents\uFFFAdocument\uFFFB\u001f. Hence, unusual \uFFF9terms\uFFFAterm\uFFFB or \uFFF9phrases\uFFFAphrase\u001eadded\uFFFAadd\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9given\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("common the term is within the set of documents. Hence, unusual terms or phrases to as a remedy). Each of the document fields in a catalog can be ", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("\u001e\uFFF9is\uFFFAbe\uFFFB within the set of \u001f\uFFF9documents\uFFFAdocument\uFFFB\u001f. \uFFF9phrases\uFFFAphrase\uFFFB\u001E\uFFFAadd\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9given\uFFFA\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("is within the set of documents. phrases to as a remedy). Each of the document fields in a catalog can be given", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("\u001eis\uFFFAbe\uFFFB within the set of \u001f\uFFF9documents\uFFFAdocument\uFFFB\u001f. \uFFF9phrases\uFFFAphrase\u001Eadd\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9given\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" within the set of documents. phrases to as a remedy). Each of the document fields in a catalog can be ", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("\u001e\uFFFAbe\uFFFB within the set of \u001f\uFFF9documents\uFFFAdocument\uFFFB\u001f. \uFFF9phrases\uFFFA\u001E\uFFFA\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" within the set of documents. phrases to as a remedy). Each of the document fields in a catalog can be ", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("\u001e\uFFFAbe\uFFFB within the set of \u001f\uFFF9documents\uFFFAdocument\uFFFB\u001f\uFFF9phrases\uFFFA\u001E\uFFFA\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" within the set of documentsphrases to as a remedy). Each of the document fields in a catalog can be ", check.hits().get(0).getField("dynteaser").toString()); + + check = createResult("\u001e\uFFFAbe\uFFFB within the set of \uFFF9documents\uFFFAdocument\uFFFB\uFFF9phrases\uFFFA\u001E\uFFFA\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" within the set of documentsphrases to as a remedy). Each of the document fields in a catalog can be ", check.hits().get(0).getField("dynteaser").toString()); + } + + @Test + public void testThatIncompleteAnnotationWithHighlightIsIgnored() { + // Look at bug 5707026 for details. + { + Result check = createResult("of\u001e\u001fyahoo\u001f\uFFFB! \uFFF9Angels\uFFFAangels\uFFFB \uFFF9\u001fYahoo\u001f\uFFFA\u001fyahoo\u001f\uFFFB! \uFFF9Angles\uFFFAangels\uFFFB \uFFF9is\uFFFAbe\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("of! Angels Yahoo! Angles is", + check.hits().get(0).getField("dynteaser").toString()); + } + { + Result check = createResult("\u001e\u001fY\u001f\uFFFA\u001fy\u001f\uFFFB! \uFFF9News\uFFFAnews\uFFFB \uFFF9RSS\uFFFArss\uFFFB \uFFF9\u001fY\u001f\uFFFA\u001fy\u001f\uFFFB!\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("! News RSS Y!", + check.hits().get(0).getField("dynteaser").toString()); + } + } + + @Test + public void testThatIncompleteAnnotationWithHighlightAtTheBeginningIsIgnored() { + { + Result check = createResult("\u001e\u001fIncomplete\uFFFAincomplete\uFFFB\u001f \uFFF9Original\uFFFAstemmed\uFFFB\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" Original", check.hits().get(0).getField("dynteaser").toString()); + } + { + Result check = createResult("\u001e\u001f\uFFFAincomplete\uFFFB\u001f \uFFF9Original\uFFFAstemmed\uFFFB\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" Original", check.hits().get(0).getField("dynteaser").toString()); + } + { + Result check = createResult("\u001e\u001fincomplete\uFFFB\u001f \uFFF9Original\uFFFAstemmed\uFFFB\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals(" Original", check.hits().get(0).getField("dynteaser").toString()); + } + } + + @Test + public void testThatIncompleteAnnotationWithHighlightAtTheEndIsIgnored() { + { + Result check = createResult("\u001e\uFFF9Original\uFFFAstemmed\uFFFB \u001f\uFFF9Incomplete\uFFFAincomplete\u001f\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("Original ", check.hits().get(0).getField("dynteaser").toString()); + } + { + Result check = createResult("\u001e\uFFF9Original\uFFFAstemmed\uFFFB \u001f\uFFF9Incomplete\uFFFA\u001f\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("Original ", check.hits().get(0).getField("dynteaser").toString()); + } + { + Result check = createResult("\u001e\uFFF9Original\uFFFAstemmed\uFFFB \u001f\uFFF9Incomplete\u001f\u001e"); + assertEquals(1, check.getHitCount()); + assertEquals("Original ", check.hits().get(0).getField("dynteaser").toString()); + } + } + + @Test + public void testExplicitTwoPhase() { + Chain searchChain = createSearchChain("one", "\u001e\uFFFAbe\uFFFB within the set of \u001f\uFFF9documents\uFFFAdocument\uFFFB\u001f. \uFFF9phrases\uFFFA\u001E\uFFFA\uFFFB to as a remedy). Each of the \u001fdocument\u001f \uFFF9fields\uFFFAfield\uFFFB in a catalog can be \uFFF9\u001e"); + Query q = new Query("?query=12"); + Result check = createExecution(searchChain).search(q); + assertEquals(1, check.getHitCount()); + assertNull(check.hits().get(0).getField("dynteaser")); + createExecution(searchChain).fill(check); + assertEquals(1, check.getHitCount()); + assertEquals(" within the set of documents. phrases to as a remedy). Each of the document fields in a catalog can be ", check.hits().get(0).getField("dynteaser").toString()); + } + + @Test + public void testCompoundWordsBolding() { + Result check = createResult("\u001eTest \u001fkommunikations\u001f\u001ffehler\u001f"); + assertEquals(1, check.getHitCount()); + assertEquals("Test kommunikationsfehler", check.hits().get(0).getField("dynteaser").toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java new file mode 100644 index 00000000000..6a329185ef1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java @@ -0,0 +1,184 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdString; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.searcher.KeyValueSearcher; +import com.yahoo.prelude.searcher.KeyvalueConfig; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import org.junit.Before; +import org.junit.Test; + +import java.util.*; +import java.util.Map.Entry; + +import static org.junit.Assert.*; + +public class KeyValueSearcherTest { + + private static class BackendMockup extends Searcher { + private final Map> dataMap; + private final String summaryType; + + public BackendMockup(Map> dataMap, String summaryType) { + this.dataMap = dataMap; + this.summaryType = summaryType; + } + + @Override + public Result search(Query query, Execution execution) { + fail("Should not do search against backend"); + return null; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + if (containsNullItem(result.getQuery().getModel().getQueryTree().getRoot())) + fail("Got a query with a NullItem root. This cannot be encoded."); + int numEmpty = 0; + for (Hit hit : result.hits()) { + FastHit fhit = (FastHit) hit; + Entry data = dataMap.get(fhit.getGlobalId()); + if (data != null) { + fhit.setField(data.getKey(), data.getValue()); + fhit.setFilled(summaryType); + } else { + numEmpty++; + } + } + if (numEmpty > 0) { + result.hits().addError(ErrorMessage.createBackendCommunicationError("One or more hits were not filled")); + } + } + } + + private Map> dataMap; + private BackendMockup backend; + @Before + public void setupBackend() { + dataMap = new HashMap<>(); + dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::foo")), new AbstractMap.SimpleEntry<>("foo", "foovalue")); + dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::bar")), new AbstractMap.SimpleEntry<>("bar", "barvalue")); + dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::this_must_be_a_key_in_part1_fsadfasdfa")), new AbstractMap.SimpleEntry<>("this_must_be_a_key_in_part1_fsadfasdfa", "blabla")); + backend = new BackendMockup(dataMap, "mysummary"); + } + + @Test + public void testKeyValueSearcher() { + Result result = executeQuery(getConfigString(1), "?keys=foo,bar"); + assertEquals(2, result.getTotalHitCount()); + for (Hit hit : result.hits()) { + FastHit fhit = (FastHit)hit; + Entry data = dataMap.get(fhit.getGlobalId()); + assertEquals(data.getValue(), hit.getField(data.getKey())); + assertTrue(hit.isFilled("mysummary")); + } + + result = executeQuery(getConfigString(1), + "?keys=blabla,fofo", new BackendMockup(dataMap, "mysummary")); + assertEquals(0, result.getTotalHitCount()); + + result = executeQuery(getConfigString(1), + "?keys=non,foo,slsl", new BackendMockup(dataMap, "mysummary")); + assertEquals(1, result.getTotalHitCount()); + } + + @Test + public void testKeyValueSearcherWithNullItemAsQuery() { + Query query = new Query("?keys=foo,bar"); + AndItem and = new AndItem(); + and.addItem(new NullItem()); + query.getModel().getQueryTree().setRoot(and); + Result result = executeQuery(getConfigString(1), query); + assertEquals(2, result.getTotalHitCount()); + } + + private static String getConfigString(int numRows) { + return "raw:numparts 2\nsummaryName \"mysummary\"\ndocIdType \"keyvalue\"\ndocIdNameSpace \"keyvalue\"\nnumrows " + numRows + "\n"; + } + + @Test + public void requireThatIgnoreRowBitsIsEnabledInGeneratedHits() { + Result result = executeQuery(getConfigString(1), + "?keys=foo,bar"); + for (Hit hit : result.hits()) { + FastHit fastHit = (FastHit)hit; + assertTrue(fastHit.shouldIgnoreRowBits()); + } + } + + @Test + public void requireThatNumRowsIsAPositiveNumber() { + for (int i = -10; i < 1; ++i) { + try { + newKeyValueSearcher(getConfigString(i)); + fail(); + } catch (IllegalArgumentException e) { + + } + } + for (int i = 1; i < 10; ++i) { + assertNotNull(newKeyValueSearcher(getConfigString(i))); + } + } + + @Test + public void requireThatNumRowBitsAreCalculatedCorrectly() { + assertRowBits(1, 0); + assertRowBits(2, 1); + assertRowBits(3, 2); + assertRowBits(4, 2); + assertRowBits(5, 3); + assertRowBits(10, 4); + assertRowBits(100, 7); + assertRowBits(1000, 10); + } + + private void assertRowBits(int numRows, int expectedNumRowBits) { + Result result = executeQuery(getConfigString(numRows), "?keys=this_must_be_a_key_in_part1_fsadfasdfa"); + assertEquals(1, result.hits().size()); + FastHit hit = (FastHit)result.hits().get(0); + assertEquals(0, hit.getPartId() & ((1 << expectedNumRowBits) - 1)); + assertEquals(1, hit.getPartId() >> expectedNumRowBits); + } + + private Result executeQuery(String configId, String queryString, Searcher... searchers) { + return executeQuery(configId, new Query(queryString), searchers); + } + + private Result executeQuery(String configId, Query query, Searcher... searchers) { + List chain = new LinkedList<>(); + chain.add(newKeyValueSearcher(configId)); + chain.addAll(Arrays.asList(searchers)); + chain.add(backend); + return new Execution(new Chain<>(chain), Execution.Context.createContextStub()).search(query); + } + + + private static KeyValueSearcher newKeyValueSearcher(String configId) { + return new KeyValueSearcher(new ConfigGetter<>(KeyvalueConfig.class).getConfig(configId)); + } + + private static boolean containsNullItem(Item item) { + if (item instanceof NullItem) return true; + if (item instanceof CompositeItem) { + for (Iterator i = ((CompositeItem)item).getItemIterator(); i.hasNext(); ) + if (containsNullItem(i.next())) + return true; + } + return false; + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/MultipleResultsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/MultipleResultsTestCase.java new file mode 100644 index 00000000000..4898be0afec --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/MultipleResultsTestCase.java @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.searcher.MultipleResultsSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; + +/** + * Test of MultipleResultsSearcher + * + * @author tonytv + */ +@SuppressWarnings("deprecation") +public class MultipleResultsTestCase extends junit.framework.TestCase { + + private DocumentSourceSearcher docSource; + + private MultipleResultsSearcher searcher; + + private Chain chain; + + protected void setUp() { + docSource=new DocumentSourceSearcher(); + searcher=new MultipleResultsSearcher(); + chain=new Chain<>("multipleresultschain",searcher,docSource); + } + + + public void testRetrieveHeterogenousHits() { + Query query = createQuery(); + + Result originalResult = new Result(query); + int n1 = 15, n2 = 25, n3 = 25, n4=25; + addHits(originalResult, "others", n1); + addHits(originalResult, "music", n2); + addHits(originalResult, "movies", n3); + addHits(originalResult, "others", n4); + originalResult.setTotalHitCount(n1 + n2 + n3 + n4); + + docSource.addResult(query, originalResult); + + query.setWindow(0,30); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + + HitGroup musicGroup = (HitGroup)result.hits().get("music"); + HitGroup moviesGroup = (HitGroup)result.hits().get("movies"); + + assertEquals( 15, musicGroup.size() ); + assertEquals( 15, moviesGroup.size() ); + assertEquals( 3, docSource.getQueryCount() ); + } + + public void testRetrieveHitsForGroup() { + Query query = createQuery(); + + Result originalResult = new Result(query); + int n1 = 200, n2=30; + addHits(originalResult, "music", n1, 1000); + addHits(originalResult, "movies", n2, 100); + originalResult.setTotalHitCount(n1 + n2); + + docSource.addResult(query, originalResult); + + Query restrictedQuery = createQuery("movies"); + Result restrictedResult = new Result(restrictedQuery); + addHits(restrictedResult, "movies", n2, 100); + restrictedResult.setTotalHitCount(n2); + + docSource.addResult(restrictedQuery, restrictedResult); + + query.setWindow(0,30); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + + HitGroup musicGroup = (HitGroup)result.hits().get("music"); + HitGroup moviesGroup = (HitGroup)result.hits().get("movies"); + + assertEquals( 15, musicGroup.size()); + assertEquals( 15, moviesGroup.size()); + } + + public void testNoHitsForResultSet() { + Query query = createQuery(); + + Result originalResult = new Result(query); + int n1 = 20; + int n2 = 100; + addHits(originalResult, "music", n1); + addHits(originalResult, "other", n2); + originalResult.setTotalHitCount(n1 + n2); + + docSource.addResult(query, originalResult); + + query.setWindow(0,30); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + + HitGroup musicGroup = (HitGroup)result.hits().get("music"); + HitGroup moviesGroup = (HitGroup)result.hits().get("movies"); + + assertEquals( 15, musicGroup.size()); + assertEquals( 0, moviesGroup.size()); + } + + private void addHits(Result result, String docName, int numHits, + int baseRelevancy) { + for (int i=0; iGunnar Gauslaa Bergem + */ +@SuppressWarnings("deprecation") +public class PosSearcherTestCase extends junit.framework.TestCase { + private PosSearcher searcher = new PosSearcher(); + private Query q; + + public PosSearcherTestCase(String name) { + super(name); + } + + /** + * Tests basic lat/long input. + */ + public void testBasics() { + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,450668,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.ll", "N60;E30"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,30000000,60000000,450668,0,1,0,2147483647)", q.getRanking().getLocation().toString()); + } + + /** + * Tests basic bounding box input. + */ + public void testBoundingBox() { + q = new Query(); + q.properties().set("pos.bb", "n=51.9,s=50.2,e=0.34,w=-10.01"); + doSearch(searcher, q, 0, 10); + assertEquals("[2,-10010000,50200000,340000,51900000]", + q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.bb", "n=0,s=0,e=123.456789,w=-123.456789"); + doSearch(searcher, q, 0, 10); + assertEquals("[2,-123456789,0,123456789,0]", + q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.bb", "n=12.345678,s=-12.345678,e=0,w=0"); + doSearch(searcher, q, 0, 10); + assertEquals("[2,0,-12345678,0,12345678]", + q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.bb", "n=0.000001,s=-0.000001,e=0.000001,w=-0.000001"); + doSearch(searcher, q, 0, 10); + assertEquals("[2,-1,-1,1,1]", + q.getRanking().getLocation().toString()); + } + + /** + * Tests basic bounding box input. + */ + public void testBoundingBoxAndRadius() { + q = new Query(); + q.properties().set("pos.bb", "n=60.111,s=59.999,e=30.111,w=29.999"); + q.properties().set("pos.ll", "N60;E30"); + doSearch(searcher, q, 0, 10); + assertEquals( + "[2,29999000,59999000,30111000,60111000]" + + "(2,30000000,60000000,450668,0,1,0,2147483647)", + q.getRanking().getLocation().toString()); + } + + /** + * Tests different ways of specifying the radius. + */ + public void testRadiusUnits() { + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + q.properties().set("pos.radius", "2km"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,18026,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + q.properties().set("pos.radius", "2000m"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,18026,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + q.properties().set("pos.radius", "20mi"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,290112,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + q.properties().set("pos.radius", "2km"); + q.properties().set("pos.units", "udeg"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,18026,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + } + + /** + * Tests xy position (internal format). + */ + public void testXY() { + q = new Query(); + q.properties().set("pos.xy", "22500;22500"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.xy", "22500;22500"); + q.properties().set("pos.units", "unknown"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString()); + } + + public void testNotOverridingOldStyleParameters() { + PosSearcher searcher = new PosSearcher(); + q = new Query("?query=test&pos.ll=N10.15;E6.08&location=(2,-1100222,0,300,0,1,0,CalcLatLon)"); + q.setTraceLevel(1); + doSearch(searcher, q, 0, 10); + assertEquals("(2,-1100222,0,300,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + //assertEquals("query already has a location set, not processing 'pos' params", + // result.getHit(0).getFeedback(0)); + } + + /** + * Tests input parameters that should report errors. + */ + public void testInvalidInput() { + PosSearcher searcher = new PosSearcher(); + Result result; + + q = new Query(); + q.properties().set("pos.ll", "NE74.14;E14.48"); + result = doSearch(searcher, q, 0, 10); + assertEquals("Error in pos parameters: Unable to parse lat/long string 'NE74.14;E14.48': already set direction once, cannot add direction: E", + ((ErrorHit)result.hits().get(0)).errors().iterator().next().getDetailedMessage()); + + q = new Query(); + q.properties().set("pos.ll", "NE74.14;E14.48"); + q.properties().set("pos.xy", "82400, 72800"); + result = doSearch(searcher, q, 0, 10); + assertEquals("Error in pos parameters: Cannot handle both lat/long and xy coords at the same time", + ((ErrorHit)result.hits().get(0)).errors().iterator().next().getDetailedMessage()); + } + + public void testWrappingTheNorthPole() { + q = new Query(); + q.properties().set("pos.ll", "N89.9985365158;E122.163600102"); + q.properties().set("pos.radius", "20mi"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,122163600,89998536,290112,0,1,0,109743)", q.getRanking().getLocation().toString()); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java new file mode 100644 index 00000000000..2cda888a2e2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.prelude.searcher.QuerySnapshotSearcher; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class QuerySnapshotSearcherTestCase extends junit.framework.TestCase { + + public void test() { + Searcher searcher=new QuerySnapshotSearcher(); + Result result = doSearch(searcher, new Query(), 0,10); + Hit hit=result.hits().get(0); + assertEquals(String.valueOf(Double.POSITIVE_INFINITY), + hit.getRelevance().toString()); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java new file mode 100644 index 00000000000..ee69fa92a17 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; +import com.yahoo.prelude.searcher.QueryValidatingSearcher; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests correct denial of query. + * + * @author Steinar Knutsen + */ +public class QueryValidatingSearcherTestCase extends junit.framework.TestCase { + + public QueryValidatingSearcherTestCase(String name) { + super(name); + } + + public void testBasic() { + // Setup + Map chained = new HashMap<>(); + Query query = new Query("?query=test"); + + Result result = new Result(query); + result.hits().add(new Hit("ymail://1111111111/AQAAAP7JgwEAj6XjQQAAAO/+ggA=",950)); + + Searcher validator = new QueryValidatingSearcher(); + DocumentSourceSearcher source = new DocumentSourceSearcher(); + chained.put(validator, source); + source.addResult(query, result); + + // Exercise + Result returnedResult = doSearch(validator, query, 0, 10, chained); + + // Validate + assertEquals(1, returnedResult.getHitCount()); + assertNull(returnedResult.hits().getError()); + + returnedResult = doSearch(validator, query, 0, 1001, chained); + assertEquals(0, returnedResult.getConcreteHitCount()); + assertEquals(4, returnedResult.hits().getError().getCode()); + + returnedResult = doSearch(validator, query, 1001, 10, chained); + assertEquals(0, returnedResult.getConcreteHitCount()); + assertEquals(4, returnedResult.hits().getError().getCode()); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits, Map chained) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher, chained).search(query); + } + + private Chain chainedAsSearchChain(Searcher topOfChain, Map chained) { + List searchers = new ArrayList<>(); + for (Searcher current = topOfChain; current != null; current = chained.get(current)) { + searchers.add(current); + } + return new Chain<>(searchers); + } + + private Execution createExecution(Searcher searcher, Map chained) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher, chained), context); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java new file mode 100644 index 00000000000..4dd8480c84c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.searcher.QrQuotetableConfig; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.HitField; +import com.yahoo.search.Searcher; +import com.yahoo.prelude.searcher.DocumentSourceSearcher; +import com.yahoo.prelude.searcher.QuotingSearcher; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests hit property quoting. + * + * @author Steinar Knutsen + */ +@SuppressWarnings("deprecation") +public class QuotingSearcherTestCase extends junit.framework.TestCase { + + public QuotingSearcherTestCase (String name) { + super(name); + } + + public static QuotingSearcher createQuotingSearcher(String configId) { + QrQuotetableConfig config = new ConfigGetter<>(QrQuotetableConfig.class).getConfig(configId); + return new QuotingSearcher(new ComponentId("QuotingSearcher"), config); + } + + public void testBasicQuoting() { + Map chained = new HashMap<>(); + Searcher s = createQuotingSearcher("file:src/test/java/com/yahoo/prelude/" + + "searcher/test/testquoting.cfg"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(s, docsource); + Query q = new Query("?query=a"); + Result r = new Result(q); + Hit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField("title", "smith & jones"); + r.hits().add(hit); + docsource.addResultSet(q, r); + Result check = doSearch(s, q, 0, 10, chained); + assertEquals("smith & jones", check.hits().get(0).getField("title").toString()); + assertTrue(check.hits().get(0).fields().containsKey("title")); + } + + public void testBasicQuotingWithNoisyStrings() { + Map chained = new HashMap<>(); + Searcher s = createQuotingSearcher("file:src/test/java/com/yahoo/prelude/" + + "searcher/test/testquoting.cfg"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(s, docsource); + Query q = new Query("?query=a"); + Result r = new Result(q); + Hit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField("title", "&smith &jo& nes"); + r.hits().add(hit); + docsource.addResultSet(q, r); + Result check = doSearch(s, q, 0, 10, chained); + assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); + assertTrue(check.hits().get(0).fields().containsKey("title")); + } + + public void testFieldQuotingWithNoisyStrings() { + Map chained = new HashMap<>(); + Searcher s = createQuotingSearcher("file:src/test/java/com/yahoo/prelude/" + + "searcher/test/testquoting.cfg"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(s, docsource); + Query q = new Query("?query=a"); + Result r = new Result(q); + Hit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField("title", new HitField("title", "&smith &jo& nes")); + r.hits().add(hit); + docsource.addResultSet(q, r); + Result check = doSearch(s, q, 0, 10, chained); + assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); + assertTrue(check.hits().get(0).fields().containsKey("title")); + } + + + public void testNoQuotingWithOtherTypes() { + Map chained = new HashMap<>(); + Searcher s = createQuotingSearcher("file:src/test/java/com/yahoo/prelude/" + + "searcher/test/testquoting.cfg"); + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + chained.put(s, docsource); + Query q = new Query("?query=a"); + Result r = new Result(q); + Hit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField("title", new Integer(42)); + r.hits().add(hit); + docsource.addResultSet(q, r); + Result check = doSearch(s, q, 0, 10, chained); + // should not quote non-string properties + assertEquals(new Integer(42), check.hits().get(0).getField("title")); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits, Map chained) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher, chained).search(query); + } + + private Chain chainedAsSearchChain(Searcher topOfChain, Map chained) { + List searchers = new ArrayList<>(); + for (Searcher current = topOfChain; current != null; current = chained.get(current)) { + searchers.add(current); + } + return new Chain<>(searchers); + } + + private Execution createExecution(Searcher searcher, Map chained) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher, chained), context); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java new file mode 100644 index 00000000000..3c3c6c921e3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.searcher.ValidatePredicateSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.yql.YqlParser; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author Magnar Nedland + */ +public class ValidatePredicateSearcherTestCase { + + @Test + public void testValidQuery() { + ValidatePredicateSearcher searcher = new ValidatePredicateSearcher(); + String q = "select * from sources * where predicate(predicate_field,0,{\"age\":20L});"; + Result r = doSearch(searcher, q, "predicate-bounds [0..99]"); + assertNull(r.hits().getError()); + } + + @Test + public void testQueryOutOfBounds() { + ValidatePredicateSearcher searcher = new ValidatePredicateSearcher(); + String q = "select * from sources * where predicate(predicate_field,0,{\"age\":200L});"; + Result r = doSearch(searcher, q, "predicate-bounds [0..99]"); + assertEquals(ErrorMessage.createIllegalQuery("age=200 outside configured predicate bounds."), r.hits().getError()); + } + + private static Result doSearch(ValidatePredicateSearcher searcher, String yqlQuery, String command) { + QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery)); + Query query = new Query(); + query.getModel().getQueryTree().setRoot(queryTree.getRoot()); + + TreeMap> masterClusters = new TreeMap<>(); + masterClusters.put("cluster", Arrays.asList("document")); + SearchDefinition searchDefinition = new SearchDefinition("document"); + Index index = new Index("predicate_field"); + index.addCommand(command); + searchDefinition.addIndex(index); + Map searchDefinitionMap = new HashMap<>(); + searchDefinitionMap.put("document", searchDefinition); + IndexFacts indexFacts = new IndexFacts(new IndexModel(masterClusters, searchDefinitionMap, searchDefinition)); + Execution.Context context = new Execution.Context(null, indexFacts, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(searcher, context).search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java new file mode 100644 index 00000000000..143604844df --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.searcher.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.searcher.ValidateSortingSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Check sorting validation behaves OK. + * + * @author Steinar Knutsen + */ +public class ValidateSortingSearcherTestCase { + + private final ValidateSortingSearcher searcher; + + public ValidateSortingSearcherTestCase() { + QrSearchersConfig.Builder qrsCfg = new QrSearchersConfig.Builder(); + qrsCfg.searchcluster(new QrSearchersConfig.Searchcluster.Builder().name("giraffes")); + ClusterConfig.Builder clusterCfg = new ClusterConfig.Builder(). + clusterId(0). + clusterName("test"); + String attributesCfg = "file:src/test/java/com/yahoo/prelude/searcher/test/validate_sorting.cfg"; + searcher = new ValidateSortingSearcher(new QrSearchersConfig(qrsCfg), + new ClusterConfig(clusterCfg), + ConfigGetter.getConfig(AttributesConfig.class, attributesCfg)); + } + + @Test + public void testBasicValidation() { + assertNotNull(quoteAndTransform("+a -b +c")); + assertNotNull(quoteAndTransform("+a")); + assertNotNull(quoteAndTransform(null)); + } + + @Test + public void testInvalidSpec() { + assertNull(quoteAndTransform("+a -e +c")); + } + + @Test + public void testConfigOverride() { + assertEquals("[ASCENDING:uca(title,en_US,TERTIARY)]", quoteAndTransform("title")); + assertEquals("[ASCENDING:uca(title,en_US,TERTIARY)]", quoteAndTransform("uca(title)")); + assertEquals("[ASCENDING:uca(title,en_US,TERTIARY)]", quoteAndTransform("+uca(title)")); + assertEquals("[ASCENDING:uca(title,en_US,TERTIARY)]", quoteAndTransform("uca(title,en_US)")); + } + + @Test + public void requireThatQueryLocaleIsDefault() { + assertEquals("[ASCENDING:lowercase(a)]", quoteAndTransform("a")); + assertEquals("[ASCENDING:uca(a,en_US,PRIMARY)]", transform("a", "en-US")); + assertEquals("[ASCENDING:uca(a,en_NO,PRIMARY)]", transform("a", "en-NO")); + assertEquals("[ASCENDING:uca(a,no_NO,PRIMARY)]", transform("a", "no-NO")); + + assertEquals("[ASCENDING:uca(a,en_US,PRIMARY)]", quoteAndTransform("uca(a)")); + assertEquals("[ASCENDING:uca(a,en_US,PRIMARY)]", transform("uca(a)", "en-US")); + assertEquals("[ASCENDING:uca(a,en_NO,PRIMARY)]", transform("uca(a)", "en-NO")); + assertEquals("[ASCENDING:uca(a,no_NO,PRIMARY)]", transform("uca(a)", "no-NO")); + } + + private String quoteAndTransform(String sorting) { + return transform(QueryTestCase.httpEncode(sorting), null); + } + + @SuppressWarnings("deprecation") + private String transform(String sorting, String language) { + String q = "/?query=a"; + if (sorting != null) { + q += "&sorting=" + sorting; + } + if (language != null) { + q += "&language=" + language; + } + new Query(q); + Result r = doSearch(searcher, new Query(q), 0, 10); + if (r.hits().getError() != null) { + return null; + } + if (r.getQuery().getRanking().getSorting() == null) { + return ""; + } + return r.getQuery().getRanking().getSorting().fieldOrders().toString(); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-searchers.cfg new file mode 100644 index 00000000000..5eb21b2756e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-searchers.cfg @@ -0,0 +1,21 @@ + +customizedsearchers.rawquery[0] +customizedsearchers.transformedquery[0] +customizedsearchers.blendedresult[0] +customizedsearchers.unblendedresult[0] +customizedsearchers.backend[0] +customizedsearchers.argument[0] + +searchcluster[2] +searchcluster[0].name music +searchcluster[0].searchdef[1] +searchcluster[0].searchdef[0] music +searchcluster[0].dispatcher[1] +searchcluster[0].dispatcher[0].host localhost +searchcluster[0].dispatcher[0].port 6328 +searchcluster[1].name andtheother +searchcluster[1].searchdef[1] +searchcluster[1].searchdef[0] music +searchcluster[1].dispatcher[1] +searchcluster[1].dispatcher[0].host localhost +searchcluster[1].dispatcher[0].port 6338 diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-summary.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-summary.cfg new file mode 100644 index 00000000000..4dee51baa16 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/qr-summary.cfg @@ -0,0 +1,349 @@ +idtype BYTE +classes[7] +classes[0].name default +classes[0].id 0 +classes[0].fields[19] +classes[0].fields[0].name URL +classes[0].fields[0].type string +classes[0].fields[1].name TITLE +classes[0].fields[1].type string +classes[0].fields[2].name TEASER +classes[0].fields[2].type string +classes[0].fields[3].name TOPIC +classes[0].fields[3].type string +classes[0].fields[4].name FASTTOPIC +classes[0].fields[4].type string +classes[0].fields[5].name EXTINFO +classes[0].fields[5].type string +classes[0].fields[6].name EXTINFOSOURCE +classes[0].fields[6].type byte +classes[0].fields[7].name DSHOST +classes[0].fields[7].type integer +classes[0].fields[8].name DSKEY +classes[0].fields[8].type integer +classes[0].fields[9].name BYTES +classes[0].fields[9].type integer +classes[0].fields[10].name WORDS +classes[0].fields[10].type integer +classes[0].fields[11].name MODDATE +classes[0].fields[11].type integer +classes[0].fields[12].name CRAWLDATE +classes[0].fields[12].type integer +classes[0].fields[13].name LANG1 +classes[0].fields[13].type byte +classes[0].fields[14].name LANG2 +classes[0].fields[14].type byte +classes[0].fields[15].name LANG3 +classes[0].fields[15].type byte +classes[0].fields[16].name LANG4 +classes[0].fields[16].type byte +classes[0].fields[17].name IPADDRESS +classes[0].fields[17].type integer +classes[0].fields[18].name DOCVECTOR +classes[0].fields[18].type data +classes[1].name version1 +classes[1].id 1 +classes[1].fields[20] +classes[1].fields[0].name URL +classes[1].fields[0].type string +classes[1].fields[1].name TITLE +classes[1].fields[1].type string +classes[1].fields[2].name TEASER +classes[1].fields[2].type string +classes[1].fields[3].name TOPIC +classes[1].fields[3].type string +classes[1].fields[4].name FASTTOPIC +classes[1].fields[4].type string +classes[1].fields[5].name EXTINFO +classes[1].fields[5].type string +classes[1].fields[6].name EXTINFOSOURCE +classes[1].fields[6].type byte +classes[1].fields[7].name DSHOST +classes[1].fields[7].type integer +classes[1].fields[8].name DSKEY +classes[1].fields[8].type integer +classes[1].fields[9].name BYTES +classes[1].fields[9].type integer +classes[1].fields[10].name WORDS +classes[1].fields[10].type integer +classes[1].fields[11].name MODDATE +classes[1].fields[11].type integer +classes[1].fields[12].name CRAWLDATE +classes[1].fields[12].type integer +classes[1].fields[13].name LANG1 +classes[1].fields[13].type byte +classes[1].fields[14].name LANG2 +classes[1].fields[14].type byte +classes[1].fields[15].name LANG3 +classes[1].fields[15].type byte +classes[1].fields[16].name LANG4 +classes[1].fields[16].type byte +classes[1].fields[17].name IPADDRESS +classes[1].fields[17].type integer +classes[1].fields[18].name DOCVECTOR +classes[1].fields[18].type data +classes[1].fields[19].name PARTNERSITEIDS +classes[1].fields[19].type string +classes[2].name version2 +classes[2].id 2 +classes[2].fields[21] +classes[2].fields[0].name URL +classes[2].fields[0].type string +classes[2].fields[1].name TITLE +classes[2].fields[1].type string +classes[2].fields[2].name TEASER +classes[2].fields[2].type string +classes[2].fields[3].name TOPIC +classes[2].fields[3].type string +classes[2].fields[4].name FASTTOPIC +classes[2].fields[4].type string +classes[2].fields[5].name EXTINFO +classes[2].fields[5].type string +classes[2].fields[6].name EXTINFOSOURCE +classes[2].fields[6].type byte +classes[2].fields[7].name DSHOST +classes[2].fields[7].type integer +classes[2].fields[8].name DSKEY +classes[2].fields[8].type integer +classes[2].fields[9].name BYTES +classes[2].fields[9].type integer +classes[2].fields[10].name WORDS +classes[2].fields[10].type integer +classes[2].fields[11].name MODDATE +classes[2].fields[11].type integer +classes[2].fields[12].name CRAWLDATE +classes[2].fields[12].type integer +classes[2].fields[13].name LANG1 +classes[2].fields[13].type byte +classes[2].fields[14].name LANG2 +classes[2].fields[14].type byte +classes[2].fields[15].name LANG3 +classes[2].fields[15].type byte +classes[2].fields[16].name LANG4 +classes[2].fields[16].type byte +classes[2].fields[17].name IPADDRESS +classes[2].fields[17].type integer +classes[2].fields[18].name DOCVECTOR +classes[2].fields[18].type data +classes[2].fields[19].name PARTNERSITEIDS +classes[2].fields[19].type string +classes[2].fields[20].name DYNTEASER +classes[2].fields[20].type string +classes[3].name version3 +classes[3].id 3 +classes[3].fields[23] +classes[3].fields[0].name URL +classes[3].fields[0].type string +classes[3].fields[1].name TITLE +classes[3].fields[1].type string +classes[3].fields[2].name TEASER +classes[3].fields[2].type string +classes[3].fields[3].name TOPIC +classes[3].fields[3].type string +classes[3].fields[4].name FASTTOPIC +classes[3].fields[4].type string +classes[3].fields[5].name EXTINFO +classes[3].fields[5].type string +classes[3].fields[6].name EXTINFOSOURCE +classes[3].fields[6].type byte +classes[3].fields[7].name DSHOST +classes[3].fields[7].type integer +classes[3].fields[8].name DSKEY +classes[3].fields[8].type integer +classes[3].fields[9].name BYTES +classes[3].fields[9].type integer +classes[3].fields[10].name WORDS +classes[3].fields[10].type integer +classes[3].fields[11].name MODDATE +classes[3].fields[11].type integer +classes[3].fields[12].name CRAWLDATE +classes[3].fields[12].type integer +classes[3].fields[13].name LANG1 +classes[3].fields[13].type byte +classes[3].fields[14].name LANG2 +classes[3].fields[14].type byte +classes[3].fields[15].name LANG3 +classes[3].fields[15].type byte +classes[3].fields[16].name LANG4 +classes[3].fields[16].type byte +classes[3].fields[17].name IPADDRESS +classes[3].fields[17].type integer +classes[3].fields[18].name DOCVECTOR +classes[3].fields[18].type data +classes[3].fields[19].name PARTNERSITEIDS +classes[3].fields[19].type string +classes[3].fields[20].name MIMETYPE +classes[3].fields[20].type string +classes[3].fields[21].name STATICRANKLOG +classes[3].fields[21].type string +classes[3].fields[22].name DYNTEASER +classes[3].fields[22].type longstring +classes[4].name version4 +classes[4].id 4 +classes[4].fields[24] +classes[4].fields[0].name URL +classes[4].fields[0].type string +classes[4].fields[1].name CCURL +classes[4].fields[1].type string +classes[4].fields[2].name TITLE +classes[4].fields[2].type string +classes[4].fields[3].name TEASER +classes[4].fields[3].type string +classes[4].fields[4].name TOPIC +classes[4].fields[4].type string +classes[4].fields[5].name FASTTOPIC +classes[4].fields[5].type string +classes[4].fields[6].name EXTINFO +classes[4].fields[6].type string +classes[4].fields[7].name EXTINFOSOURCE +classes[4].fields[7].type byte +classes[4].fields[8].name DSHOST +classes[4].fields[8].type integer +classes[4].fields[9].name DSKEY +classes[4].fields[9].type integer +classes[4].fields[10].name BYTES +classes[4].fields[10].type integer +classes[4].fields[11].name WORDS +classes[4].fields[11].type integer +classes[4].fields[12].name MODDATE +classes[4].fields[12].type integer +classes[4].fields[13].name CRAWLDATE +classes[4].fields[13].type integer +classes[4].fields[14].name LANG1 +classes[4].fields[14].type byte +classes[4].fields[15].name LANG2 +classes[4].fields[15].type byte +classes[4].fields[16].name LANG3 +classes[4].fields[16].type byte +classes[4].fields[17].name LANG4 +classes[4].fields[17].type byte +classes[4].fields[18].name IPADDRESS +classes[4].fields[18].type integer +classes[4].fields[19].name DOCVECTOR +classes[4].fields[19].type data +classes[4].fields[20].name PARTNERSITEIDS +classes[4].fields[20].type string +classes[4].fields[21].name MIMETYPE +classes[4].fields[21].type string +classes[4].fields[22].name STATICRANKLOG +classes[4].fields[22].type string +classes[4].fields[23].name DYNTEASER +classes[4].fields[23].type longstring +classes[5].name version5 +classes[5].id 5 +classes[5].fields[25] +classes[5].fields[0].name URL +classes[5].fields[0].type string +classes[5].fields[1].name URLLIST +classes[5].fields[1].type string +classes[5].fields[2].name CCURL +classes[5].fields[2].type string +classes[5].fields[3].name TITLE +classes[5].fields[3].type string +classes[5].fields[4].name TEASER +classes[5].fields[4].type string +classes[5].fields[5].name TOPIC +classes[5].fields[5].type string +classes[5].fields[6].name FASTTOPIC +classes[5].fields[6].type string +classes[5].fields[7].name EXTINFO +classes[5].fields[7].type string +classes[5].fields[8].name EXTINFOSOURCE +classes[5].fields[8].type byte +classes[5].fields[9].name DSHOST +classes[5].fields[9].type integer +classes[5].fields[10].name DSKEY +classes[5].fields[10].type integer +classes[5].fields[11].name BYTES +classes[5].fields[11].type integer +classes[5].fields[12].name WORDS +classes[5].fields[12].type integer +classes[5].fields[13].name MODDATE +classes[5].fields[13].type integer +classes[5].fields[14].name CRAWLDATE +classes[5].fields[14].type integer +classes[5].fields[15].name LANG1 +classes[5].fields[15].type byte +classes[5].fields[16].name LANG2 +classes[5].fields[16].type byte +classes[5].fields[17].name LANG3 +classes[5].fields[17].type byte +classes[5].fields[18].name LANG4 +classes[5].fields[18].type byte +classes[5].fields[19].name IPADDRESS +classes[5].fields[19].type integer +classes[5].fields[20].name DOCVECTOR +classes[5].fields[20].type data +classes[5].fields[21].name PARTNERSITEIDS +classes[5].fields[21].type string +classes[5].fields[22].name MIMETYPE +classes[5].fields[22].type string +classes[5].fields[23].name STATICRANKLOG +classes[5].fields[23].type string +classes[5].fields[24].name DYNTEASER +classes[5].fields[24].type longstring +classes[6].name withranklog +classes[6].id 237 +classes[6].fields[31] +classes[6].fields[0].name BYTES +classes[6].fields[0].type integer +classes[6].fields[1].name CCURL +classes[6].fields[1].type string +classes[6].fields[2].name CRAWLDATE +classes[6].fields[2].type integer +classes[6].fields[3].name DOCVECTOR +classes[6].fields[3].type data +classes[6].fields[4].name DSHOST +classes[6].fields[4].type integer +classes[6].fields[5].name DSKEY +classes[6].fields[5].type integer +classes[6].fields[6].name DYNTEASER +classes[6].fields[6].type longstring +classes[6].fields[7].name DYNTEASERINPUT +classes[6].fields[7].type longstring +classes[6].fields[8].name EXTINFO +classes[6].fields[8].type string +classes[6].fields[9].name EXTINFOSOURCE +classes[6].fields[9].type byte +classes[6].fields[10].name FASTTOPIC +classes[6].fields[10].type string +classes[6].fields[11].name IPADDRESS +classes[6].fields[11].type integer +classes[6].fields[12].name JUNIPER +classes[6].fields[12].type longstring +classes[6].fields[13].name JUNIPERMETRIC +classes[6].fields[13].type integer +classes[6].fields[14].name LABEL +classes[6].fields[14].type string +classes[6].fields[15].name LANG1 +classes[6].fields[15].type byte +classes[6].fields[16].name LANG2 +classes[6].fields[16].type byte +classes[6].fields[17].name LANG3 +classes[6].fields[17].type byte +classes[6].fields[18].name LANG4 +classes[6].fields[18].type byte +classes[6].fields[19].name MIMETYPE +classes[6].fields[19].type string +classes[6].fields[20].name MODDATE +classes[6].fields[20].type integer +classes[6].fields[21].name PARTNERSITEIDS +classes[6].fields[21].type string +classes[6].fields[22].name RANKLOG +classes[6].fields[22].type string +classes[6].fields[23].name STATICRANK +classes[6].fields[23].type integer +classes[6].fields[24].name STATICRANKLOG +classes[6].fields[24].type string +classes[6].fields[25].name TEASER +classes[6].fields[25].type string +classes[6].fields[26].name TITLE +classes[6].fields[26].type string +classes[6].fields[27].name TOPIC +classes[6].fields[27].type string +classes[6].fields[28].name URL +classes[6].fields[28].type string +classes[6].fields[29].name URLLIST +classes[6].fields[29].type string +classes[6].fields[30].name WORDS +classes[6].fields[30].type integer diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserfieldinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserfieldinfo.cfg new file mode 100644 index 00000000000..02cfb5bb618 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserfieldinfo.cfg @@ -0,0 +1,11 @@ +doctype[1] +doctype[0].name one +doctype[0].field[1] + +doctype[0].field[0].name dynteaser +doctype[0].field[0].command[1] +doctype[0].field[0].command[0] bold +doctype[0].field[0].index[2] +doctype[0].field[0].index[0] default +doctype[0].field[0].index[1] dynteaser +doctype[0].field[0].type string diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserquoting.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserquoting.cfg new file mode 100644 index 00000000000..26d9e7b6cea --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testdynteaserquoting.cfg @@ -0,0 +1,16 @@ +character[5] +# & +character[0].ordinal 38 +character[0].quoting & +# < +character[1].ordinal 60 +character[1].quoting < +# ASCII Unit Separator (0x1F) +character[2].ordinal 31 +character[2].quoting US +# Z +character[3].ordinal 90 +character[3].quoting z +# h, to get character contained in default highlight tags +character[4].ordinal 104 +character[4].quoting H diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testfieldinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testfieldinfo.cfg new file mode 100644 index 00000000000..2bfd09230d1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testfieldinfo.cfg @@ -0,0 +1,19 @@ +doctype[1] +doctype[0].name one +doctype[0].field[2] + +doctype[0].field[0].name bigteaser +doctype[0].field[0].command[1] +doctype[0].field[0].command[0] bold +doctype[0].field[0].index[2] +doctype[0].field[0].index[0] default +doctype[0].field[0].index[1] bigteaser +doctype[0].field[0].type string + +doctype[0].field[1].name otherteaser +doctype[0].field[1].command[1] +doctype[0].field[1].command[0] bold +doctype[0].field[1].index[2] +doctype[0].field[1].index[0] default +doctype[0].field[1].index[1] otherteaser +doctype[0].field[1].type string diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testhit.xml b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testhit.xml new file mode 100644 index 00000000000..5f23f575a13 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testhit.xml @@ -0,0 +1,29 @@ + + + + Unspecified error + + + An error as ordered + + + + http://def + 75 + 0 + + + http://a.b/c + 73 + 0 + test/stuff\tsome/other + dklf øæå sdf > & < +Ipsum, etc. + + + http://def + 75 + 0 + hablablbl
&fdlkkgj]]>;lk + + diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testindexinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testindexinfo.cfg new file mode 100644 index 00000000000..0e981e87b15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testindexinfo.cfg @@ -0,0 +1,28 @@ +indexinfo[2] +indexinfo[0].name one +indexinfo[0].command[8] +indexinfo[0].command[0].indexname exactemento +indexinfo[0].command[0].command compact-to-term +indexinfo[0].command[1].indexname default +indexinfo[0].command[1].command stem +indexinfo[0].command[2].indexname default +indexinfo[0].command[2].command normalize +indexinfo[0].command[3].indexname dynteaser +indexinfo[0].command[3].command dynteaser +indexinfo[0].command[4].indexname bigteaser +indexinfo[0].command[4].command highlight +indexinfo[0].command[5].indexname bigteaser +indexinfo[0].command[5].command xmlstring +indexinfo[0].command[6].indexname skey +indexinfo[0].command[6].command term-boost skey 700 +indexinfo[0].command[7].indexname otherteaser +indexinfo[0].command[7].command highlight +indexinfo[1].name two +indexinfo[1].command[3] +indexinfo[1].command[0].indexname default +indexinfo[1].command[0].command compact-to-term +indexinfo[1].command[1].indexname b +indexinfo[1].command[1].command compact-to-term +indexinfo[1].command[2].indexname absolute +indexinfo[1].command[2].command complete-boost 23 + diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testlazymapping.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testlazymapping.cfg new file mode 100644 index 00000000000..3da52628b1b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testlazymapping.cfg @@ -0,0 +1,5 @@ +mapping[1] +mapping[0].name lazy +mapping[0].summary[1] +mapping[0].summary[0].fromname title +mapping[0].summary[0].toname TITLE diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testphysicalmapping.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testphysicalmapping.cfg new file mode 100644 index 00000000000..6ac99c1be42 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testphysicalmapping.cfg @@ -0,0 +1,98 @@ +mapping[3] +mapping[0].name logical1 +mapping[0].field[7] +mapping[0].field[0].fromname special +mapping[0].field[0].toname fs1 +mapping[0].field[1].fromname title +mapping[0].field[1].toname d1 +mapping[0].field[2].fromname description +mapping[0].field[2].toname d2 +mapping[0].field[3].fromname size +mapping[0].field[3].toname fi1 +mapping[0].field[4].fromname onlyin1 +mapping[0].field[4].toname xy8 +mapping[0].field[5].fromname default +mapping[0].field[5].toname logical1default +mapping[0].field[6].fromname weird.fieldname +mapping[0].field[6].toname d3 +mapping[0].context[3] +mapping[0].context[0].toname onlyin1 +mapping[0].context[0].fromname xy8 +mapping[0].context[1].toname size +mapping[0].context[1].fromname fi1 +mapping[0].context[2].toname title +mapping[0].context[2].fromname d1 +mapping[0].summary[5] +mapping[0].summary[0].fromname title +mapping[0].summary[0].toname ss1 +mapping[0].summary[1].fromname description +mapping[0].summary[1].toname ss2 +mapping[0].summary[2].fromname probability +mapping[0].summary[2].toname si1 +mapping[0].summary[3].fromname justsummary +mapping[0].summary[3].toname ss3 +mapping[0].summary[4].fromname size +mapping[0].summary[4].toname si2 +mapping[0].attribute[5] +mapping[0].attribute[0].fromname special +mapping[0].attribute[0].toname as1 +mapping[0].attribute[1].fromname probability +mapping[0].attribute[1].toname af1 +mapping[0].attribute[2].fromname notindexed +mapping[0].attribute[2].toname as2 +mapping[0].attribute[3].fromname weight +mapping[0].attribute[3].toname ai1 +mapping[0].attribute[4].fromname l1mapa9 +mapping[0].attribute[4].toname a9 +mapping[1].name logical2 +mapping[1].field[6] +mapping[1].field[0].fromname special +mapping[1].field[0].toname fs1 +mapping[1].field[1].fromname title +mapping[1].field[1].toname d1 +mapping[1].field[2].fromname description +mapping[1].field[2].toname d2 +mapping[1].field[3].fromname size +mapping[1].field[3].toname fi1 +mapping[1].field[4].fromname onlyin2 +mapping[1].field[4].toname xy9 +mapping[1].field[5].fromname default +mapping[1].field[5].toname logical2default +mapping[1].context[0] +mapping[1].summary[5] +mapping[1].summary[0].fromname title +mapping[1].summary[0].toname ss1 +mapping[1].summary[1].fromname description +mapping[1].summary[1].toname ss2 +mapping[1].summary[2].fromname probability +mapping[1].summary[2].toname si1 +mapping[1].summary[3].fromname justsummary +mapping[1].summary[3].toname ss3 +mapping[1].summary[4].fromname size +mapping[1].summary[4].toname si2 +mapping[1].attribute[5] +mapping[1].attribute[0].fromname special +mapping[1].attribute[0].toname as1 +mapping[1].attribute[1].fromname probability +mapping[1].attribute[1].toname af1 +mapping[1].attribute[2].fromname notindexed +mapping[1].attribute[2].toname as2 +mapping[1].attribute[3].fromname weight +mapping[1].attribute[3].toname ai1 +mapping[1].attribute[4].fromname l2mapa9 +mapping[1].attribute[4].toname a9 +mapping[2].name logical3 +mapping[2].field[0] +mapping[2].context[0] +mapping[2].summary[2] +mapping[2].summary[0].fromname title +mapping[2].summary[0].toname s_1 +mapping[2].summary[1].fromname description +mapping[2].summary[1].toname s_2 +mapping[2].attribute[3] +mapping[2].attribute[0].fromname special +mapping[2].attribute[0].toname a_1 +mapping[2].attribute[1].fromname probability +mapping[2].attribute[1].toname a_2 +mapping[2].attribute[2].fromname l3mapa9 +mapping[2].attribute[2].toname a9 diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/testquoting.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testquoting.cfg new file mode 100644 index 00000000000..28a9755dddd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/testquoting.cfg @@ -0,0 +1,10 @@ +character[3] +# & +character[0].ordinal 38 +character[0].quoting & +# < +character[1].ordinal 60 +character[1].quoting < +# ASCII Unit Separator (0x1F) +character[2].ordinal 31 +character[2].quoting US diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/validate_sorting.cfg b/container-search/src/test/java/com/yahoo/prelude/searcher/test/validate_sorting.cfg new file mode 100644 index 00000000000..ba61858ce5e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/validate_sorting.cfg @@ -0,0 +1,17 @@ +attribute[4] +attribute[0].name title +attribute[0].datatype STRING +attribute[0].collectiontype SINGLE +attribute[0].sortascending true +attribute[0].sortfunction UCA +attribute[0].sortstrength TERTIARY +attribute[0].sortlocale en_US +attribute[1].name a +attribute[1].datatype STRING +attribute[1].collectiontype SINGLE +attribute[2].name b +attribute[2].datatype STRING +attribute[2].collectiontype SINGLE +attribute[3].name c +attribute[3].datatype STRING +attribute[3].collectiontype SINGLE diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/compatibility/test/.gitignore b/container-search/src/test/java/com/yahoo/prelude/semantics/compatibility/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java new file mode 100644 index 00000000000..9fc0863f5a7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/config/test/RuleConfigDeriverTestCase.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.config.test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.prelude.semantics.config.RuleConfigDeriver; +import com.yahoo.prelude.semantics.parser.ParseException; + +/** + * Tests the rule config deriver by reusing the files in ../test/inheritingrules + * + * @author bratseth + */ +public class RuleConfigDeriverTestCase extends junit.framework.TestCase { + + private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + + public RuleConfigDeriverTestCase(String name) { + super(name); + } + + public void testRuleConfig() throws IOException, ParseException { + new File("temp/ruleconfigderiver/").mkdirs(); + new RuleConfigDeriver().derive(root + "inheritingrules/","temp/ruleconfigderiver"); + assertEqualFiles(root + "semantic-rules.cfg","temp/ruleconfigderiver/semantic-rules.cfg"); + } + + public void testRuleConfigFromReader() throws IOException, ParseException { + FileReader reader = new FileReader(new File(root) + "/numbers.sr"); + NamedReader namedReader = new NamedReader("numbers", reader); + List readers = new ArrayList<>(); + readers.add(namedReader); + RuleConfigDeriver deriver = new RuleConfigDeriver(); + deriver.derive(readers); + } + + protected void assertEqualFiles(String correctFileName,String checkFileName) + throws java.io.IOException { + BufferedReader correct=null; + BufferedReader check=null; + try { + correct=IOUtils.createReader(correctFileName); + check = IOUtils.createReader(checkFileName); + String correctLine; + int lineNumber=1; + while ( null != (correctLine=correct.readLine())) { + String checkLine=check.readLine(); + assertNotNull("Too few lines, in " + checkFileName + + ", first missing is\n" + lineNumber + + ": " + correctLine,checkLine); + assertTrue("\nIn " + checkFileName + ":\n" + + "Expected line " + lineNumber + ":\n" + + correctLine.replaceAll("\\\\n","\n") + + "\nGot line " + lineNumber + ":\n" + + checkLine.replaceAll("\\\\n","\n") + "\n", + correctLine.trim().equals(checkLine.trim())); + lineNumber++; + } + assertNull("Excess line(s) in " + checkFileName + " starting at " + + lineNumber, + check.readLine()); + + } + finally { + IOUtils.closeReader(correct); + IOUtils.closeReader(check); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java new file mode 100644 index 00000000000..eba1dd87c95 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.parser.test; + +import java.util.Iterator; + +import com.yahoo.javacc.UnicodeUtilities; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleImporter; +import com.yahoo.prelude.semantics.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests parsing of semantic rules bases + * + * @author bratseth + */ +public class SemanticsParserTestCase { + + private final static String ROOT = "src/test/java/com/yahoo/prelude/semantics/parser/test/"; + + @Test + public void testRuleReading() throws java.io.IOException, ParseException { + RuleBase rules=new RuleImporter().importFile(ROOT + "rules.sr"); + Iterator i=rules.ruleIterator(); + assertEquals("[listing] [preposition] [place] -> listing:[listing] place:[place]!150", + i.next().toString()); + assertEquals("[listing] [place] +> place:[place]", + i.next().toString()); + assertEquals("[brand] -> brand:[brand]", + i.next().toString()); + assertEquals("[category] -> category:[category]", + i.next().toString()); + assertEquals("digital camera -> digicamera", + i.next().toString()); + assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one",i.next().toString()); + assertFalse(i.hasNext()); + + i=rules.conditionIterator(); + assertEquals("[listing] :- restaurant, shop, cafe, hotel", + i.next().toString()); + assertEquals("[preposition] :- in, at, near", + i.next().toString()); + assertEquals("[place] :- geary", + i.next().toString()); + assertEquals("[brand] :- sony, dell", + i.next().toString()); + assertEquals("[category] :- digital camera, camera, phone", + i.next().toString()); + assertFalse(i.hasNext()); + + assertTrue(rules.isDefault()); + assertEquals(ROOT + "semantics.fsa",rules.getAutomataFile()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/rules.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/rules.sr new file mode 100644 index 00000000000..56433f732a5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/rules.sr @@ -0,0 +1,32 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@default +@automata(src/test/java/com/yahoo/prelude/semantics/parser/test/semantics.fsa) + +# Local use case + +[listing] [preposition] [place] -> listing:[listing] place:[place]!150; + +[listing] :- restaurant, shop, cafe, hotel; + +[preposition] :- in, at, near; + +[place] :- geary; + +# Just to see additive rules working +[listing] [place] +> place:[place]; + +# Shopping use case + +[brand] -> brand:[brand]; +[category] -> category:[category]; + +digital camera -> digicamera; + +[brand] :- sony, dell; +[category] :- digital camera, camera, phone; + +# Answers use case + +# why is [noun] ... [adjective] +> ?about:[noun] + +parameter.ranking='cat', parameter.ranking='cat0' -> one; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/semantics.fsa b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/semantics.fsa new file mode 100644 index 00000000000..0b45cb9784a Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/semantics.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/AlibabaTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AlibabaTestCase.java new file mode 100644 index 00000000000..cea65790644 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AlibabaTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Test a case reported by Alibaba + * + * @author Jon Bratseth + */ +public class AlibabaTestCase extends RuleBaseAbstractTestCase { + + public AlibabaTestCase(String name) { + super(name,"alibaba.sr"); + } + + public void testNumberReplacement() { + assertSemantics("AND nokia 3100","3100"); + } + + public void testRuleFollowingNumber() { + assertSemantics("lenovo","legend"); + } + + public void testCombinedNumberAndRegular1() { + assertSemantics("AND lenovo nokia 3100","legend 3100"); + } + + public void testCombinedNumberAndRegular2() { + assertSemantics("AND nokia 3100 lenovo","3100 legend"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/AnchorTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AnchorTestCase.java new file mode 100644 index 00000000000..4f477dca5c9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AnchorTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests anchoring + * + * @author Jon Bratseth + */ +public class AnchorTestCase extends RuleBaseAbstractTestCase { + + public AnchorTestCase(String name) { + super(name,"anchor.sr"); + } + + public void testSingleWordAnchoredBothSides() { + assertSemantics("anchor","word"); + assertSemantics("anchor","anotherword"); + assertSemantics("notthisword","notthisword"); + assertSemantics("AND word anotherword","word anotherword"); + } + + public void testMultiwordAnchored() { + assertSemantics("anchor","this is complete"); + assertSemantics("AND this is complete toomuch","this is complete toomuch"); + assertSemantics("anchor","a phrase"); + assertSemantics("anchor","another phrase"); + } + + public void testFirstAnchored() { + assertSemantics("anchor","first"); + assertSemantics("AND anchor andmore","first andmore"); + assertSemantics("AND before first","before first"); + assertSemantics("AND before first andmore","before first andmore"); + } + + public void testLastAnchored() { + assertSemantics("anchor","last"); + assertSemantics("AND andmore anchor","andmore last"); + assertSemantics("AND last after","last after"); + assertSemantics("AND andmore last after","andmore last after"); + } + + public void testFirstAndLastAnchored() { + assertSemantics("AND anchor anchor","first last"); + assertSemantics("AND last first","last first"); + assertSemantics("AND anchor between anchor","first between last"); + assertSemantics("AND anchor last after","first last after"); + assertSemantics("AND before first anchor","before first last"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataNotTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataNotTestCase.java new file mode 100644 index 00000000000..92815b74ca1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataNotTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests that ![a] is interpreted as "default:![a]", not as "!default:[a]", + * that is, in negative conditions we still only want to match the default index by default. + * + * @author Jon Bratseth + */ +public class AutomataNotTestCase extends RuleBaseAbstractTestCase { + + public AutomataNotTestCase(String name) { + super(name,"automatanot.sr","semantics.fsa"); + } + + public void testAutomataNot() { + if (System.currentTimeMillis() > 0) return; // TODO: MAKE THIS WORK! + assertSemantics("carpenter","carpenter"); + assertSemantics("RANK brukbar busname:brukbar","brukbar"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataTestCase.java new file mode 100644 index 00000000000..7a34674f554 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/AutomataTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; + +/** + * Tests rule bases using automatas for matching + * + * @author Jon S Bratseth + */ +public class AutomataTestCase extends RuleBaseAbstractTestCase { + + private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + + public AutomataTestCase(String name) { + super(name,"automatarules.sr","semantics.fsa"); + } + + public void testAutomataRuleBase() throws Exception { + RuleBase ruleBase=searcher.getDefaultRuleBase(); + assertEquals(RuleBase.class,ruleBase.getClass()); + assertTrue(ruleBase.getSource().endsWith(root + "automatarules.sr")); + assertEquals(root + "semantics.fsa",ruleBase.getAutomataFile()); + + Query query=new Query("?query=sony+digital+camera"); + ruleBase.analyze(query,0); + assertEquals("RANK (AND sony digital camera) dsp1:sony dsp5:digicamera", query.getModel().getQueryTree().getRoot().toString()); + + query=new Query("?query=sony+digital+camera&rules.reload"); + ruleBase=searcher.getDefaultRuleBase(); + assertTrue(ruleBase.getSource().endsWith(root + "automatarules.sr")); + assertEquals(root + "semantics.fsa",ruleBase.getAutomataFile()); + ruleBase.analyze(query,0); + assertEquals("RANK (AND sony digital camera) dsp1:sony dsp5:digicamera", query.getModel().getQueryTree().getRoot().toString()); + } + + public void testAutomataSingleQuery() throws Exception { + assertSemantics("RANK sony dsp1:sony","sony"); + } + + public void testAutomataFilterIsIgnored() throws Exception { + assertSemantics("RANK sony |something dsp1:sony","sony&filter=something"); + assertSemantics("RANK something |sony","something&filter=sony"); + } + + public void testAutomataPluralMatches() throws Exception { + assertSemantics("RANK sonys dsp1:sony","sonys"); + + assertSemantics("RANK (AND car cleaner) dsp1:\"car cleaners\" dsp5:\"car cleaners\"","car cleaner"); + + assertSemantics("RANK (AND sony digitals cameras) dsp1:sony dsp5:digicamera","sony digitals cameras"); + } + + public void testMatchingMultipleAutomataConditionsSingleWord() { + assertSemantics("RANK carpenter dsp1:carpenter dsp5:carpenter","carpenter"); + } + + public void testMatchingMultipleAutomataConditionsPhrase() { + assertSemantics("RANK (AND car cleaners) dsp1:\"car cleaners\" dsp5:\"car cleaners\"","car cleaners"); + } + + public void tstReplaceOnNoMatch() { // TODO: Make this work again + assertSemantics("nomatch:sonny","sonny&donomatch"); + assertSemantics("RANK sony dsp1:sony","sony&donomatch"); + assertSemantics("RANK sonys dsp1:sony","sonys&donomatch"); + assertSemantics("AND nomatch:sonny nomatch:boy","sonny boy&donomatch"); + assertSemantics("RANK (AND sony nomatch:boy) dsp1:sony","sony boy&donomatch"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java new file mode 100644 index 00000000000..eac1c0cf002 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleImporter; +import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.prelude.semantics.parser.ParseException; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class BacktrackingTestCase extends junit.framework.TestCase { + + private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + + public BacktrackingTestCase(String name) throws ParseException, IOException { + super(name); + RuleBase rules=new RuleImporter().importFile(root + "backtrackingrules.sr"); + searcher=new SemanticSearcher(rules); + } + + private SemanticSearcher searcher; + + protected void assertSemantics(String result,String input) { + assertSemantics(result,input,0); + } + + protected void assertSemantics(String result,String input,int tracelevel) { + Query query=new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel); + doSearch(searcher, query, 0,10); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + } + + // Literal terms --------------- + + public void testMultilevelBacktrackingLiteralTerms() { + assertSemantics("replaced","word1 word2 word5 word8"); + } + + public void testMultilevelBacktrackingWontReorderOthertermsLiteralTerms() { + assertSemantics("AND other1 other2 other3 replaced","other1 other2 other3 word1 word2 word5 word8"); + } + + public void testMultilevelBacktrackingWithMulticompoundMatchLiteralTerms() { + assertSemantics("AND other1 other2 other3 replaced","other1 other2 other3 word1 word2 word5-word8"); + } + + public void testMultilevelBacktrackingPreservePartialMatchBeforeLiteralTerms() { + assertSemantics("AND word1 word2 word5 replaced","word1 word2 word5 word1 word2 word5 word8"); + } + + public void testMultilevelBacktrackingPreservePartialMatchAfterLiteralTerms() { + assertSemantics("AND replaced word1 word2 word5","word1 word2 word5 word8 word1 word2 word5 "); + } + + // reference terms --------------- + + public void testMultilevelBacktrackingReferenceTerms() { + assertSemantics("AND ref:ref1 ref:ref2 ref:ref5 ref:ref8","ref1 ref2 ref5 ref8"); + } + + public void testMultilevelBacktrackingPreservePartialMatchBeforeReferenceTerms() { + assertSemantics("AND ref1 ref2 ref5 ref:ref1 ref:ref2 ref:ref5 ref:ref8", + "ref1 ref2 ref5 ref1 ref2 ref5 ref8"); + } + + public void testMultilevelBacktrackingPreservePartialMatchAfterReferenceTerms() { + assertSemantics("AND ref:ref1 ref:ref2 ref:ref5 ref:ref8 ref1 ref2 ref5", + "ref1 ref2 ref5 ref8 ref1 ref2 ref5"); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BlendingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BlendingTestCase.java new file mode 100644 index 00000000000..f30bfffc18d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BlendingTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; + +/** + * Tests blending rules + * + * @author Jon Bratseth + */ +public class BlendingTestCase extends RuleBaseAbstractTestCase { + + public BlendingTestCase(String name) { + super(name,"blending.sr"); + } + + /** Tests parameter literal matching */ + public void testLiteralEquals() { + assertParameterSemantics("AND a sun came cd","a sun came cd","search","[music]"); + assertParameterSemantics("AND driving audi","driving audi","search","[cars]"); + //assertParameterSemantics("AND audi music quality","audi music quality","search","carstereos",1); + } + + private void assertParameterSemantics(String producedQuery,String inputQuery, + String producedParameterName,String producedParameterValue) { + assertParameterSemantics(producedQuery,inputQuery,producedParameterName,producedParameterValue,0); + } + + private void assertParameterSemantics(String producedQuery,String inputQuery, + String producedParameterName,String producedParameterValue,int tracing) { + Query query=assertSemantics(producedQuery,inputQuery,tracing); + assertEquals(producedParameterValue, query.properties().getString(producedParameterName)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/CJKTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/CJKTestCase.java new file mode 100644 index 00000000000..00c2ebf8d58 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/CJKTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests that using rule bases containing cjk characters work + * + * @author Jon Bratseth + */ +public class CJKTestCase extends RuleBaseAbstractTestCase { + + public CJKTestCase(String name) { + super(name,"cjk.sr"); + } + + public void testIt() { + assertSemantics("\u7d22a","a\u7d22"); + assertSemantics("\u7d22a","\u7d22a"); + assertSemantics("brand:\u7d22\u5c3c","\u7d22\u5c3c"); + assertSemantics("brand:\u60e0\u666e","\u60e0\u666e"); + assertSemantics("brand:\u4f73\u80fd","\u4f73\u80fd"); + assertSemantics("AND brand:\u4f73\u80fd \u7d22a","\u4f73\u80fd a\u7d22"); + assertSemantics("\u4f73\u80fd\u7d22\u5c3c","\u4f73\u80fd\u7d22\u5c3c"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonTestCase.java new file mode 100644 index 00000000000..80f41ad61a4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * @author Jon Bratseth + */ +public class ComparisonTestCase extends RuleBaseAbstractTestCase { + + public ComparisonTestCase(String name) { + super(name,"comparison.sr"); + } + + /** + * Tests that we can wriote rules which depends on the same term (java) being matched by two + * different conditions (coffee, island) + */ + public void testNamedConditionReturnComparison() { + // Not sufficient that both conditions are matched + assertSemantics("AND borneo arabica island:borneo coffee:arabica","borneo arabica"); + + // They must match the same word + assertSemantics("AND java noise island:java coffee:java control:ambigous off","java noise"); + + // Works also when there are other, not-equal matches + assertSemantics("AND borneo arabica java island:borneo island:java coffee:arabica coffee:java control:ambigous off", + "borneo arabica java"); + } + + public void testContainsAsSubstring() { + assertSemantics("AND java island:java coffee:java control:ambigous off","java"); + assertSemantics("AND kanava island:kanava off","kanava"); + assertSemantics("AND borneo island:borneo","borneo"); + } + + public void testAlphanumericComparison() { + assertSemantics("a","a"); + assertSemantics("AND z highletter","z"); + assertSemantics("AND p highletter","p"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonsTestCase.java new file mode 100644 index 00000000000..8b85f8bc587 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ComparisonsTestCase.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * @author Jon Bratseth + */ +public class ComparisonsTestCase extends RuleBaseAbstractTestCase { + + public ComparisonsTestCase(String name) { + super(name,"comparisons.sr"); + } + + public void testLiteralEquals() { + assertSemantics("a","a"); + assertSemantics("RANK a foo:a","a&ranking=category"); + assertSemantics("a","a&ranking=somethingelse"); + assertSemantics("a","a&otherparam=category"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java new file mode 100644 index 00000000000..ece03da5262 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.engine.Evaluation; +import com.yahoo.prelude.semantics.rule.ChoiceCondition; +import com.yahoo.prelude.semantics.rule.ConditionReference; +import com.yahoo.prelude.semantics.rule.NamedCondition; +import com.yahoo.prelude.semantics.rule.ProductionList; +import com.yahoo.prelude.semantics.rule.ProductionRule; +import com.yahoo.prelude.semantics.rule.ReplacingProductionRule; +import com.yahoo.prelude.semantics.rule.SequenceCondition; +import com.yahoo.prelude.semantics.rule.TermCondition; + +/** + * @author Jon S Bratseth + */ +public class ConditionTestCase extends junit.framework.TestCase { + + public ConditionTestCase(String name) { + super(name); + } + + public void testTermCondition() { + TermCondition term=new TermCondition("foo"); + Query query=new Query("?query=foo"); + assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation())); + } + + public void testSequenceCondition() { + TermCondition term1=new TermCondition("foo"); + TermCondition term2=new TermCondition("bar"); + SequenceCondition sequence=new SequenceCondition(); + sequence.addCondition(term1); + sequence.addCondition(term2); + Query query=new Query("?query=foo+bar"); + assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query).freshRuleEvaluation())); + Query query2=new Query("?query=foo"); + assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2).freshRuleEvaluation())); + Query query3=new Query("?query=bar"); + assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3).freshRuleEvaluation())); + } + + public void testChoiceCondition() { + TermCondition term1=new TermCondition("foo"); + TermCondition term2=new TermCondition("bar"); + ChoiceCondition choice=new ChoiceCondition(); + choice.addCondition(term1); + choice.addCondition(term2); + Query query1=new Query("?query=foo+bar"); + assertTrue(query1 + " matches " + choice,choice.matches(new Evaluation(query1).freshRuleEvaluation())); + Query query2=new Query("?query=foo"); + assertTrue(query2 + " matches " + choice,choice.matches(new Evaluation(query2).freshRuleEvaluation())); + Query query3=new Query("?query=bar"); + assertTrue(query3 + " matches " + choice,choice.matches(new Evaluation(query3).freshRuleEvaluation())); + } + + public void testNamedConditionReference() { + TermCondition term=new TermCondition("foo"); + NamedCondition named=new NamedCondition("cond",term); + ConditionReference reference=new ConditionReference("cond"); + + // To initialize the condition reference... + ProductionRule rule=new ReplacingProductionRule(); + rule.setCondition(reference); + rule.setProduction(new ProductionList()); + RuleBase ruleBase=new RuleBase(); + ruleBase.setName("test"); + ruleBase.addCondition(named); + ruleBase.addRule(rule); + ruleBase.initialize(); + + Query query=new Query("?query=foo"); + assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query).freshRuleEvaluation())); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java new file mode 100644 index 00000000000..4619a07ad74 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.semantics.SemanticRulesConfig; +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +import java.util.ArrayList; +import java.util.List; + +/** + * Tests creating a set of rule bases (the same set as in inheritingrules) from config + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class ConfigurationTestCase extends junit.framework.TestCase { + + private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + + private SemanticSearcher searcher; + private SemanticRulesConfig semanticRulesConfig; + + public ConfigurationTestCase(String name) { + super(name); + semanticRulesConfig = new ConfigGetter<>(SemanticRulesConfig.class).getConfig("file:" + root + "semantic-rules.cfg"); + searcher=new SemanticSearcher(semanticRulesConfig); + } + + protected void assertSemantics(String result, String input, String baseName) { + Query query = new Query(QueryTestCase.httpEncode("?query=" + input + "&tracelevel=0&tracelevel.rules=0&rules.rulebase=" + baseName)); + doSearch(searcher, query, 0, 10); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + } + + protected void assertSemanticsRulesOff(String result, String input) { + Query query = new Query(QueryTestCase.httpEncode("?query=" + input + "&tracelevel=0&tracelevel.rules=0&rules.off")); + doSearch(searcher, query, 0, 10); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + } + + public void testReadingConfigurationRuleBase() { + RuleBase parent=searcher.getRuleBase("parent"); + assertNotNull(parent); + assertEquals("parent",parent.getName()); + assertEquals("semantic-rules.cfg",parent.getSource()); + } + + public void testParent() throws Exception { + assertSemantics("vehiclebrand:audi","audi cars","parent"); + assertSemantics("vehiclebrand:alfa","alfa bus","parent"); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","parent.sr"); + assertSemantics("AND vw car", "vw cars","parent"); + assertSemantics("AND skoda car", "skoda cars","parent.sr"); + } + + public void testChild1() throws Exception { + assertSemantics("vehiclebrand:skoda","audi cars","child1.sr"); + assertSemantics("vehiclebrand:alfa", "alfa bus","child1"); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child1"); + assertSemantics("vehiclebrand:skoda","vw cars","child1"); + assertSemantics("AND skoda car", "skoda cars","child1"); + } + + public void testChild2() throws Exception { + assertSemantics("vehiclebrand:audi","audi cars","child2"); + assertSemantics("vehiclebrand:alfa","alfa bus","child2.sr"); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child2.sr"); + assertSemantics("AND vw car","vw cars","child2"); + assertSemantics("vehiclebrand:skoda","skoda cars","child2"); + } + + public void testGrandchild() throws Exception { + assertSemantics("vehiclebrand:skoda","audi cars","grandchild.sr"); + assertSemantics("vehiclebrand:alfa","alfa bus","grandchild"); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","grandchild"); + assertSemantics("vehiclebrand:skoda","vw cars","grandchild"); + assertSemantics("vehiclebrand:skoda","skoda cars","grandchild"); + } + + public void testSearcher() { + assertSemantics("vehiclebrand:skoda", "vw cars", "grandchild"); + assertSemantics("vehiclebrand:skoda", "vw cars", "grandchild.sd"); + try { + assertSemantics("AND vw cars", "vw cars", "doesntexist"); + fail("No exception on missing rule base"); + } + catch (RuleBaseException e) { + // Success + } + assertSemantics("AND vw cars", "vw cars", "grandchild.sd&rules.off"); + assertSemanticsRulesOff("AND vw cars", "vw cars"); + + assertSemantics("AND vw car", "vw cars", "child2"); + assertSemantics("vehiclebrand:skoda","skoda cars","child2"); + + assertSemantics("vehiclebrand:skoda","audi cars", "child1"); + assertSemantics("vehiclebrand:skoda","vw cars", "child1"); + assertSemantics("AND skoda car", "skoda cars","child1"); + + assertSemantics("AND vw car", "vw cars", "parent"); + assertSemantics("AND skoda car", "skoda cars","parent"); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java new file mode 100644 index 00000000000..e1d5b93a32f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.RuleImporter; +import com.yahoo.prelude.semantics.parser.ParseException; + +/** + * @author Jon S Bratseth + */ +public class DuplicateRuleTestCase extends junit.framework.TestCase { + + private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + + public DuplicateRuleTestCase(String name) { + super(name); + } + + public void testDuplicateRuleBaseLoading() throws java.io.IOException, ParseException { + if (System.currentTimeMillis() > 0) return; // TODO: Include this test... + + try { + new RuleImporter().importFile(root + "rules.sr"); + fail("Did not detect duplicate condition names"); + } + catch (RuleBaseException e) { + assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'",e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/Ellipsis2TestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/Ellipsis2TestCase.java new file mode 100644 index 00000000000..7aa60630db3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/Ellipsis2TestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * tersts the ellipsis rule base + * + * @author Jon Bratseth + */ +public class Ellipsis2TestCase extends RuleBaseAbstractTestCase { + + public Ellipsis2TestCase(String name) { + super(name,"ellipsis2.sr"); + } + + public void testUnreferencedEllipsis() { + assertSemantics("AND a b c someindex:\"a b c\"","a b c"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/EllipsisTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EllipsisTestCase.java new file mode 100644 index 00000000000..329006414c3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EllipsisTestCase.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * tersts the ellipsis rule base + * + * @author Jon Bratseth + */ +public class EllipsisTestCase extends RuleBaseAbstractTestCase { + + public EllipsisTestCase(String name) { + super(name,"ellipsis.sr"); + } + + public void testUnreferencedEllipsis() { + assertSemantics("AND why is stench unpleasant about:stench","why is stench unpleasant"); + assertSemantics("AND why is the sky blue about:\"the sky\"","why is the sky blue"); + assertSemantics("AND why is aardwark almost always most relevant in dictionaries about:aardwark", + "why is aardwark almost always most relevant in dictionaries"); + } + + public void testReferencedEllipsis() { + assertSemantics("album:parade","parade album"); + assertSemantics("album:\"a sun came\"","a sun came album"); + assertSemantics("album:parade","parade cd"); + assertSemantics("album:\"a sun came\"","a sun came cd"); + } + + public void testEllipsisInNamedCondition() { + assertSemantics("AND name:\"a sun came\" product:video","buy a sun came"); + assertSemantics("AND name:stalker product:video","buy stalker video"); + assertSemantics("AND name:\"the usual suspects\" product:video","buy the usual suspects video"); + } + + public void testMultipleEllipsis() { + assertSemantics("AND from:paris to:texas","from paris to texas"); + assertSemantics("AND from:\"sao paulo\" to:\"real madrid\"","from sao paulo to real madrid"); + assertSemantics("AND from:\"from from\" to:oslo","from from from to oslo"); + assertSemantics("AND from:\"from from to\" to:koko","from from from to to koko"); // Matching is greedy left-right + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTestCase.java new file mode 100644 index 00000000000..6cc56478fc9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTestCase.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * @author Jon Bratseth + */ +public class ExactMatchTestCase extends RuleBaseAbstractTestCase { + + public ExactMatchTestCase(String name) { + super(name,"exactmatch.sr"); + } + + public void testCompleteMatch() { + assertSemantics("AND primetime in no time","primetime notime"); + } + + /* + public void testCompleteMatchWithNegative() { + assertSemantics("AND primetime in no time ...fix",new Query(HttpRequest.fromString("?query=primetime+ANDNOT+regionexcl:us&type=adv"))); + } + public void testCompleteMatchWithFilterAndNegative() { + assertSemantics("AND primetime in no time ...fix",new Query(HttpRequest.fromString("?query=primetime+ANDNOT+regionexcl:us&type=adv&filter=%2Blang:en"))); + } + */ + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java new file mode 100644 index 00000000000..27ec6dc4133 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; +import com.yahoo.search.test.QueryTestCase; + +/** + * @author bratseth + */ +public class ExactMatchTrickTestCase extends RuleBaseAbstractTestCase { + + public ExactMatchTrickTestCase(String name) { + super(name,"exactmatchtrick.sr"); + } + + public void testCompleteMatch() { + assertSemantics("AND default:primetime default:in default:no default:time", "primetime notime"); + } + + public void testCompleteMatchWithNegative() { // Notice ordering bug + assertSemantics("+(AND default:primetime default:in default:time default:no) -regionexcl:us", + new Query(QueryTestCase.httpEncode("?query=primetime ANDNOT regionexcl:us&type=adv"))); + } + + public void testCompleteMatchWithFilterAndNegative() { + assertSemantics("AND (+(AND default:primetime default:in default:time default:no) -regionexcl:us) |lang:en", + new Query(QueryTestCase.httpEncode("?query=primetime ANDNOT regionexcl:us&type=adv&filter=+lang:en"))); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java new file mode 100644 index 00000000000..c86db996f68 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class InheritanceTestCase extends junit.framework.TestCase { + + private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + + private RuleBase parent, child1, child2, grandchild; + private SemanticSearcher searcher; + + public InheritanceTestCase(String name) throws Exception { + super(name); + parent=RuleBase.createFromFile(root + "inheritingrules/parent.sr",null); + child1=RuleBase.createFromFile(root + "inheritingrules/child1.sr",null); + child2=RuleBase.createFromFile(root + "inheritingrules/child2.sr",null); + grandchild=RuleBase.createFromFile(root + "inheritingrules/grandchild.sr",null); + grandchild.setDefault(true); + + searcher=new SemanticSearcher(parent,child1,child2,grandchild); + } + + protected void assertSemantics(String result,String input,RuleBase base) { + assertSemantics(result,input,base,0); + } + + protected void assertSemantics(String result,String input,RuleBase base,int tracelevel) { + Query query=new Query("?query=" + QueryTestCase.httpEncode(input)); + base.analyze(query,tracelevel); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + } + + public void testInclusion() { + assertTrue(grandchild.includes("child1")); + assertTrue(grandchild.includes("child2")); + assertTrue(grandchild.includes("parent")); + assertTrue(grandchild.includes("grandfather")); + assertTrue(grandchild.includes("grandmother")); + assertFalse(grandchild.includes("nonexisting")); + + assertFalse(parent.includes("child1")); + assertFalse(parent.includes("child2")); + assertFalse(parent.includes("parent")); + assertTrue(parent.includes("grandfather")); + assertTrue(parent.includes("grandmother")); + } + + public void testInclusionOrderAndContentDump() { + StringTokenizer lines=new StringTokenizer(grandchild.toContentString(),"\n",false); + assertEquals("vw -> audi",lines.nextToken()); + assertEquals("cars -> car",lines.nextToken()); + assertEquals("[brand] [vehicle] -> vehiclebrand:[brand]",lines.nextToken()); + assertEquals("vehiclebrand:bmw +> expensivetv",lines.nextToken()); + assertEquals("vehiclebrand:audi -> vehiclebrand:skoda",lines.nextToken()); + assertEquals("vehiclebrand:vw -> vehiclebrand:audi",lines.nextToken()); + assertEquals("causesphrase -> \"a produced phrase\"",lines.nextToken()); + assertEquals("[vehicle] :- car, motorcycle, bus",lines.nextToken()); + assertEquals("[brand] :- alfa, audi, bmw, skoda",lines.nextToken()); + } + + public void testParent() throws Exception { + assertSemantics("vehiclebrand:audi","audi cars",parent); + assertSemantics("vehiclebrand:alfa","alfa bus",parent); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle",parent); + assertSemantics("AND vw car", "vw cars",parent); + assertSemantics("AND skoda car", "skoda cars",parent); + } + + public void testChild1() throws Exception { + assertSemantics("vehiclebrand:skoda","audi cars",child1); + assertSemantics("vehiclebrand:alfa", "alfa bus",child1); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle",child1); + assertSemantics("vehiclebrand:skoda","vw cars",child1); + assertSemantics("AND skoda car", "skoda cars",child1); + } + + public void testChild2() throws Exception { + assertSemantics("vehiclebrand:audi","audi cars",child2); + assertSemantics("vehiclebrand:alfa","alfa bus",child2); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle",child2); + assertSemantics("AND vw car","vw cars",child2); + assertSemantics("vehiclebrand:skoda","skoda cars",child2); + } + + public void testGrandchild() throws Exception { + assertSemantics("vehiclebrand:skoda","audi cars",grandchild); + assertSemantics("vehiclebrand:alfa","alfa bus",grandchild); + assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle",grandchild); + assertSemantics("vehiclebrand:skoda","vw cars",grandchild); + assertSemantics("vehiclebrand:skoda","skoda cars",grandchild); + } + + public void testRuleBaseNames() { + assertEquals("parent",parent.getName()); + assertEquals("child1",child1.getName()); + assertEquals("child2",child2.getName()); + assertEquals("grandchild",grandchild.getName()); + } + + public void testSearcher() { + assertSemantics("vehiclebrand:skoda","vw cars", ""); + assertSemantics("vehiclebrand:skoda","vw cars", "&rules.rulebase=grandchild"); + assertSemantics("vehiclebrand:skoda","vw cars", "&rules.rulebase=grandchild.sd"); + try { + assertSemantics("AND vw cars", "vw cars", "&rules.rulebase=doesntexist"); + fail("No exception on missing rule base"); + } + catch (RuleBaseException e) { + // Success + } + assertSemantics("AND vw cars", "vw cars", "&rules.rulebase=grandchild.sd&rules.off"); + assertSemantics("AND vw cars", "vw cars", "&rules.off"); + + assertSemantics("AND vw car", "vw cars", "&rules.rulebase=child2"); + assertSemantics("vehiclebrand:skoda","skoda cars","&rules.rulebase=child2"); + + assertSemantics("vehiclebrand:skoda","audi cars", "&rules.rulebase=child1"); + assertSemantics("vehiclebrand:skoda","vw cars", "&rules.rulebase=child1"); + assertSemantics("AND skoda car", "skoda cars","&rules.rulebase=child1"); + + assertSemantics("AND vw car", "vw cars", "&rules.rulebase=parent"); + assertSemantics("AND skoda car", "skoda cars","&rules.rulebase=parent"); + } + + protected void assertSemantics(String result,String input,String ruleSelection) { + Query query=new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=0" + ruleSelection); + doSearch(searcher, query, 0,10); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/LabelMatchingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/LabelMatchingTestCase.java new file mode 100644 index 00000000000..382870a97b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/LabelMatchingTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import java.io.IOException; + +import com.yahoo.prelude.semantics.parser.ParseException; + +/** + * Tests label-dependent matching + * + * @author Jon Bratseth + */ +public class LabelMatchingTestCase extends RuleBaseAbstractTestCase { + + public LabelMatchingTestCase(String name) { + super(name,"labelmatching.sr"); + } + + /** Tests that matching with no label matches the default label (index) only */ + public void testDefaultLabelMatching() throws IOException, ParseException { + assertSemantics("matched:term","term"); + assertSemantics("alabel:term","alabel:term"); + + assertSemantics("AND term2 hit","term2"); + assertSemantics("alabel:term2","alabel:term2"); + } + + public void testSpecificLabelMatchingInConditionReference() throws IOException, ParseException { + assertSemantics("+dcattitle:restaurants -dcat:hotel","dcattitle:restaurants"); + } + + public void testSpecificlabelMatchingInNestedCondition() throws IOException, ParseException { + assertSemantics("three","foo:one"); + assertSemantics("three","foo:two"); + assertSemantics("bar:one","bar:one"); + assertSemantics("bar:two","bar:two"); + assertSemantics("foo:three","foo:three"); + assertSemantics("one","one"); + assertSemantics("two","two"); + assertSemantics("AND three three","foo:one foo:two"); + assertSemantics("AND bar:one bar:two","bar:one bar:two"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchAllTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchAllTestCase.java new file mode 100644 index 00000000000..6cb2a3052d7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchAllTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * tersts the ellipsis rule base + * + * @author Jon Bratseth + */ +public class MatchAllTestCase extends RuleBaseAbstractTestCase { + + public MatchAllTestCase(String name) { + super(name,"matchall.sr"); + } + + public void testMatchAll() { + assertSemantics("RANK a normtitle:a","a"); + assertSemantics("RANK (AND a b) normtitle:\"a b\"","a b"); + assertSemantics("RANK (AND a a b a) normtitle:\"a a b a\"","a a b a"); + } + + public void testMatchAllFilterIsIgnored() { + assertSemantics("RANK a |b normtitle:a","a&filter=b"); + assertSemantics("RANK (AND a b) |b |c normtitle:\"a b\"","a b&filter=b c"); + assertSemantics("RANK (AND a a b a) |a |a |c |d |b normtitle:\"a a b a\"","a a b a&filter=a a c d b"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java new file mode 100644 index 00000000000..9907c710411 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Experiments with a way to match only if it doesn't remove all hard conditions in the query. + * The problem is that a straightforward use case of replacement leads to nonsensical queries as shown. + * + * @author Jon Bratseth + */ +public class MatchOnlyIfNotOnlyTermTestCase extends RuleBaseAbstractTestCase { + + public MatchOnlyIfNotOnlyTermTestCase(String name) { + super(name,"match-only-if-not-only-term.sr"); + } + + public void testMatch() { + assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000","justin timberlake snl"); + assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000","justin timberlake saturday night live"); + } + + public void testNoMatch() { + // TODO: This shows that we do match, i.e that currently the behavior is undesired + assertSemantics("showname:\"saturday night live\"!1000", "snl"); + assertSemantics("showname:\"saturday night live\"!1000", "saturday night live"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java new file mode 100644 index 00000000000..fd71c7f682c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests a case reported by tularam + * + * @author Jon Bratseth + */ +public class NoStemmingTestCase extends RuleBaseAbstractTestCase { + + public NoStemmingTestCase(String name) { + super(name,"nostemming.sr"); + } + + /** Should rewrite correctly */ + public void testCorrectRewriting1() { + assertSemantics("+(AND i:arts i:sciences) -i:b","i:as -i:b"); + } + + /** Should rewrite correctly too */ + public void testCorrectRewriting2() { + assertSemantics("+(AND i:arts i:sciences i:crafts) -i:b","i:asc -i:b"); + } + + /** Should not rewrite */ + public void testNoRewriting() { + assertSemantics("+i:a -i:s","i:a -i:s"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/NotTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NotTestCase.java new file mode 100644 index 00000000000..5d7db4e0260 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NotTestCase.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * @author Jon Bratseth + */ +public class NotTestCase extends RuleBaseAbstractTestCase { + + public NotTestCase(String name) { + super(name,"not.sr"); + } + + public void testLiteralEquals() { + assertSemantics("RANK a foo:a","a"); + assertSemantics("a","a&ranking=category"); + assertSemantics("RANK a foo:a","a&ranking=somethingelse"); + assertSemantics("RANK a foo:a","a&otherparam=category"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/NumbersTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NumbersTestCase.java new file mode 100644 index 00000000000..abf407d7b9c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NumbersTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests numbers as conditions and productions + * + * @author Jon S Bratseth + */ +public class NumbersTestCase extends RuleBaseAbstractTestCase { + + public NumbersTestCase(String name) { + super(name,"numbers.sr"); + } + + public void testNumbers() { + assertSemantics("elite","1337"); + assertSemantics("1","one"); + assertSemantics("AND bort ned","opp"); + assertSemantics("AND kanoo knagg","foo bar"); + assertSemantics("AND 3 three","two 2"); + assertSemantics("AND 1 elite","one 1337"); + } + + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/NumericTermsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NumericTermsTestCase.java new file mode 100644 index 00000000000..c2b4bdbab35 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NumericTermsTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests numeric terms + * + * @author Jon Bratseth + */ +public class NumericTermsTestCase extends RuleBaseAbstractTestCase { + + public NumericTermsTestCase(String name) { + super(name,"numericterms.sr"); + } + + public void testNumericProduction() { + assertSemantics("+restaurants -ycat2gc:96929265","restaurants"); + } + + public void testNumericConditionAndProduction() { + assertSemantics("48","49"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java new file mode 100644 index 00000000000..9a8fc7d6b41 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * @author Jon Bratseth + */ +public class OrPhraseTestCase extends RuleBaseAbstractTestCase { + + public OrPhraseTestCase(String name) { + super(name,"orphrase.sr"); + } + + public void testReplacing1() { + assertSemantics("OR (AND new york) title:\"software engineer\"","software engineer new york"); + assertSemantics("title:\"software engineer\"","software engineer"); // Skip or when there is nothing else + } + + public void testReplacing2() { + assertSemantics("OR lotr \"lord of the rings\"","lotr"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/Parameter2TestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/Parameter2TestCase.java new file mode 100644 index 00000000000..45c1cf5d4ec --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/Parameter2TestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; + +/** + * Tests parameter matching and production + * + * @author Jon Bratseth + */ +public class Parameter2TestCase extends RuleBaseAbstractTestCase { + + public Parameter2TestCase(String name) { + super(name,"parameter2.sr"); + } + + /** Tests parameter production */ + public void testParameterProduction() { + assertRankParameterSemantics("a","a&ranking=usrank","date",0); + } + + private void assertRankParameterSemantics(String producedQuery,String inputQuery, + String rankParameterValue,int tracelevel) { + Query query=new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel); + query.properties().set("tracelevel.rules", tracelevel); + assertSemantics(producedQuery, query); + assertEquals(rankParameterValue,query.getRanking().getProfile()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java new file mode 100644 index 00000000000..ce40ea6f6c5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; + +/** + * Tests parameter matching and production + * + * @author Jon Bratseth + */ +public class ParameterTestCase extends RuleBaseAbstractTestCase { + + public ParameterTestCase(String name) { + super(name,"parameter.sr"); + } + + /** Tests parameter literal matching */ + public void testLiteralEquals() { + assertSemantics("a","a"); + assertSemantics("RANK a foo:a","a&ranking=category"); + assertSemantics("a","a&ranking=somethingelse"); + assertSemantics("a","a&otherparam=category"); + } + + /** Tests parameter matching of larger */ + public void testLarger() { + assertSemantics("a","a"); + assertSemantics("AND a largepage","a&hits=11"); + assertSemantics("AND a largepage","a&hits=12"); + } + + /** Tests parameter containment matching */ + public void testContainsAsList() { + assertSemantics("a","a"); + assertSemantics("AND a intent:music","a&search=music"); + assertSemantics("AND a intent:music","a&search=music,books"); + assertSemantics("AND a intent:music","a&search=kanoos,music,books"); + } + + /** Tests parameter production */ + public void testParameterProduction() { + assertParameterSemantics("AND a b c","a b c","search","[letters, alphabet]"); + assertParameterSemantics("AND a c d","a c d","search","[letters, someletters]"); + assertParameterSemantics("+(AND a d e) -letter:c","a d e","search","[someletters]"); + assertParameterSemantics("AND a d f","a d f","rank-profile","foo"); + assertParameterSemantics("AND a f g","a f g","grouping.nolearning","true"); + } + + public void testMultipleAlternativeParameterValuesInCondition() { + assertInputRankParameterSemantics("one","foo","cat"); + assertInputRankParameterSemantics("one","foo","cat0"); + assertInputRankParameterSemantics("one","bar","cat"); + assertInputRankParameterSemantics("one","bar","cat0"); + assertInputRankParameterSemantics("AND one one","foo+bar","cat0"); + assertInputRankParameterSemantics("AND fuki sushi","fuki+sushi","cat0"); + } + + private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, + String rankParameterValue) { + assertInputRankParameterSemantics(producedQuery,inputQuery,rankParameterValue,0); + } + + private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, + String rankParameterValue,int tracelevel) { + Query query=new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel); + query.getRanking().setProfile(rankParameterValue); + query.properties().set("tracelevel.rules", tracelevel); + assertSemantics(producedQuery, query); + } + + private void assertParameterSemantics(String producedQuery,String inputQuery, + String producedParameterName,String producedParameterValue) { + Query query=assertSemantics(producedQuery,inputQuery); + assertEquals(producedParameterValue,query.properties().getString(producedParameterName)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java new file mode 100644 index 00000000000..2a8f7008ac1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/PhraseMatchTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests that the phrase produced by an automata match can subsequently be replaced by an AND of the + * same terms. + * + * @author Jon Bratseth + */ +public class PhraseMatchTestCase extends RuleBaseAbstractTestCase { + + public PhraseMatchTestCase(String name) { + super(name,"phrasematch.sr","semantics.fsa"); + } + + public void testLiteralEquals() { + if (1==1) return; // TODO: Work in progress + assertSemantics("AND retailer:digital retailer:camera","keyword:digital keyword:camera"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java new file mode 100644 index 00000000000..595a2b4c75c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.engine.Evaluation; +import com.yahoo.prelude.semantics.engine.RuleEvaluation; +import com.yahoo.prelude.semantics.rule.ConditionReference; +import com.yahoo.prelude.semantics.rule.NamedCondition; +import com.yahoo.prelude.semantics.rule.ProductionList; +import com.yahoo.prelude.semantics.rule.ProductionRule; +import com.yahoo.prelude.semantics.rule.ReferenceTermProduction; +import com.yahoo.prelude.semantics.rule.ReplacingProductionRule; +import com.yahoo.prelude.semantics.rule.TermCondition; +import com.yahoo.prelude.semantics.rule.TermProduction; + +/** + * @author Jon S Bratseth + */ +public class ProductionRuleTestCase extends junit.framework.TestCase { + + public ProductionRuleTestCase(String name) { + super(name); + } + + public void testProductionRule() { + TermCondition term=new TermCondition("sony"); + NamedCondition named=new NamedCondition("brand",term); + ConditionReference reference=new ConditionReference("brand"); + + TermProduction termProduction =new ReferenceTermProduction("brand","brand"); + ProductionList productionList =new ProductionList(); + productionList.addProduction(termProduction); + + ProductionRule rule=new ReplacingProductionRule(); + rule.setCondition(reference); + rule.setProduction(productionList); + + // To initialize the condition reference... + RuleBase ruleBase=new RuleBase(); + ruleBase.setName("test"); + ruleBase.addCondition(named); + ruleBase.addRule(rule); + ruleBase.initialize(); + + assertTrue("Brand is referenced",rule.matchReferences().contains("brand")); + + Query query=new Query("?query=sony"); + RuleEvaluation e=new Evaluation(query).freshRuleEvaluation(); + assertTrue(rule.matches(e)); + rule.produce(e); + assertEquals("brand:sony", query.getModel().getQueryTree().getRoot().toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java new file mode 100644 index 00000000000..102f4c95926 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Query; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +import java.util.ArrayList; +import java.util.List; + +/** + * Tests semantic searching + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public abstract class RuleBaseAbstractTestCase extends junit.framework.TestCase { + + protected final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + protected final SemanticSearcher searcher; + + protected RuleBaseAbstractTestCase(String name,String ruleBaseName) { + this(name,ruleBaseName,null); + } + + protected RuleBaseAbstractTestCase(String name,String ruleBaseName,String automataFileName) { + super(name); + searcher = createSearcher(ruleBaseName,automataFileName); + } + + public void setUp() { + } + + protected SemanticSearcher createSearcher(String ruleBaseName,String automataFileName) { + try { + if (automataFileName!=null) + automataFileName=root + automataFileName; + RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName,automataFileName); + return new SemanticSearcher(ruleBase); + } catch (Exception e) { + throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed",e); + } + } + + protected Query assertSemantics(String result, String input) { + return assertSemantics(result, input, 0); + } + + protected Query assertSemantics(String result, String input, int tracelevel) { + return assertSemantics(result, input, tracelevel, Query.Type.ALL); + } + + protected Query assertSemantics(String result, String input, int tracelevel, Query.Type queryType) { + Query query=new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel + + "&language=und&type=" + queryType.toString()); + return assertSemantics(result, query); + } + + protected Query assertSemantics(String result, Query query) { + createExecution(searcher).search(query); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + return query; + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java new file mode 100644 index 00000000000..6929d9e37c7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.language.Language; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.parser.AllParser; +import com.yahoo.prelude.query.parser.TestLinguistics; +import com.yahoo.search.Query; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; +import org.junit.Ignore; + +public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { + + public SegmentSubstitutionTestCase(String name) { + super(name,"substitution.sr"); + } + + public void testBasicSubstitution() { + Item a = parseQuery("firstsecond"); + Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); + q.getModel().getQueryTree().setRoot(a); + + assertSemantics("\"first third\"", q); + } + + public void testSubstitutionAndMoreTerms() { + Item a = parseQuery("bcfirstsecondfg"); + Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); + q.getModel().getQueryTree().setRoot(a); + + assertSemantics("\"bc first third fg\"", q); + } + + public void testSubstitutionAndNot() { + Item a = parseQuery("-firstsecond bc"); + Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); + q.getModel().getQueryTree().setRoot(a); + + assertSemantics("+bc -\"first third\"", q); + } + + public void testSubstitutionSomeNoise() { + Item a = parseQuery("9270bcsecond2389"); + Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); + q.getModel().getQueryTree().setRoot(a); + + assertSemantics("\"9 2 7 0 bc third 2 3 8 9\"", q); + } + + private static Item parseQuery(String query) { + AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE)); + return parser.parse(new Parsable().setQuery(query).setLanguage(Language.CHINESE_SIMPLIFIED)).getRoot(); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java new file mode 100644 index 00000000000..8d4ef630fe7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.search.Query; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.List; + +/** + * Tests semantic searching + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { + + public SemanticSearcherTestCase(String name) { + super(name,"rules.sr"); + } + + public void testSingleShopping() { + assertSemantics("brand:sony", + "sony"); + } + + public void testCombinedShopping() { + assertSemantics("AND brand:sony category:camera", + "sony camera"); + } + + public void testPhrasedShopping() { + assertSemantics("AND brand:sony category:\"digital camera\"", + "sony digital camera"); + } + + public void testSimpleLocal() { + assertSemantics("AND listing:restaurant place:geary", + "restaurant in geary"); + } + + public void testLocal() { + assertSemantics("AND listing:restaurant place:\"geary street san francisco\"", + "restaurant in geary street san francisco"); + } + + public void testLiteralReplacing() { + assertSemantics("AND lord of rings","lotr"); + } + + public void testAddingAnd() { + assertSemantics("AND bar foobar:bar", + "bar"); + } + + public void testAddingRank() { + assertSemantics("RANK word foobar:word", + "word"); + } + + public void testFilterIsIgnored() { + assertSemantics("RANK word |a |word |b foobar:word", + "word&filter=a word b"); + assertSemantics("RANK a |word |b", + "a&filter=word b"); + } + + public void testAddingNegative() { + assertSemantics("+java -coffee", + "java"); + } + + public void testAddingNegativePluralToSingular() { + assertSemantics("+javas -coffee", + "javas"); + } + + public void testCombined() { + assertSemantics("AND bar listing:restaurant place:\"geary street san francisco\" foobar:bar", + "bar restaurant in geary street san francisco"); + } + + public void testStopWord() { + assertSemantics("strokes","the strokes"); + } + + public void testStopWords1() { + assertSemantics("strokes","be the strokes"); + } + + public void testStopWords2() { + assertSemantics("strokes","the strokes be"); + } + + public void testDontRemoveEverything() { + assertSemantics("the","the the the"); + } + + public void testMoreStopWordRemoval() { + assertSemantics("hamlet","hamlet to be or not to be"); + } + + public void testTypeChange() { + assertSemantics("RANK doors default:typechange","typechange doors"); + } + + public void testTypeChangeWithSingularToPluralButNonReplaceWillNotSingularify() { + assertSemantics("RANK door default:typechange","typechange door"); + } + + public void testExplicitContext() { + assertSemantics("AND from:paris to:texas","paris to texas"); + } + + public void testPluralReplaceBecomesSingular() { + assertSemantics("AND from:paris to:texas","pariss to texass"); + } + + public void testOrProduction() { + assertSemantics("OR something somethingelse","something"); + } + + //This test is order dependent. Fix it!! + public void testWeightedSetItem() { + Query q = new Query(); + WeightedSetItem weightedSet=new WeightedSetItem("fieldName"); + weightedSet.addToken("a",1); + weightedSet.addToken("b",2); + q.getModel().getQueryTree().setRoot(weightedSet); + assertSemantics("WEIGHTEDSET fieldName{[1]:\"a\",[2]:\"b\"}",q); + } + + public void testNullQuery() { + Query query=new Query(""); // Causes a query containing a NullItem + doSearch(searcher, query, 0, 10); + assertEquals(NullItem.class, query.getModel().getQueryTree().getRoot().getClass()); // Still a NullItem + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java new file mode 100644 index 00000000000..17f95d37dc7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests a case reported by tularam + * + * @author Jon Bratseth + */ +public class StemmingTestCase extends RuleBaseAbstractTestCase { + + public StemmingTestCase(String name) { + super(name,"stemming.sr"); + } + + public void testRewritingDueToStemmingInQuery() { + assertSemantics("+i:vehicle -i:s","i:cars -i:s"); + } + + public void testRewritingDueToStemmingInRule() { + assertSemantics("+i:animal -i:s","i:horse -i:s"); + } + + public void testRewritingDueToExactMatch() { + assertSemantics("+(AND i:arts i:sciences) -i:s","i:as -i:s"); + } + + public void testNoRewritingBecauseShortWordsAreNotStemmed() { + assertSemantics("+i:a -i:s","i:a -i:s"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StopwordTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StopwordTestCase.java new file mode 100644 index 00000000000..bfac8149a99 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StopwordTestCase.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.search.Query; +import com.yahoo.search.test.QueryTestCase; + +/** + * Tests numeric terms + * + * @author Jon Bratseth + */ +public class StopwordTestCase extends RuleBaseAbstractTestCase { + + public StopwordTestCase(String name) { + super(name,"stopwords.sr"); + } + + public void testStopwords() { + assertSemantics("AND mlr:ve mlr:heard mlr:beautiful mlr:world", + new Query(QueryTestCase.httpEncode("?query=i don't know if you've heard, but it's a beautiful world&default-index=mlr&tracelevel.rules=0"))); + } + + public void testStopwordsInPhrase() { + assertSemantics("AND mlr:\"ve heard\" mlr:beautiful mlr:world", + new Query(QueryTestCase.httpEncode("?query=\"i don't know if you've heard\", but it's a beautiful world&default-index=mlr&tracelevel.rules=0"))); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/UrlTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/UrlTestCase.java new file mode 100644 index 00000000000..3dbb27332db --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/UrlTestCase.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * Tests working with url indexes + * + * @author Jon S Bratseth + */ +public class UrlTestCase extends RuleBaseAbstractTestCase { + + public UrlTestCase(String name) { + super(name,"url.sr"); + } + + public void testFromDefaultToUrlIndex() { + assertSemantics("fromurl:\"youtube com\"","youtube.com"); + } + + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java new file mode 100644 index 00000000000..d8263f915b0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +/** + * @author Jon Bratseth + */ +public class WeightingTestCase extends RuleBaseAbstractTestCase { + + public WeightingTestCase(String name) { + super(name,"weighting.sr"); + } + + public void testWeighting() { + assertSemantics("foo!150","foo"); + assertSemantics("AND foo!150 snip","foo snip"); + assertSemantics("AND foo!150 bar","foo bar"); + assertSemantics("AND bar!57 foo","bar foo"); + assertSemantics("AND foo!150 fu","foo fu"); + assertSemantics("AND foo!150 bar kanoo boat!237","foo bar kanoo"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/alibaba.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/alibaba.sr new file mode 100644 index 00000000000..dcf9512ecc9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/alibaba.sr @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +3100 -> nokia 3100; +legend -> lenovo; + + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/anchor.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/anchor.sr new file mode 100644 index 00000000000..fbdbddb003c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/anchor.sr @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +. first -> anchor; +last . -> anchor; +. word. -> anchor; +. [phrases] -> anchor; +.anotherword. -> anchor; +. this is complete . -> anchor; + +[phrases] :- a phrase, another phrase; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatanot.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatanot.sr new file mode 100644 index 00000000000..7faddc0ad77 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatanot.sr @@ -0,0 +1,3 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +![B] +> $busname:[B]; + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatarules.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatarules.sr new file mode 100644 index 00000000000..30fc47eace0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/automatarules.sr @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# For testing referenced inverted matches +parameter.donomatch ![C] -> nomatch:[C]; + + +# Shopping use case + +[brand] +> $dsp1:[brand]; +[category] +> $dsp5:[category]; + +[brand] :- [C]; +[category] :- [B]; + +dsp5:digital dsp5:camera -> dsp5:digicamera; + + + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/backtrackingrules.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/backtrackingrules.sr new file mode 100644 index 00000000000..68e1459d6a7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/backtrackingrules.sr @@ -0,0 +1,28 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Literals + +[case1pos1],[case1pos2] -> replaced; + +[case1pos1] :- (word1 word2 word3 word4), (word1 word2 word5 word6); + +[case1pos2] :- (word1 word2 word5 word7), (word1 word2 word5 word8); + + +# References +# This is rather artificial because this rule references contexts which are +# conditionally present. Should we detect and make that illegal? + +[case2pos1],[case2pos2] -> ref:[ref1] ref:[ref2] ref:[ref5] ref:[ref8]; + +[case2pos1] :- ([ref1] [ref2] [ref3] [ref4]), ([ref1] [ref2] [ref5] [ref6]); + +[case2pos2] :- ([ref1] [ref2] [ref5] [ref7]), ([ref1] [ref2] [ref5] [ref8]); + +[ref1] :- ref1; +[ref2] :- ref2; +[ref3] :- ref3; +[ref4] :- ref4; +[ref5] :- ref5; +[ref6] :- ref6; +[ref7] :- ref7; +[ref8] :- ref8; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/blending.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/blending.sr new file mode 100644 index 00000000000..9e624182155 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/blending.sr @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[...] cd +> parameter.search='music'; + +[car] +> parameter.search='cars'; + +parameter.search='music,cars' +> parameter.search='carstereos'; + +[car] :- audi, vw, ford; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.fsa b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.fsa new file mode 100644 index 00000000000..e0285a5b277 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.txt b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.txt new file mode 100644 index 00000000000..45275d7e605 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/catchoose.txt @@ -0,0 +1,4 @@ +a b c [ {"rule": "biz", "weight": 0.5}, {"rule": "cat", "weight": 0.1} ] +a b c d [ {"rule": "cat", "weight": 0.2} ] +d e f [ {"rule": "biz", "weight": 0.5}, {"rule": "cat", "weight": 0.01} ] +d e f g [ {"rule": "cat", "weight": 0.8} ] diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk-rules.cfg b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk-rules.cfg new file mode 100644 index 00000000000..b040a44ad90 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk-rules.cfg @@ -0,0 +1,6 @@ +compatibility false +rulebase[1] +rulebase[0].name "cjk" +rulebase[0].isdefault false +rulebase[0].automata +rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n\n" \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk.sr new file mode 100644 index 00000000000..bcecbc31861 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/cjk.sr @@ -0,0 +1,19 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Use unicode equivalents in java source: +# +# 佳:\u4f73 +# 能:\u80fd +# 索:\u7d22 +# 尼:\u5c3c +# 惠:\u60e0 +# 普:\u666e + +@default + +a索 -> 索a; + +[brand] -> brand:[brand]; + +土豆 -> 马铃薯; + +[brand] :- 索尼,惠普,佳能; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparison.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparison.sr new file mode 100644 index 00000000000..23623d7e282 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparison.sr @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[island] +> island:[island]; + +[coffee] +> coffee:[coffee]; + +[island]=[coffee] +> control:ambigous; + +[island] :- java, borneo, kanava; + +[coffee] :- arabica, java; + +[island]=~'av' +> off; + +[...]>'o' +> highletter; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparisons.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparisons.sr new file mode 100644 index 00000000000..7ac73035972 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/comparisons.sr @@ -0,0 +1,3 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[...] parameter.ranking='category' +> $foo:[...]; + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/duplicaterules.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/duplicaterules.sr new file mode 100644 index 00000000000..ddd16aaa8f0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/duplicaterules.sr @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Duplicate rule definition + +[something] -> hello there + +[something] :- foo bar + +[something] :- cafe babe diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis.sr new file mode 100644 index 00000000000..7ae563841a3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis.sr @@ -0,0 +1,36 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# tests rules containing ellipses (wildcards) + +# From tutorial, referenced ellipsis +[...] album -> album:[...]; + +# From tutorial, referenced ellipsis +[...] [album] -> album:[...]; +[album] :- album, cd, record, lp; + + + +# Invented answerish use case, unreferenced ellipsis +why is [noun] ... [adjective] +> about:[noun]; + +[noun] :- stench, the sky, aardwark; +[adjective] :- unpleasant, blue, most relevant; + + + +# Ellipsis in named condition +buy [video] -> name:[videoname] product:video; + +[video] :- videoname/[...] [videosynonym], videoname/[knownvideoname]; + +[knownvideoname] :- a sun came, illinois, the avalance, seven swans; + +[videosynonym] :- dvd, video; + + + +# Multiple ellipsis + +from from/[...] to to/[...] -> from:[from] to:[to]; + + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis2.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis2.sr new file mode 100644 index 00000000000..47bc121b085 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/ellipsis2.sr @@ -0,0 +1,2 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[...] +> someindex:[...] ; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/empty.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/empty.sr new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatch.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatch.sr new file mode 100644 index 00000000000..942bf7fe2f7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatch.sr @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +. primetime notime . -> primetime in no time; +. primetime . -> primetime in no time; +. prime time in no time . -> primetime in no time; +. prime time . -> primetime in no time; +. pint . -> primetime in no time; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatchtrick.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatchtrick.sr new file mode 100644 index 00000000000..041d759de4f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/exactmatchtrick.sr @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +primetime notime -> default:primetime default:in default:no default:time; +primetime -> default:primetime default:in default:no default:time; +prime time in no time -> default:primetime default:in default:no default:time; +prime time -> default:primetime default:in default:no default:time; +pint -> default:primetime default:in default:no default:time; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child1.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child1.sr new file mode 100644 index 00000000000..e76f5ac2dd9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child1.sr @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vw -> audi; + +@include(parent.sr) + +vehiclebrand:audi -> vehiclebrand:skoda; + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child2.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child2.sr new file mode 100644 index 00000000000..2ae11bcd058 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/child2.sr @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@include(parent) + +vehiclebrand:vw -> vehiclebrand:audi; + +[brand] :- @super, skoda; + + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/cjk.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/cjk.sr new file mode 100644 index 00000000000..b2a54fbc5eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/cjk.sr @@ -0,0 +1,3 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +?? -> ???; +@default diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandchild.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandchild.sr new file mode 100644 index 00000000000..c3e1b74f235 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandchild.sr @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@include(child1.sr) +@include(child2.sr) + +causesphrase -> "a produced phrase"; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandfather.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandfather.sr new file mode 100644 index 00000000000..dbd3b3587a6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandfather.sr @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[vehicle] :- car, motorcycle, bus; + +cars -> car; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandmother.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandmother.sr new file mode 100644 index 00000000000..430227ca23c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/grandmother.sr @@ -0,0 +1,2 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vehiclebrand:bmw +> expensivetv; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/parent.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/parent.sr new file mode 100644 index 00000000000..9791561e8c7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/inheritingrules/parent.sr @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@include(grandfather.sr) + +[brand] [vehicle] -> vehiclebrand:[brand]; + +@include(grandmother.sr) + +[brand] :- alfa, audi, bmw; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/labelmatching.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/labelmatching.sr new file mode 100644 index 00000000000..b70d7f89791 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/labelmatching.sr @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@default + +term -> matched:term; + +term2 +> hit; + +dcattitle:[REST] +> -dcat:hotel; +[REST] :- restaurant, restaurants; + +foo:(one, two) -> three; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/match-only-if-not-only-term.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/match-only-if-not-only-term.sr new file mode 100644 index 00000000000..452fe2658aa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/match-only-if-not-only-term.sr @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +snl -> saturday night live; +[snlterm] -> $showname:[snlterm]!1000; +[snlterm] :- saturday night live; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/matchall.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/matchall.sr new file mode 100644 index 00000000000..2e6e3426d1b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/matchall.sr @@ -0,0 +1,2 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[...] +> $normtitle:[...]; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/nostemming.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/nostemming.sr new file mode 100644 index 00000000000..ff731b7bc08 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/nostemming.sr @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@stemming(false) +i:as -> i:arts i:sciences; +i:asc -> i:arts i:sciences i:crafts; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/not.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/not.sr new file mode 100644 index 00000000000..e4dadf3ed01 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/not.sr @@ -0,0 +1,2 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[...] !parameter.ranking='category' +> $foo:[...]; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numbers.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numbers.sr new file mode 100644 index 00000000000..a22c49b14ec --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numbers.sr @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +1337 -> leet; + +leet -> elite; + +one -> 1; + +two 2 -> 3 three; + +opp -> bort ned; + +foo bar -> kanoo knagg; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numericterms.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numericterms.sr new file mode 100644 index 00000000000..f2270eadfdd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/numericterms.sr @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[REST] +> -ycat2gc:96929265; +[REST] :- restaurant, restaurants; + +49 -> 48; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/orphrase.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/orphrase.sr new file mode 100644 index 00000000000..8e168dde591 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/orphrase.sr @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@default +[title] -> ?title:"software engineer"; + +[title] :- java engineer,software engineer,it engineer; +[title] -> title:[title]; + +# lotr +> date:656288040!0; +lotr +> ?"lord of the rings"; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter.sr new file mode 100644 index 00000000000..d9c0019a3da --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter.sr @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[...] parameter.ranking='category' +> $foo:[...]; + +parameter.hits>='11' +> largepage; + +parameter.search=~'music' +> intent:music; + +# Adding parameters +a b c +> parameter.search='letters,alphabet'; +a c d +> parameter.search='letters, someletters'; +a d e +> parameter.search=someletters -letter:c; +a d f +> parameter.rank-profile=foo; +a f g +> parameter.grouping.nolearning=true; + + +[REST] ( parameter.ranking='cat', parameter.ranking='cat0' ) -> one; +[REST] :- foo, bar; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter2.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter2.sr new file mode 100644 index 00000000000..d648b314d7e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter2.sr @@ -0,0 +1,2 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +parameter.ranking='usrank' -> parameter.ranking='date'; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter3.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/parameter3.sr new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr new file mode 100644 index 00000000000..1b1103899be --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/phrasematch.sr @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[ret] -> retailer:[ret]; +[ret] :- keyword:[B]; + +retailer:"[...]" -> retailer:[...]; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/rules.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/rules.sr new file mode 100644 index 00000000000..96f3207edcd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/rules.sr @@ -0,0 +1,71 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Local use case + +[listing] [preposition] [place] -> listing:[listing] + place:[place]; + +[listing] :- restaurant, shop, cafe, hotel; + +[preposition] :- in, at, near; + +[place] :- [street] [city], [street], [city], [state]; + +[street] :- geary street, geary; +[city] :- san francisco, paris; +[state] :- texas; + +# Shopping use case + +[brand] -> brand:[brand]; +[category] -> category:[category]; + +[brand] :- sony, dell; # Refer to automata later +[category] :- digital camera, camera, phone; # Ditto + +# Travel use case, note how explicit reference name overrides named condition as reference name + +from/[place] to to/[place] -> from:[from] to:[to]; + +# Adding rule using the default query mode (and/or) + +[foobar] +> foobar:[foobar]; + +[foobar] :- foo, bar; + +# Adding rank rule + +[word] +> $foobar:[word]; + +[word] :- aardwark, word; + +# Literal production + +lotr -> lord of the rings; + +# Adding a negative + +java +> -coffee; + +# Adding an or term +something +> ?somethingelse; + +# Adding another negative +# TODO: Term types in conditions +# java -coffee +> -island + +# "Stopwords" removal + +be -> ; +the -> ; + +[stopword] -> ; + +[stopword] :- to, + or, + not; + +# Changing term type + +[typechange] -> $default:[typechange] ; + +[typechange] :- typechange; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantic-rules.cfg b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantic-rules.cfg new file mode 100644 index 00000000000..3b713977cd7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantic-rules.cfg @@ -0,0 +1,15 @@ +rulebase[7] +rulebase[0].name "child1" +rulebase[0].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\nvw -> audi;\n\n@include(parent.sr)\n\nvehiclebrand:audi -> vehiclebrand:skoda;\n\n" +rulebase[1].name "child2" +rulebase[1].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n@include(parent)\n\nvehiclebrand:vw -> vehiclebrand:audi;\n\n[brand] :- @super, skoda;\n\n\n" +rulebase[2].name "cjk" +rulebase[2].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n?? -> ???;\n@default\n" +rulebase[3].name "grandchild" +rulebase[3].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n@include(child1.sr)\n@include(child2.sr)\n\ncausesphrase -> "a produced phrase";\n" +rulebase[4].name "grandfather" +rulebase[4].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n[vehicle] :- car, motorcycle, bus;\n\ncars -> car;\n" +rulebase[5].name "grandmother" +rulebase[5].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\nvehiclebrand:bmw +> expensivetv;\n" +rulebase[6].name "parent" +rulebase[6].rules "# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.\n@include(grandfather.sr)\n\n[brand] [vehicle] -> vehiclebrand:[brand];\n\n@include(grandmother.sr)\n\n[brand] :- alfa, audi, bmw;\n" diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.fsa b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.fsa new file mode 100644 index 00000000000..5b329d073e6 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.fsa differ diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.txt b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.txt new file mode 100644 index 00000000000..27499e33f8d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/semantics.txt @@ -0,0 +1,5 @@ +car cleaners B;sothebys car cleanery|C;car repair;cleaning +carpenter B|C +carpenter 1 X +digital camera B +sony C;hardware;audio and video;video games;photography;computers & software hardware pc notebooks sony;receiver;walkman;lan;cameras;stick;insten;unlocked;arms;toy;remote;commander;camcorde;faceplate;vision;disk;cassette;usb;zoom;psp;fury;trails;reciver;rebate;ear;ide;viewfinder;see;qtys;ghz;kinetic;sdram;vtr;video games handheld other;lcjra;loops;heaphones;downhill;projection;dsc;handycam;vaio;grs;np;eyetoy;video games playstation games adventure;sysytem;pspbundle;clie;minutes;musi;minidisc;plat;cfr;nhl;devic;looper;headphon;video games playstation games recreation & sports;normal;reconditioned;automotive parts & accessories accessories audio other;lsfvha;has;detect;gripshift;blazin;shakur;electronics & cameras cameras & equipment equipment cases & carrying equipment;jamtrax;aux;wooofer;xg;fxa;electronics & cameras audio portable walkmans;walkma;multiplier;had;input;consumer;tdm diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr new file mode 100644 index 00000000000..2dcb4d8775a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@stemming(true) +i:as -> i:arts i:sciences; +i:car -> i:vehicle; +i:horses -> i:animal; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stopwords.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stopwords.sr new file mode 100644 index 00000000000..71b4329379d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stopwords.sr @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@default + +[stopword] :- the, to, a,i,and,is,of,in,you,it,for,what,that,do,can,have,on,are,or,if,with,how,my,be,but,not,s,this,your,t,get,like,they,me,there,so,know,from,just,as,will,at,all,one,about,when,out,an,would,was,any,has,who,some,good,want,up,by,think,does,no,why,don,more,go,them,then,he,where,need,time,people,other,am,should,we,find,make,help,also,really,because,only,best,which,m,way,their,now,than,see,been,much,could,had,com,very,most ,its,anyone,him,many,use,first,take,his,well,even,say,her,she,work,try,u,too,please,something,were,did,someone,after,question,here,back,give,right,over,going,still,new,http,www,it's,doesn't,what's,that's,can't,how's,there's,when's; + +mlr:[stopword] -> ; +classic:[stopword] -> ; + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/substitution.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/substitution.sr new file mode 100644 index 00000000000..79a7781b914 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/substitution.sr @@ -0,0 +1,2 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +second -> third; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/url.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/url.sr new file mode 100644 index 00000000000..e01031709c7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/url.sr @@ -0,0 +1,3 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +[youtube] -> fromurl:"youtube com"; +[youtube] :- http www youtube com,youtube com,youtube,www utube com,utube com,utube,you tube,u tube; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/weighting.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/weighting.sr new file mode 100644 index 00000000000..735b1af6090 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/weighting.sr @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +foo -> foo!150; +[bars] foo -> [bars]!57 foo; +kanoo +> boat!237; + +[bars] :- bar, fu; diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java new file mode 100644 index 00000000000..7def3faaf9a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.templates.test; + +import java.io.IOException; +import java.io.Writer; + +import com.yahoo.prelude.templates.Context; +import com.yahoo.prelude.templates.UserTemplate; + +/** + * Test template which throws a runtime exception in its footer. + * + * @author Steinar Knutsen + */ +@SuppressWarnings("rawtypes") +public class BoomTemplate extends UserTemplate { + public BoomTemplate(String name, String mimeType, String encoding) { + super(name, mimeType, encoding); + } + + @Override + public void error(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void footer(Context context, Writer writer) throws IOException { + throw new RuntimeException("Boom!"); + } + + @Override + public void header(Context context, Writer writer) throws IOException { + writer.write("header"); + } + + @Override + public void hit(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void hitFooter(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void noHits(Context context, Writer writer) throws IOException { + // NOP + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java new file mode 100644 index 00000000000..7a8779ef70a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.templates.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * Tests composition of grouped results using the HitGroup class + * + * @author bratseth + */ +public class GroupedResultTestCase extends junit.framework.TestCase { + + public GroupedResultTestCase(String name) { + super(name); + } + + public void testGroupedResult() { + Result result=new Result(new Query("?query=foo")); + HitGroup hitGroup1=new HitGroup("group1",300); + hitGroup1.add(new Hit("group1.1",200)); + HitGroup hitGroup2=new HitGroup("group2",600); + Hit topLevelHit1=new Hit("toplevel.1",500); + Hit topLevelHit2=new Hit("toplevel.2",700); + result.hits().add(hitGroup1); + result.hits().add(topLevelHit1); + result.hits().add(hitGroup2); + result.hits().add(topLevelHit2); + hitGroup1.add(new Hit("group1.2",800)); + hitGroup2.add(new Hit("group2.1",800)); + hitGroup2.add(new Hit("group2.2",300)); + hitGroup2.add(new Hit("group2.3",500)); + + // Should have 7 concrete hits, ordered as + // toplevel.2 + // group2 + // group2.1 + // group2.3 + // group2.2 + // toplevel.1 + // group1 + // group1.2 + // group1.1 + // Assert this: + + assertEquals(7,result.getConcreteHitCount()); + assertEquals(4,result.getHitCount()); + + Hit topLevel2=result.hits().get(0); + assertEquals("toplevel.2",topLevel2.getId().stringValue()); + + HitGroup returnedGroup2=(HitGroup)result.hits().get(1); + assertEquals(3,returnedGroup2.getConcreteSize()); + assertEquals(3,returnedGroup2.size()); + assertEquals("group2.1",returnedGroup2.get(0).getId().stringValue()); + assertEquals("group2.3",returnedGroup2.get(1).getId().stringValue()); + assertEquals("group2.2",returnedGroup2.get(2).getId().stringValue()); + + Hit topLevel1=result.hits().get(2); + assertEquals("toplevel.1",topLevel1.getId().stringValue()); + + HitGroup returnedGroup1=(HitGroup)result.hits().get(3); + assertEquals(2,returnedGroup1.getConcreteSize()); + assertEquals(2,returnedGroup1.size()); + assertEquals("group1.2",returnedGroup1.get(0).getId().stringValue()); + assertEquals("group1.1",returnedGroup1.get(1).getId().stringValue()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java new file mode 100644 index 00000000000..55c4a2ab074 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.templates.test; + +import static org.junit.Assert.assertEquals; + +import java.lang.reflect.Method; +import java.util.List; + +import org.junit.Test; + +import com.yahoo.prelude.templates.HitContext; +import com.yahoo.protect.ClassValidator; + +/** + * Check the entire Context class is correctly masked. + * + * @author Steinar Knutsen + */ +public class HitContextTestCase { + + @Test + public void checkMethods() { + List unmasked = ClassValidator.unmaskedMethodsFromSuperclass(HitContext.class); + assertEquals("Unmasked methods from superclass: " + unmasked, 0, unmasked.size()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java new file mode 100644 index 00000000000..0e0829051ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.templates.test; + + +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; + +import com.yahoo.io.ByteWriter; +import com.yahoo.prelude.templates.UserTemplate; + +/** + * @author Steinar Knutsen + */ +public class TemplateTestCase extends junit.framework.TestCase { + + private CharsetEncoder encoder; + private ByteArrayOutputStream stream; + + public TemplateTestCase (String name) { + super(name); + Charset cs = Charset.forName("UTF-8"); + encoder = cs.newEncoder(); + stream = new ByteArrayOutputStream(); + } + + public void testASCIIQuoting() throws java.io.IOException { + stream.reset(); + byte[] c = new byte[] { 97, 98, 99, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }; + ByteWriter bw = new ByteWriter(stream, encoder); + UserTemplate.dumpAndXMLQuoteUTF8(bw, c); + bw.close(); + String res = stream.toString("UTF-8"); + String correct = "abc\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\t\n\\u000B\\u000C\r\\u000E\\u000F\\u0010\\u0011"; + assertEquals(correct, res); + + } + + public void testXMLQuoting() throws java.io.IOException { + stream.reset(); + // c = > + byte[] c = new byte[] { 60, 115, 62, 38, 103, 116, 59 }; + ByteWriter bw = new ByteWriter(stream, encoder); + UserTemplate.dumpAndXMLQuoteUTF8(bw, c); + bw.close(); + String res = stream.toString("UTF-8"); + String correct = "<s>&gt;"; + assertEquals(correct, res); + + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java new file mode 100644 index 00000000000..ca82d8695bb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.templates.test; + +import java.io.IOException; +import java.io.Writer; + +import com.yahoo.prelude.templates.Context; +import com.yahoo.prelude.templates.UserTemplate; + +/** + * Test basic UserTemplate functionality of detecting + * overridden group rendering methods. + * + * @author Steinar Knutsen + */ +@SuppressWarnings("rawtypes") +public class TestTemplate extends UserTemplate { + + public TestTemplate(String name, String mimeType, String encoding) { + super(name, mimeType, encoding); + } + + @Override + public void error(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void footer(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void header(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void hit(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void hitFooter(Context context, Writer writer) throws IOException { + // NOP + } + + @Override + public void noHits(Context context, Writer writer) throws IOException { + // NOP + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java new file mode 100644 index 00000000000..d7134ebe405 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java @@ -0,0 +1,307 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.templates.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.io.IOUtils; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.prelude.templates.TiledTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.http.HTTPProviderSearcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; + +/** + * Tests representing a federated and grouped result as a Result object and + * rendering a tiled output of the result + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class TilingTestCase extends junit.framework.TestCase { + + public TilingTestCase(String name) { + super(name); + } + + /** + * This result contains two blocks (center and right). + * The center block contains multiple subblocks while the right one contains a single block of ads. + *

+ * Incidentally, this also tests using an old searcher in new search chains. + */ + public void testTiling() throws IOException { + Chain chain=new Chain<>("tiling", new TiledResultProducer()); + + // Query it + Query query = new Query("/tiled?query=foo"); + Result result = callSearchAndSetRenderer(chain, query); + assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml")),result); + } + + /** + * This result contains center section and meta blocks. + *

+ * Incidentally, this also tests using an old searcher in new search chains. + */ + public void testTiling2() throws IOException { + Chain chain= new Chain<>("tiling", new TiledResultProducer2()); + + // Query it + Query query=new Query("/tiled?query=foo"); + Result result= callSearchAndSetRenderer(chain, query); + assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml")),result); + } + + private Result callSearchAndSetRenderer(Chain chain, Query query) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + Result result = new Execution(chain, context).search(query); + result.getTemplating().setRenderer(new SearchRendererAdaptor(new TiledTemplateSet())); + return result; + } + + public static void assertRenderedStartsWith(String expected,Result result) throws IOException { + assertRendered(expected,result,false); + } + + public static void assertRendered(String expected,Result result) throws IOException { + assertRendered(expected,result,true); + } + + public static void assertRendered(String expected, Result result,boolean checkFullEquality) throws IOException { + if (checkFullEquality) + assertEquals(filterComments(expected), getRendered(result)); + else + assertTrue(getRendered(result).startsWith(expected)); + } + + private static String filterComments(String s) { + StringBuilder b = new StringBuilder(); + for (String line : s.split("\\\n")) + if ( ! line.startsWith(" + + +

+ + + + http://www.hotels.com/ + Cheap <hi>hotels</hi> + Low Rates Guaranteed. Call a Hotel Expert. + + + http://www.expedia.com/ + Cheap <hi>hotels</hi> at Expedia + Expedia Special Rates Means We Guarantee Our Low Rates on Rooms. + + + + + www.hotels.com + Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging + Hotels.com helps you find great rates on hotels and discount hotel packages. + + + www.indigohotels.com + Hotel Indigo Hotels United States - Official Web Site + Make Hotel Indigo online hotel reservations and book your hotel rooms today. + + + www.all-hotels.com + All hotels + Online hotel directory and reservations. + + + + + www.daysinn.com + Days Inn Special Deal + Buy now and Save 15% Off Our Best Available Rate with Days Inn. + + + http://www.expedia.com/ + Cheap <hi>hotels</hi> at Expedia + Expedia Special Rates Means We Guarantee Our Low Rates on Rooms. + + +
+ +
+ + + www.daysinn.com + Days Inn Special Deal + Buy now and Save 15% Off Our Best Available Rate with Days Inn. + + + www.holidayinn.com + Holiday Inn: Official Site + Book with Holiday Inn. Free Internet. Kids eat free. + + +
+ + diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml b/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml new file mode 100644 index 00000000000..387118ec9a4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml @@ -0,0 +1,23 @@ + + + + +
+ + + 159 + Yahoo + + +
+ + + + com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search + http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2 + 757 + 16217 + + + + diff --git a/container-search/src/test/java/com/yahoo/prelude/test/DummySearcher.java b/container-search/src/test/java/com/yahoo/prelude/test/DummySearcher.java new file mode 100644 index 00000000000..e5fa79c4f62 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/DummySearcher.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * Only returns the first hit for a query. + * + * @author Steinar Knutsen + */ +public class DummySearcher extends Searcher { + + public DummySearcher() { + } + + public DummySearcher(ComponentId id) { + super(id); + } + + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result=new Result(query); + result.hits().add(new Hit("http://a.com/b", 100)); + + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/GetRawWordTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/GetRawWordTestCase.java new file mode 100644 index 00000000000..3f423830780 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/GetRawWordTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Tests a case reported by MSBE + * + * @author bratseth + */ +public class GetRawWordTestCase { + + @Test + public void testGetRawWord() { + Query query = new Query("?query=%C4%B0%C5%9EBANKASI%20GAZ%C4%B0EM%C4%B0R&searchChain=vespa"); + assertEquals("AND \u0130\u015EBANKASI GAZ\u0130EM\u0130R", query.getModel().getQueryTree().toString()); + AndItem root=(AndItem)query.getModel().getQueryTree().getRoot(); + + { + WordItem word=(WordItem)root.getItem(0); + assertEquals("\u0130\u015EBANKASI",word.getRawWord()); + assertEquals(0,word.getOrigin().start); + assertEquals(9,word.getOrigin().end); + } + + { + WordItem word=(WordItem)root.getItem(1); + assertEquals("GAZ\u0130EM\u0130R",word.getRawWord()); + assertEquals(10,word.getOrigin().start); + assertEquals(18,word.getOrigin().end); + } + + assertEquals("Total string is just these words",18,((WordItem)root.getItem(0)).getOrigin().getSuperstring().length()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java new file mode 100644 index 00000000000..0700d4489e7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -0,0 +1,277 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Alias; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Command; +import com.yahoo.language.process.StemMode; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests using synthetic index names for IndexFacts class. + * + * @author Steinar Knutsen + */ +@SuppressWarnings({"rawtypes", "unchecked"}) +public class IndexFactsTestCase extends junit.framework.TestCase { + + private static final String INDEXFACTS_TESTING = "file:src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg"; + + public IndexFactsTestCase(String name) { + super(name); + } + + private IndexFacts createIndexFacts() { + ConfigGetter getter = new ConfigGetter<>(IndexInfoConfig.class); + IndexInfoConfig config = getter.getConfig(INDEXFACTS_TESTING); + + List clusterOne = new ArrayList<>(); + List clusterTwo = new ArrayList<>(); + clusterOne.addAll(Arrays.asList("one", "two")); + clusterTwo.addAll(Arrays.asList("one", "three")); + Map> clusters = new HashMap<>(); + clusters.put("clusterOne", clusterOne); + clusters.put("clusterTwo", clusterTwo); + IndexFacts indexFacts = new IndexFacts(new IndexModel(config, clusters)); + + return indexFacts; + } + + public void testBasicCases() { + // First check default behavior + IndexFacts indexFacts = createIndexFacts(); + Query q = newQuery("?query=a:b", indexFacts); + assertEquals("a:b", q.getModel().getQueryTree().getRoot().toString()); + q = newQuery("?query=notarealindex:b", indexFacts); + assertEquals("\"notarealindex b\"", q.getModel().getQueryTree().getRoot().toString()); + + // Add an index to an SD which also happens to be the default + indexFacts.addIndex("one", "yetanothersynthetic"); + q = newQuery("?query=yetanothersynthetic:b", indexFacts); + assertEquals("yetanothersynthetic:b", q.getModel().getQueryTree().getRoot().toString()); + } + + public void testDefaultPosition() { + Index a = new Index("a"); + assertFalse(a.isDefaultPosition()); + a.addCommand("any"); + assertFalse(a.isDefaultPosition()); + a.addCommand("default-position"); + assertTrue(a.isDefaultPosition()); + + SearchDefinition sd = new SearchDefinition("sd"); + sd.addCommand("b", "any"); + assertNull(sd.getDefaultPosition()); + sd.addCommand("c", "default-position"); + assertTrue(sd.getDefaultPosition().equals("c")); + + SearchDefinition sd2 = new SearchDefinition("sd"); + sd2.addIndex(new Index("b").addCommand("any")); + assertNull(sd2.getDefaultPosition()); + sd2.addIndex(a); + assertTrue(sd2.getDefaultPosition().equals("a")); + + Map m = new TreeMap<>(); + m.put(sd.getName(), sd); + IndexFacts indexFacts = createIndexFacts(); + indexFacts.setSearchDefinitions(m,sd2); + assertTrue(indexFacts.getDefaultPosition(null).equals("a")); + assertTrue(indexFacts.getDefaultPosition("sd").equals("c")); + } + + public void testIndicesInAnyConfigurationAreIndicesInDefault() { + IndexFacts.Session indexFacts = createIndexFacts().newSession(new Query()); + assertTrue(indexFacts.isIndex("a")); + assertTrue(indexFacts.isIndex("b")); + assertTrue(indexFacts.isIndex("c")); + assertTrue(indexFacts.isIndex("d")); + assertFalse(indexFacts.isIndex("anythingelse")); + } + + public void testDefaultIsUnionHostIndex() { + IndexFacts.Session session = createIndexFacts().newSession(new Query()); + assertTrue(session.getIndex("c").isHostIndex()); + assertFalse(session.getIndex("a").isHostIndex()); + } + + public void testDefaultIsUnionUriIndex() { + IndexFacts indexFacts = createIndexFacts(); + assertTrue(indexFacts.newSession(new Query()).getIndex("d").isUriIndex()); + assertFalse(indexFacts.newSession(new Query()).getIndex("a").isUriIndex()); + } + + public void testDefaultIsUnionStemMode() { + IndexFacts.Session session = createIndexFacts().newSession(new Query()); + assertEquals(StemMode.NONE, session.getIndex("a").getStemMode()); + assertEquals(StemMode.NONE, session.getIndex("b").getStemMode()); + } + + private void assertExactIsWorking(String indexName) { + Index index=new Index(indexName); + index.setExact(true,"^^^"); + IndexFacts indexFacts = createIndexFacts(); + indexFacts.addIndex("artist",index); + Query query = new Query(); + query.getModel().getSources().add("artist"); + assertTrue(indexFacts.newSession(query).getIndex(indexName).isExact()); + Query q = newQuery("?query=" + indexName + ":foo...&search=artist", indexFacts); + assertEquals(indexName + ":foo...", q.getModel().getQueryTree().getRoot().toString()); + } + + public void testExactMatching() { + assertExactIsWorking("test"); + assertExactIsWorking("artist_name_ft_norm1"); + + List search=new ArrayList(); + search.add("three"); + Query query = new Query(); + query.getModel().getSources().add("three"); + IndexFacts.Session threeSession = createIndexFacts().newSession(query); + IndexFacts.Session nullSession = createIndexFacts().newSession(new Query()); + + Index d3 = threeSession.getIndex("d"); + assertTrue(d3.isExact()); + assertEquals(" ", d3.getExactTerminator()); + + Index e = nullSession.getIndex("e"); + assertTrue(e.isExact()); + assertEquals("kj(/&",e.getExactTerminator()); + + Index a = nullSession.getIndex("a"); + assertFalse(a.isExact()); + assertNull(a.getExactTerminator()); + + Index wem = threeSession.getIndex("twewm"); + assertTrue(wem.isExact()); + assertNull(wem.getExactTerminator()); + } + + public void testComplexExactMatching() { + IndexFacts indexFacts = createIndexFacts(); + String u_name = "foo_bar"; + Index u_index = new Index(u_name); + u_index.setExact(true, "^^^"); + Index b_index = new Index("bar"); + indexFacts.addIndex("foobar", u_index); + indexFacts.addIndex("foobar", b_index); + Query query = new Query(); + query.getModel().getSources().add("foobar"); + IndexFacts.Session session = indexFacts.newSession(query); + assertFalse(session.getIndex("bar").isExact()); + assertTrue(session.getIndex(u_name).isExact()); + Query q = newQuery("?query=" + u_name + ":foo...&search=foobar", indexFacts); + assertEquals(u_name + ":foo...", q.getModel().getQueryTree().getRoot().toString()); + } + + // This is also backed by a system test on cause of complex config + public void testRestrictLists1() { + Query query = new Query(); + query.getModel().getSources().add("nalle"); + query.getModel().getSources().add("one"); + query.getModel().getRestrict().add("two"); + + IndexFacts.Session indexFacts = createIndexFacts().newSession(Collections.singleton("clusterOne"), Collections.emptyList()); + assertTrue(indexFacts.isIndex("a")); + assertFalse(indexFacts.isIndex("b")); + assertTrue(indexFacts.isIndex("d")); + } + + public void testRestrictLists2() { + Query query = new Query(); + query.getModel().getSources().add("clusterTwo"); + query.getModel().getRestrict().add("three"); + IndexFacts indexFacts = createIndexFacts(); + IndexFacts.Session session = indexFacts.newSession(query); + assertFalse(session.getIndex("c").isNull()); + assertTrue(session.getIndex("e").isNull()); + assertEquals("c", session.getCanonicName("C")); + assertTrue(session.getIndex("c").isHostIndex()); + assertFalse(session.getIndex("a").isNull()); + assertFalse(session.getIndex("a").isHostIndex()); + assertEquals(StemMode.SHORTEST, session.getIndex("a").getStemMode()); + assertFalse(session.getIndex("b").isNull()); + assertFalse(session.getIndex("b").isUriIndex()); + assertFalse(session.getIndex("b").isHostIndex()); + assertEquals(StemMode.NONE, session.getIndex("b").getStemMode()); + } + + public void testRestrictLists3() { + Query query = new Query(); + query.getModel().getSources().add("clusterOne"); + query.getModel().getRestrict().add("two"); + IndexFacts indexFacts = createIndexFacts(); + IndexFacts.Session session = indexFacts.newSession(query); + assertTrue(session.getIndex("a").isNull()); + assertFalse(session.getIndex("d").isNull()); + assertTrue(session.getIndex("d").isUriIndex()); + assertTrue(session.getIndex("e").isExact()); + } + + public void testOverlappingAliases() { + IndexInfoConfig cfg = new IndexInfoConfig(new IndexInfoConfig.Builder() + .indexinfo( + new Indexinfo.Builder() + .name("music2") + .command( + new Command.Builder().indexname( + "btitle").command("index")) + .alias(new Alias.Builder().alias("title") + .indexname("btitle"))).indexinfo( + new Indexinfo.Builder().name("music").command( + new Command.Builder().indexname("title") + .command("index")))); + IndexModel m = new IndexModel(cfg, (QrSearchersConfig)null); + assertNotNull(m.getSearchDefinitions().get("music").getIndex("title")); + assertNull(m.getSearchDefinitions().get("music").getIndex("btitle")); + assertNotNull(m.getSearchDefinitions().get("music2").getIndex("btitle")); + assertNotNull(m.getSearchDefinitions().get("music2").getIndex("title")); + assertSame(m.getSearchDefinitions().get("music2").getIndex("btitle"), + m.getSearchDefinitions().get("music2").getIndex("title")); + assertNotSame(m.getSearchDefinitions().get("music").getIndex("title"), + m.getSearchDefinitions().get("music2").getIndex("title")); + } + + private Query newQuery(String queryString, IndexFacts indexFacts) { + Query query = new Query(queryString); + query.getModel().setExecution(new Execution(new Execution.Context(null, indexFacts, null, null, null))); + return query; + } + + public void testPredicateBounds() { + Index index = new Index("a"); + assertEquals(Long.MIN_VALUE, index.getPredicateLowerBound()); + assertEquals(Long.MAX_VALUE, index.getPredicateUpperBound()); + index.addCommand("predicate-bounds [2..300]"); + assertEquals(2L, index.getPredicateLowerBound()); + assertEquals(300L, index.getPredicateUpperBound()); + index.addCommand("predicate-bounds [-20000..30000]"); + assertEquals(-20_000L, index.getPredicateLowerBound()); + assertEquals(30_000L, index.getPredicateUpperBound()); + index.addCommand("predicate-bounds [-40000000000..-300]"); + assertEquals(-40_000_000_000L, index.getPredicateLowerBound()); + assertEquals(-300L, index.getPredicateUpperBound()); + index.addCommand("predicate-bounds [..300]"); + assertEquals(Long.MIN_VALUE, index.getPredicateLowerBound()); + assertEquals(300L, index.getPredicateUpperBound()); + index.addCommand("predicate-bounds [2..]"); + assertEquals(2L, index.getPredicateLowerBound()); + assertEquals(Long.MAX_VALUE, index.getPredicateUpperBound()); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java new file mode 100644 index 00000000000..b3715c3a944 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/IntegrationTestCase.java @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Runs a query thru the configured search chain from a real http channel + * to a mock fdispatch channel. The setup is rather complicated, as the point is + * to span as much of the processing from query to result as possible. + * + * @author bratseth + */ +public class IntegrationTestCase extends junit.framework.TestCase { + + public IntegrationTestCase (String name) { + super(name); + } + + public static class SecondSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("searcher:2",1000)); + return result; + } + } + public static class ThirdSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("searcher:3",1000)); + return result; + } + } + + public void testQuery() throws java.io.IOException { + /* + TODO: (JSB) This blocks forever on Linux (not Windows) because + the ServerSocketChannel.accept method in Server + seems to starve the test running thread, + causing it to get stuck in waitForServerInitialization. + This must be caused by starvation because + replacing the test with Thread.sleep(n) + gives the same result if n is large enough (2000 + is large enough, 1000 is not. + Resolve this in some way, perhaps by switching to + non-blocking io (and then remember to remove the next line). + */ + } + + /* + if (1==1) return; + ServerThread serverThread=new ServerThread(); + try { + serverThread.start(); + waitForServerInitialization(); + insertMockFs4Channel(); + ByteBuffer buffer=ByteBuffer.allocate(4096); + buffer.put(getBytes("GET /?query=hans HTTP/1.1\n\n")); + SocketChannel socket= + SocketChannel.open(new InetSocketAddress(Server.get().getHost(), + Server.get().getPort())); + buffer.flip(); + socket.write(buffer); + + buffer.clear(); + socket.read(buffer); + // TODO: Validate return too + + } + finally { + serverThread.interrupt(); + } + } + + private static void assertCorrectQueryData(QueryPacket packet) { + assertEquals("Query x packet " + + "[query: query 'RANK hans bcatpat.bidxpatlvl1:hans' [path='/']]", + packet.toString()); + } + + private void insertMockFs4Channel() { + Searcher current=SearchChain.get(); + while (current.getChained().getChained()!=null) + current=current.getChained(); + assertTrue(current.getChained() instanceof FastSearcher); + FastSearcher mockFastSearcher= + new FastSearcher(new MockFSChannel(), + "file:etc/qr-summary.cf", + "testhittype"); + current.setChained(mockFastSearcher); + } + + private void waitForServerInitialization() { + int sleptMs=0; + while (Server.get().getHost()==null) { + try { Thread.sleep(10); } catch (Exception e) {} + sleptMs+=10; + } + } + + private class ServerThread extends Thread { + + public void run() { + try { + Server.get().start(8081,new SearchRequestHandler()); + } + catch (java.io.IOException e) { + throw new RuntimeException("Failed",e); + } + } + } + + private byte[] getBytes(String string) { + try { + return string.getBytes("utf-8"); + } + catch (java.io.UnsupportedEncodingException e) { + throw new RuntimeException("Won't happen",e); + } + } + */ + /** A channel which returns hardcoded packets of the same type as fdispatch */ + /* + private static class MockFSChannel extends Channel { + + public MockFSChannel() {} + + public void sendPacket(Packet packet) { + if (packet instanceof QueryPacket) { + assertCorrectQueryData((QueryPacket)packet); + } + else { + throw new RuntimeException("Mock channel don't know what to reply to " + + packet); + } + } + + public Packet[] receivePackets() { + List packets=new java.util.ArrayList(); + QueryResultPacket result=QueryResultPacket.create(); + result.addDocument(new DocumentInfo(123,2003,234,1000,1)); + result.addDocument(new DocumentInfo(456,1855,234,1001,1)); + packets.add(result); + addDocsums(packets); + return (Packet[])packets.toArray(new Packet[packets.size()]); + } + + private void addDocsums(List packets) { + ByteBuffer buffer=createDocsumPacketData(DocsumDefinitionTestCase.docsum4); + buffer.position(0); + packets.add(PacketDecoder.decode(buffer)); + + buffer=createDocsumPacketData(DocsumDefinitionTestCase.docsum4); + buffer.position(0); + packets.add(PacketDecoder.decode(buffer)); + + packets.add(EolPacket.create()); + } + + private ByteBuffer createDocsumPacketData(byte[] docsumData) { + ByteBuffer buffer=ByteBuffer.allocate(docsumData.length+12+4); + buffer.limit(buffer.capacity()); + buffer.position(0); + buffer.putInt(docsumData.length+8+4); + buffer.putInt(205); // Docsum packet code + buffer.putInt(0); + buffer.putInt(0); + buffer.put(docsumData); + return buffer; + } + + } + */ +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/LocationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/LocationTestCase.java new file mode 100644 index 00000000000..d7398b70528 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/LocationTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import com.yahoo.prelude.Location; + +/** + * Tests the Location class. Currently does not test all "features" of Location class. + * + * @author Einar M. R. Rosenvinge + */ +public class LocationTestCase extends junit.framework.TestCase { + + public LocationTestCase (String name) { + super(name); + } + + public void testAspect() { + //0 degrees latitude, on the equator + Location loc1 = new Location("[2,-1110000,330000,-1160000,340000](2,-1100222,0,300,0,1,0,CalcLatLon)"); + assertEquals(loc1.toString(), "[2,-1110000,330000,-1160000,340000](2,-1100222,0,300,0,1,0,4294967295)"); + + //90 degrees latitude, on the north pole + Location loc2 = new Location("[2,-1110000,330000,-1160000,340000](2,-1100222,90000000,300,0,1,0,CalcLatLon)"); + assertEquals(loc2.toString(), "[2,-1110000,330000,-1160000,340000](2,-1100222,90000000,300,0,1,0)"); + + Location loc3 = new Location("attr1:[2,-1110000,330000,-1160000,340000](2,-1100222,0,300,0,1,0,CalcLatLon)"); + assertEquals(loc3.toString(), "attr1:[2,-1110000,330000,-1160000,340000](2,-1100222,0,300,0,1,0,4294967295)"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java new file mode 100644 index 00000000000..23eded6b561 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/NullSetMemberTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import java.util.HashSet; + +/** + * Tests null members in HashSet + */ +public class NullSetMemberTestCase extends junit.framework.TestCase { + + public NullSetMemberTestCase (String name) { + super(name); + } + + public void testNullMember() { + HashSet s = new HashSet(); + assertEquals(s.size(), 0); + assertFalse(s.contains(null)); + s.add(null); + assertEquals(s.size(), 1); + assertTrue(s.contains(null)); + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java new file mode 100644 index 00000000000..e8c3f405fca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -0,0 +1,398 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.query.Sorting; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.Exceptions; +import org.hamcrest.Matcher; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.anyOf; + +import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for query class + * + * @author Bjorn Borud + */ +public class QueryTestCase { + + private final IndexFacts facts = new IndexFacts(); + + @Before + public void setUp() { + // Setup the indices we expect + facts.addIndex(null,"fast.type"); + facts.addIndex(null,"def"); + facts.addIndex(null,"default"); + facts.addIndex(null,"keyword"); + facts.addIndex(null,"content"); + } + + /** + * Basic test + */ + @Test + public void testSimpleQueryParsing () { + Query q = newQuery("/search?query=foobar&offset=10&hits=20"); + assertEquals("foobar",((WordItem) q.getModel().getQueryTree().getRoot()).getWord()); + assertEquals(10,q.getOffset()); + assertEquals(20,q.getHits()); + } + + /** + * Not quite so basic + */ + @Test + public void testAdvancedQueryParsing () { + Query q = newQuery("/search?query=fOObar and kanoo&offset=10&hits=20&filter=-foo +bar&type=adv&suggestonly=true"); + assertEquals("AND (+(AND fOObar kanoo) -|foo) |bar", q.getModel().getQueryTree().getRoot().toString()); + assertEquals(10,q.getOffset()); + assertEquals(20,q.getHits()); + assertEquals(true, q.properties().getBoolean("suggestonly", false)); + } + + /** + * Not quite so basic + */ + @Test + public void testAnyQueryParsing () { + Query q = newQuery("/search?query=foobar and kanoo&offset=10&hits=10&type=any&suggestonly=true&filter=-fast.type:offensive&encoding=latin1"); + assertEquals("+(OR foobar and kanoo) -|fast.type:offensive", q.getModel().getQueryTree().getRoot().toString()); + assertEquals(10,q.getOffset()); + assertEquals(10,q.getHits()); + assertEquals(true, q.properties().getBoolean("suggestonly", false)); + assertEquals("latin1",q.getModel().getEncoding()); + } + + /** + * A long string + */ + @Test + public void testLongQueryParsing() { + Query q = newQuery("/p13n?query=news" + +"interest:cnn!254+interest:cnnfn!171+interest:cnn+" + +"financial!96+interest:" + +"yahoo+com!253+interest:www+yahoo!138+" + +"interest:www+yahoo+com!136" + +"&hits=20&offset=0&vectorranking=queryrank"); + assertEquals("/p13n", q.getHttpRequest().getUri().getPath()); + assertEquals(0,q.getOffset()); + assertEquals(20,q.getHits()); + assertEquals("queryrank", q.properties().get("vectorranking")); + } + + /** + * Test that the integer convenience wrapper works as documented, + * throwing NumberFormatException when applied to something that + * is not a number. + */ + @Test + public void testGetParamInt() { + Query q = newQuery("/search?query=foo%20bar&someint=10¬int=hello"); + assertEquals(10,(int)q.properties().getInteger("someint")); + + // provoke an exception. if exception is not triggered + // we fail the test. + try { + q.properties().getInteger("notint"); + fail("Trying to access non-integer as integer should fail"); + } catch (java.lang.NumberFormatException e) { + // NOP + } + } + + /** + * Test UTF-8 decoding + */ + @Test + public void testUtf8Decoding() { + Query q = new Query("/?query=beyonc%C3%A9"); + assertEquals("beyonc\u00e9",((WordItem) q.getModel().getQueryTree().getRoot()).getWord()); + } + + /** + * Check sortspec "parsing" is correct. + */ + @Test + public void testSortSpec() { + Query q = newQuery("?query=test&sortspec=+a -b c +[rank]"); + assertNotNull(q.getRanking().getSorting()); + List sortSpec = q.getRanking().getSorting().fieldOrders(); + assertEquals(sortSpec.size(), 4); + assertEquals(Sorting.Order.ASCENDING, sortSpec.get(0).getSortOrder()); + assertEquals("a", sortSpec.get(0).getFieldName()); + assertEquals(Sorting.Order.DESCENDING, sortSpec.get(1).getSortOrder()); + assertEquals("b", sortSpec.get(1).getFieldName()); + assertEquals(Sorting.Order.UNDEFINED, sortSpec.get(2).getSortOrder()); + assertEquals("c", sortSpec.get(2).getFieldName()); + assertEquals(Sorting.Order.ASCENDING, sortSpec.get(3).getSortOrder()); + assertEquals("[rank]", sortSpec.get(3).getFieldName()); + } + + @Test + public void testSortSpecLowerCase() { + Query q = newQuery("?query=test&sortspec=-lowercase(name)"); + assertNotNull(q.getRanking().getSorting()); + List sortSpec = q.getRanking().getSorting().fieldOrders(); + assertEquals(sortSpec.size(), 1); + assertEquals(Sorting.Order.DESCENDING, + sortSpec.get(0).getSortOrder()); + assertEquals("name", sortSpec.get(0).getFieldName()); + assertTrue(sortSpec.get(0).getSorter() instanceof Sorting.LowerCaseSorter); + } + + public void checkUcaUS(String spec) { + Query q = newQuery(spec); + assertNotNull(q.getRanking().getSorting()); + List sortSpec = q.getRanking().getSorting().fieldOrders(); + assertEquals(sortSpec.size(), 1); + assertEquals(Sorting.Order.DESCENDING, + sortSpec.get(0).getSortOrder()); + assertTrue(sortSpec.get(0).getSorter() instanceof Sorting.UcaSorter); + assertEquals("name", sortSpec.get(0).getFieldName()); + } + + @Test + public void testSortSpecUca() { + checkUcaUS("?query=test&sortspec=-uca(name,en_US)"); + checkUcaUS("?query=test&sortspec=-UCA(name,en_US)"); + checkSortSpecUcaUSOptional("?query=test&sortspec=-uca(name,en_US,tertiary)"); + checkSortSpecUcaUSOptional("?query=test&sortspec=-uca(name,en_US,TERTIARY)"); + } + + @Test + public void testInvalidSortFunction() { + assertQueryError( + "?query=test&sortspec=-ucca(name,en_US)", + containsString("Could not set 'ranking.sorting' to '-ucca(name,en_US)': Unknown sort function 'ucca'")); + } + + @Test + public void testMissingSortFunction() { + assertQueryError( + "?query=test&sortspec=-(name)", + containsString("Could not set 'ranking.sorting' to '-(name)': No sort function specified")); + } + + @Test + public void testInvalidUcaStrength() { + assertQueryError( + "?query=test&sortspec=-uca(name,en_US,tertary)", + containsString("Could not set 'ranking.sorting' to '-uca(name,en_US,tertary)': Unknown collation strength: 'tertary'")); + } + + public void checkSortSpecUcaUSOptional(String spec) { + Query q = newQuery(spec); + assertNotNull(q.getRanking().getSorting()); + List sortSpec = q.getRanking().getSorting().fieldOrders(); + assertEquals(sortSpec.size(), 1); + assertEquals(Sorting.Order.DESCENDING, + sortSpec.get(0).getSortOrder()); + assertTrue(sortSpec.get(0).getSorter() instanceof Sorting.UcaSorter); + assertEquals(((Sorting.UcaSorter)sortSpec.get(0).getSorter()).getLocale(), "en_US" ); + assertEquals(((Sorting.UcaSorter)sortSpec.get(0).getSorter()).getStrength(), Sorting.UcaSorter.Strength.TERTIARY ); + assertEquals("name", sortSpec.get(0).getFieldName()); + } + + /** + * Check query hash function. + * Extremely simple for now, will be used for pathological cases. + */ + @Test + public void testHashCode() { + Query p = newQuery("?query=foo&type=any"); + Query q = newQuery("?query=foo&type=all"); + assertTrue(p.hashCode() != q.hashCode()); + } + + /** Test using the defauultindex feature */ + @Test + public void testDefaultIndex() { + Query q = newQuery("?query=hi hello keyword:kanoo " + + "default:munkz \"phrases too\"&default-index=def"); + assertEquals("AND def:hi def:hello keyword:kanoo " + + "default:munkz def:\"phrases too\"", + q.getModel().getQueryTree().getRoot().toString()); + } + + /** Test that GET parameter names are case in-sensitive */ + @Test + public void testGETParametersCase() { + Query q = newQuery("?QUERY=testing&hits=10&oFfSeT=10"); + + assertEquals("testing", q.getModel().getQueryString()); + assertEquals(10, q.getHits()); + assertEquals(10, q.getOffset()); + } + + + @Test + public void testNegativeHitValue() { + assertQueryError( + "?query=test&hits=-1", + containsString("Could not set 'hits' to '-1': Must be a positive number")); + } + + @Test + public void testNaNHitValue() { + assertQueryError( + "?query=test&hits=NaN", + containsString("Could not set 'hits' to 'NaN': Not a valid integer")); + } + + @Test + public void testNoneHitValue() { + assertQueryError( + "?query=test&hits=(none)", + containsString("Could not set 'hits' to '(none)': Not a valid integer")); + } + + @Test + public void testNegativeOffsetValue() { + assertQueryError( + "?query=test&offset=-1", + containsString("Could not set 'offset' to '-1': Must be a positive number")); + } + + @Test + public void testNaNOffsetValue() { + assertQueryError( + "?query=test&offset=NaN", + containsString("Could not set 'offset' to 'NaN': Not a valid integer")); + } + + @Test + public void testNoneOffsetValue() { + assertQueryError( + "?query=test&offset=(none)", + containsString("Could not set 'offset' to '(none)': Not a valid integer")); + } + + @Test + public void testNoneHitsNegativeOffsetValue() { + assertQueryError( + "?query=test&hits=(none)&offset=-10", + anyOf( + containsString("Could not set 'offset' to '-10': Must be a positive number"), + containsString("Could not set 'hits' to '(none)': Not a valid integer"))); + } + + @Test + public void testFeedbackIsTransferredToResult() { + assertQueryError( + "?query=test&hits=(none)&offset=-10", + anyOf( + containsString("Could not set 'hits' to '(none)': Not a valid integer"), + containsString("Could not set 'offset' to '-10': Must be a positive number"))); + } + + @Test + public void testUnicodeNormalization() { + Linguistics linguistics = new SimpleLinguistics(); + Query query = newQueryFromEncoded("?query=content:%EF%BC%B3%EF%BC%AF%EF%BC%AE%EF%BC%B9", Language.ENGLISH, + linguistics); + assertEquals("SONY",((WordItem) query.getModel().getQueryTree().getRoot()).getWord()); + + query = newQueryFromEncoded("?query=foo&filter=+%EF%BC%B3%EF%BC%AF%EF%BC%AE%EF%BC%B9", Language.ENGLISH, + linguistics); + assertEquals("RANK foo |SONY", query.getModel().getQueryTree().getRoot().toString()); + + query = newQueryFromEncoded("?query=foo+AND+%EF%BC%B3%EF%BC%AF%EF%BC%AE%EF%BC%B9)&type=adv", + Language.ENGLISH, linguistics); + assertEquals("AND foo SONY", query.getModel().getQueryTree().getRoot().toString()); + } + + /** Test a vertical specific patch, see Tokenizer */ + @Test + @Ignore + public void testPrivateUseCharacterParsing() { + Query query=newQuery("?query=%EF%89%AB"); + assertEquals(Character.UnicodeBlock.PRIVATE_USE_AREA, + Character.UnicodeBlock.of(query.getModel().getQueryTree().getRoot().toString().charAt(0))); + } + + /** Test a vertical specific patch, see Tokenizer */ + @Test + @Ignore + public void testOtherharactersParsing() { + Query query=newQuery(com.yahoo.search.test.QueryTestCase.httpEncode("?query=\u3007\u2e80\u2eff\u2ed0")); + assertEquals(Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION, + Character.UnicodeBlock.of(query.getModel().getQueryTree().getRoot().toString().charAt(0))); + assertEquals(Character.UnicodeBlock.CJK_RADICALS_SUPPLEMENT, + Character.UnicodeBlock.of(query.getModel().getQueryTree().getRoot().toString().charAt(1))); + assertEquals(Character.UnicodeBlock.CJK_RADICALS_SUPPLEMENT, + Character.UnicodeBlock.of(query.getModel().getQueryTree().getRoot().toString().charAt(2))); + assertEquals(Character.UnicodeBlock.CJK_RADICALS_SUPPLEMENT, + Character.UnicodeBlock.of(query.getModel().getQueryTree().getRoot().toString().charAt(3))); + } + + @Test + public void testFreshness() { + Query query = newQuery("?query=test&datetime=103"); + assertTrue(query.getRanking().getFreshness().getRefTime()==103); + query.getRanking().setFreshness("193"); + + assertTrue(query.getRanking().getFreshness().getRefTime()==193); + query.getRanking().setFreshness("now"); + + assertTrue(query.getRanking().getFreshness().getSystemTimeInSecondsSinceEpoch() >= query.getRanking().getFreshness().getRefTime()); + int presize= query.errors().size(); + query.getRanking().setFreshness("sometimeslater"); + + int postsize = query.errors().size(); + assertTrue(postsize > presize); + } + + @Test + public void testCopy() { + Query qs = newQuery("?query=test&rankfeature.something=2"); + assertEquals("test",qs.getModel().getQueryTree().toString()); + assertEquals((int)qs.properties().getInteger("rankfeature.something"),2); + Query qp = new Query(qs); + assertEquals("test", qp.getModel().getQueryTree().getRoot().toString()); + assertFalse(qp.getRanking().getFeatures().isEmpty()); + assertEquals("2", qp.getRanking().getFeatures().get("something")); + } + + private Query newQuery(String queryString) { + return newQuery(queryString, null, new SimpleLinguistics()); + } + + private Query newQuery(String queryString, Language language, Linguistics linguistics) { + return newQueryFromEncoded(com.yahoo.search.test.QueryTestCase.httpEncode(queryString), language, linguistics); + } + + private Query newQueryFromEncoded(String encodedQueryString, Language language, Linguistics linguistics) { + Query query = new Query(encodedQueryString); + query.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, linguistics))); + query.getModel().setLanguage(language); + return query; + } + + private void assertQueryError(final String queryString, final Matcher expectedErrorMessage) { + try { + newQuery(queryString); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat(Exceptions.toMessageString(e), expectedErrorMessage); + } + } + +} + diff --git a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java new file mode 100644 index 00000000000..16d1b92260f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.templates.test.TilingTestCase; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests that rank features are rendered when requested in the query + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class RankFeatureDumpTestCase extends junit.framework.TestCase { + + private static final String rankFeatureString= + "{\"match.weight.as1\":10,\"attribute(ai1)\":1.000000,\"proximity(as1, 1, 2)\":2.000000}"; + + public void test() throws IOException { + Query query=new Query("?query=five&rankfeatures"); + assertTrue(query.getRanking().getListFeatures()); // New api + Result result = doSearch(new MockBackend(), query, 0,10); + assertTrue(TilingTestCase.getRendered(result).contains( + "" + rankFeatureString + "")); + } + + private static class MockBackend extends Searcher { + + @Override + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result=new Result(query); + Hit hit=new FastHit("test",1000); + hit.setField(com.yahoo.search.result.Hit.RANKFEATURES_FIELD,rankFeatureString); + result.hits().add(hit); + return result; + } + + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/ResultTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/ResultTestCase.java new file mode 100644 index 00000000000..01842a15ab2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/ResultTestCase.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test; + +import java.util.Iterator; + +import com.yahoo.search.Query; +import com.yahoo.search.result.Hit; +import com.yahoo.search.Result; + +/** + * Tests the result class + * + * @author bratseth + */ +public class ResultTestCase extends junit.framework.TestCase { + + public ResultTestCase (String name) { + super(name); + + } + + public void testHitOrdering() { + Result result=new Result(new Query("dummy")); + result.hits().add(new Hit("test:hit1",80)); + result.hits().add(new Hit("test:hit2",90)); + result.hits().add(new Hit("test:hit3",70)); + + Iterator hits= result.hits().deepIterator(); + assertEquals(new Hit("test:hit2",90),hits.next()); + assertEquals(new Hit("test:hit1",80),hits.next()); + assertEquals(new Hit("test:hit3",70),hits.next()); + } + + private void resultInit(Result result){ + result.hits().add(new Hit("test:hit1",80)); + result.hits().add(new Hit("test:hit2",90)); + result.hits().add(new Hit("test:hit3",70)); + result.hits().add(new Hit("test:hit4",40)); + result.hits().add(new Hit("test:hit5",50)); + result.hits().add(new Hit("test:hit6",20)); + result.hits().add(new Hit("test:hit7",20)); + result.hits().add(new Hit("test:hit8",55)); + result.hits().add(new Hit("test:hit9",75)); + } + + public void testHitTrimming(){ + Result result=new Result(new Query("dummy")); + + //case 1: keep some hits in the middle + resultInit(result); + result.hits().trim(3, 3); + assertEquals(3,result.getHitCount()); + Iterator hits= result.hits().deepIterator(); + assertEquals(new Hit("test:hit3",70),hits.next()); + assertEquals(new Hit("test:hit8",55),hits.next()); + assertEquals(new Hit("test:hit5",50),hits.next()); + assertEquals(false,hits.hasNext()); + + //case 2: keep some hits at the end + result=new Result(new Query("dummy")); + resultInit(result); + result.hits().trim(5, 4); + hits= result.hits().deepIterator(); + assertEquals(new Hit("test:hit5",50),hits.next()); + assertEquals(new Hit("test:hit4",40),hits.next()); + assertEquals(new Hit("test:hit6",20),hits.next()); + assertEquals(new Hit("test:hit7",20),hits.next()); + assertEquals(false,hits.hasNext()); + + + //case 3: keep some hits at the beginning + result=new Result(new Query("dummy")); + resultInit(result); + result.hits().trim(0, 4); + hits= result.hits().deepIterator(); + assertEquals(new Hit("test:hit2",90),hits.next()); + assertEquals(new Hit("test:hit1",80),hits.next()); + assertEquals(new Hit("test:hit9",75),hits.next()); + assertEquals(new Hit("test:hit3",70),hits.next()); + assertEquals(false,hits.hasNext()); + + //case 4: keep no hits + result=new Result(new Query("dummy")); + resultInit(result); + result.hits().trim(10, 4); + hits= result.hits().deepIterator(); + assertEquals(false,hits.hasNext()); + } + + + //This test is broken + /* + public void testNavigationalLinks() { + Query query = new Query("/abc?query=dummy&def=ghi"); + Result result=new Result(query); + result.setTotalHitCount(500); + result.add(new Hit("test:hit1",80,1,null,true)); + assertEquals("/abc?query=dummy&def=ghi&offset=1", + result.getNextResultURL()); + assertEquals("/abc?query=dummy&def=ghi&offset=0", + result.getPreviousResultURL()); + }*/ + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/fieldtypes/field-info.cfg b/container-search/src/test/java/com/yahoo/prelude/test/fieldtypes/field-info.cfg new file mode 100644 index 00000000000..9c220817e03 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/fieldtypes/field-info.cfg @@ -0,0 +1,21 @@ +doctype[1] +doctype[0].name foobar +doctype[0].field[2] + +doctype[0].field[0].name foo +doctype[0].field[0].command[1] +doctype[0].field[0].command[0] bold +doctype[0].field[0].index[2] +doctype[0].field[0].index[0] default +doctype[0].field[0].index[1] bigteaser +doctype[0].field[0].type long + +doctype[0].field[1].name bar +doctype[0].field[1].command[1] +doctype[0].field[1].command[0] bold +doctype[0].field[1].index[2] +doctype[0].field[1].index[0] default +doctype[0].field[1].index[1] otherteaser +doctype[0].field[1].type string + +field[0] diff --git a/container-search/src/test/java/com/yahoo/prelude/test/index-info.cfg b/container-search/src/test/java/com/yahoo/prelude/test/index-info.cfg new file mode 100644 index 00000000000..7cf14f74d9a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/index-info.cfg @@ -0,0 +1,14 @@ +indexinfo[2] +indexinfo[0].name one +indexinfo[0].command[2] +indexinfo[0].command[0].indexname exactemento +indexinfo[0].command[0].command compact-to-term +indexinfo[0].command[1].indexname default +indexinfo[0].command[1].command stem +indexinfo[1].name two +indexinfo[1].command[2] +indexinfo[1].command[0].indexname default +indexinfo[1].command[0].command compact-to-term +indexinfo[1].command[1].indexname b +indexinfo[1].command[1].command compact-to-term + diff --git a/container-search/src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg b/container-search/src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg new file mode 100644 index 00000000000..c5bd1fcd883 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/indexfactstesting.cfg @@ -0,0 +1,29 @@ +indexinfo[3] +indexinfo[0].name one +indexinfo[0].command[1] +indexinfo[0].command[0].indexname a +indexinfo[0].command[0].command index +indexinfo[1].name two +indexinfo[1].command[3] +indexinfo[1].command[0].indexname d +indexinfo[1].command[0].command index +indexinfo[1].command[1].indexname d +indexinfo[1].command[1].command fullurl +indexinfo[1].command[2].indexname e +indexinfo[1].command[2].command exact kj(/& +indexinfo[2].name three +indexinfo[2].command[7] +indexinfo[2].command[0].indexname a +indexinfo[2].command[0].command index +indexinfo[2].command[1].indexname a +indexinfo[2].command[1].command stem +indexinfo[2].command[2].indexname b +indexinfo[2].command[2].command index +indexinfo[2].command[3].indexname c +indexinfo[2].command[3].command index +indexinfo[2].command[4].indexname c +indexinfo[2].command[4].command urlhost +indexinfo[2].command[5].indexname d +indexinfo[2].command[5].command exact +indexinfo[2].command[6].indexname twewm +indexinfo[2].command[6].command word diff --git a/container-search/src/test/java/com/yahoo/prelude/test/integration/FirstSearcher.java b/container-search/src/test/java/com/yahoo/prelude/test/integration/FirstSearcher.java new file mode 100644 index 00000000000..bd40bf29f40 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/integration/FirstSearcher.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test.integration; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +public class FirstSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("searcher:1",995)); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/integration/SecondSearcher.java b/container-search/src/test/java/com/yahoo/prelude/test/integration/SecondSearcher.java new file mode 100644 index 00000000000..d9f9d5be76e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/integration/SecondSearcher.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test.integration; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +public class SecondSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("searcher:2",996)); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/integration/ThirdSearcher.java b/container-search/src/test/java/com/yahoo/prelude/test/integration/ThirdSearcher.java new file mode 100644 index 00000000000..2599619bb27 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/integration/ThirdSearcher.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.test.integration; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +public class ThirdSearcher extends Searcher { + public Result search(com.yahoo.search.Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("searcher:3",997)); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/integration/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/prelude/test/integration/qr-searchers.cfg new file mode 100644 index 00000000000..2825585627c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/integration/qr-searchers.cfg @@ -0,0 +1,8 @@ +customizedsearchers.rawquery[3] +customizedsearchers.rawquery[0] "com.yahoo.prelude.test.integration.FirstSearcher" +customizedsearchers.rawquery[1] "com.yahoo.prelude.test.integration.SecondSearcher" +customizedsearchers.rawquery[2] "com.yahoo.prelude.test.integration.ThirdSearcher" + + + + diff --git a/container-search/src/test/java/com/yahoo/prelude/test/integration/qr.cfg b/container-search/src/test/java/com/yahoo/prelude/test/integration/qr.cfg new file mode 100644 index 00000000000..33ac59c0d7e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/integration/qr.cfg @@ -0,0 +1,4 @@ +port.search 18081 +port.stats 18085 +maxthreads 200 +requestbuffersize 65536 diff --git a/container-search/src/test/java/com/yahoo/prelude/test/qr-fileserver.cfg b/container-search/src/test/java/com/yahoo/prelude/test/qr-fileserver.cfg new file mode 100644 index 00000000000..e43c9d7f004 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/qr-fileserver.cfg @@ -0,0 +1 @@ +rootdir "" diff --git a/container-search/src/test/java/com/yahoo/prelude/test/qr-logging.cfg b/container-search/src/test/java/com/yahoo/prelude/test/qr-logging.cfg new file mode 100644 index 00000000000..0847de467fe --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/qr-logging.cfg @@ -0,0 +1,44 @@ +logger com.yahoo +speciallog[6] +speciallog[0].name QueryAccessLog +speciallog[0].type file +speciallog[0].filehandler.name QueryAccessLog +speciallog[0].filehandler.pattern ./QueryAccessLog.%Y%m%d%H%M%S +speciallog[0].filehandler.rotation 0 60 ... +speciallog[0].cachehandler.name QueryAccessLog +speciallog[0].cachehandler.size 1000 +speciallog[1].name QueryResultLog +speciallog[1].type cache +speciallog[1].filehandler.name QueryResultLog +speciallog[1].filehandler.pattern ./QueryResultLog.%Y%m%d%H%M%S +speciallog[1].filehandler.rotation 0 60 ... +speciallog[1].cachehandler.name QueryResultLog +speciallog[1].cachehandler.size 1000 +speciallog[2].name ResultImpressionLog +speciallog[2].type file +speciallog[2].filehandler.name ResultImpressionLog +speciallog[2].filehandler.pattern ./ResultImpressionLog.%Y%m%d%H%M%S +speciallog[2].filehandler.rotation 0 60 ... +speciallog[2].cachehandler.name ResultImpressionLog +speciallog[2].cachehandler.size 1000 +speciallog[3].name ServiceEventLog +speciallog[3].type cache +speciallog[3].filehandler.name ServiceEventLog +speciallog[3].filehandler.pattern ./ServiceEventLog.%Y%m%d%H%M%S +speciallog[3].filehandler.rotation 0 60 ... +speciallog[3].cachehandler.name ServiceEventLog +speciallog[3].cachehandler.size 1000 +speciallog[4].name ServiceStatusLog +speciallog[4].type off +speciallog[4].filehandler.name ServiceStatusLog +speciallog[4].filehandler.pattern ./ServiceStatusLog.%Y%m%d%H%M%S +speciallog[4].filehandler.rotation 0 60 ... +speciallog[4].cachehandler.name ServiceStatusLog +speciallog[4].cachehandler.size 1000 +speciallog[5].name ServiceTraceLog +speciallog[5].type parent +speciallog[5].filehandler.name ServiceTraceLog +speciallog[5].filehandler.pattern ./ServiceTraceLog.%Y%m%d%H%M%S +speciallog[5].filehandler.rotation 0 60 ... +speciallog[5].cachehandler.name ServiceTraceLog +speciallog[5].cachehandler.size 1000 diff --git a/container-search/src/test/java/com/yahoo/prelude/test/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/prelude/test/qr-searchers.cfg new file mode 100644 index 00000000000..500d2b12f1f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/qr-searchers.cfg @@ -0,0 +1,14 @@ + +customizedsearchers.rawquery[0] +customizedsearchers.transformedquery[0] +customizedsearchers.blendedresult[0] +customizedsearchers.unblendedresult[0] +customizedsearchers.backend[0] +customizedsearchers.argument[0] + +searchcluster[1] +searchcluster[0].searchdef[1] +searchcluster[0].searchdef[0] music +searchcluster[0].dispatcher[1] +searchcluster[0].dispatcher[0].host localhost +searchcluster[0].dispatcher[0].port 6328 diff --git a/container-search/src/test/java/com/yahoo/prelude/test/qr-summary.cfg b/container-search/src/test/java/com/yahoo/prelude/test/qr-summary.cfg new file mode 100644 index 00000000000..2701dc0fe64 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/qr-summary.cfg @@ -0,0 +1,69 @@ +idtype INTEGER +classes[1] +classes[0].name music +classes[0].id 2115365230 +classes[0].fields[32] +classes[0].fields[0].name title +classes[0].fields[0].type string +classes[0].fields[1].name artist +classes[0].fields[1].type string +classes[0].fields[2].name song +classes[0].fields[2].type string +classes[0].fields[3].name bgndata +classes[0].fields[3].type string +classes[0].fields[4].name sales +classes[0].fields[4].type integer +classes[0].fields[5].name pto +classes[0].fields[5].type integer +classes[0].fields[6].name mid +classes[0].fields[6].type integer +classes[0].fields[7].name ew +classes[0].fields[7].type string +classes[0].fields[8].name surl +classes[0].fields[8].type string +classes[0].fields[9].name userrate +classes[0].fields[9].type integer +classes[0].fields[10].name pid +classes[0].fields[10].type string +classes[0].fields[11].name weight +classes[0].fields[11].type integer +classes[0].fields[12].name url +classes[0].fields[12].type string +classes[0].fields[13].name isbn +classes[0].fields[13].type string +classes[0].fields[14].name fmt +classes[0].fields[14].type string +classes[0].fields[15].name albumid +classes[0].fields[15].type string +classes[0].fields[16].name disp_song +classes[0].fields[16].type string +classes[0].fields[17].name pfrom +classes[0].fields[17].type integer +classes[0].fields[18].name bgnpfrom +classes[0].fields[18].type integer +classes[0].fields[19].name categories +classes[0].fields[19].type string +classes[0].fields[20].name data +classes[0].fields[20].type string +classes[0].fields[21].name numreview +classes[0].fields[21].type integer +classes[0].fields[22].name bgnsellers +classes[0].fields[22].type integer +classes[0].fields[23].name image +classes[0].fields[23].type string +classes[0].fields[24].name artistspid +classes[0].fields[24].type string +classes[0].fields[25].name newestedition +classes[0].fields[25].type integer +classes[0].fields[26].name bgnpto +classes[0].fields[26].type string +classes[0].fields[27].name year +classes[0].fields[27].type integer +classes[0].fields[28].name did +classes[0].fields[28].type integer +classes[0].fields[29].name scorekey +classes[0].fields[29].type integer +classes[0].fields[30].name cbid +classes[0].fields[30].type integer +classes[0].fields[31].name ranklog +classes[0].fields[31].type data diff --git a/container-search/src/test/java/com/yahoo/prelude/test/qr.cfg b/container-search/src/test/java/com/yahoo/prelude/test/qr.cfg new file mode 100644 index 00000000000..2b547e3b703 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/qr.cfg @@ -0,0 +1 @@ +# The 5.1 ConfigGetter needs this to be here \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/test/specialtokens.cfg b/container-search/src/test/java/com/yahoo/prelude/test/specialtokens.cfg new file mode 100644 index 00000000000..c9a6a01d904 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/test/specialtokens.cfg @@ -0,0 +1,14 @@ +tokenlist[2] +tokenlist[0].name default +tokenlist[0].tokens[5] +tokenlist[0].tokens[0].token .... +tokenlist[0].tokens[1].token c++ +tokenlist[0].tokens[2].token b.s.d. +tokenlist[0].tokens[3].token with space +tokenlist[0].tokens[4].token c# +tokenlist[1].name other +tokenlist[1].tokens[4] +tokenlist[1].tokens[0].token [huh] +tokenlist[1].tokens[1].token &&&%%% +tokenlist[1].tokens[2].token ------ +tokenlist[1].tokens[3].token !!!*** diff --git a/container-search/src/test/java/com/yahoo/prelude/test/statistics.cfg b/container-search/src/test/java/com/yahoo/prelude/test/statistics.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java b/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java new file mode 100644 index 00000000000..6c3f1eba4c0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search; + +import com.yahoo.text.Utf8; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * As the name implies, a stupid, single-threaded bad-excuse-for-HTTP server. + * + * @author Einar M R Rosenvinge + */ +public class StupidSingleThreadedHttpServer implements Runnable { + + private static final Logger log = Logger.getLogger(StupidSingleThreadedHttpServer.class.getName()); + + private final ServerSocket serverSocket; + private final int delaySeconds; + private Thread serverThread = null; + private CompletableFuture requestFuture = new CompletableFuture<>(); + private final Pattern contentLengthPattern = Pattern.compile("content-length: (\\d+)", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE); + + public StupidSingleThreadedHttpServer() throws IOException { + this(0, 0); + } + + public StupidSingleThreadedHttpServer(int port, int delaySeconds) throws IOException { + this.delaySeconds = delaySeconds; + this.serverSocket = new ServerSocket(port); + } + + public void start() { + serverThread = new Thread(this); + serverThread.setDaemon(true); + serverThread.start(); + } + + public void run() { + try { + while(true) { + Socket socket = serverSocket.accept(); + StringBuilder request = new StringBuilder(); + socket.setSoLinger(true, 60); + BufferedReader in = new BufferedReader( + new InputStreamReader( + socket.getInputStream())); + + int contentLength = -1; + String inputLine; + while (!"".equals(inputLine = in.readLine())) { //read header: + request.append(inputLine).append("\r\n"); + if (inputLine.toLowerCase(Locale.US).contains("content-length")) { + Matcher contentLengthMatcher = contentLengthPattern.matcher(inputLine); + if (contentLengthMatcher.matches()) { + contentLength = Integer.parseInt(contentLengthMatcher.group(1)); + } + } + } + request.append("\r\n"); + + if (contentLength < 0) { + System.err.println("WARNING! Got no Content-Length header!!"); + } else { + char[] requestBody = new char[contentLength]; + int readRemaining = contentLength; + + do { + int read = in.read(requestBody, (contentLength - readRemaining), readRemaining); + if (read < 0) { + throw new IllegalStateException("Should not get EOF here!!"); + } + readRemaining -= read; + } while (readRemaining > 0); + + request.append(new String(requestBody)); + } + + // Simulate service slowness + if (delaySeconds > 0) { + try { + System.out.println(this.getClass().getCanonicalName() + " sleeping in " + delaySeconds + " s before responding..."); + Thread.sleep((long) (delaySeconds * 1000)); + System.out.println("done sleeping, responding"); + } catch (InterruptedException e) { + //ignore + } + } + + socket.getOutputStream().write(getResponse(request.toString())); + socket.getOutputStream().flush(); + in.close(); + socket.close(); + + boolean wasCompleted = requestFuture.complete(request.toString()); + if (!wasCompleted) { + log.log(Level.INFO, "Only the first request will be stored, ignoring. " + + "Old value: " + requestFuture.get() + + ", New value: " + request.toString()); + } + } + } catch (SocketException se) { + if ("Socket closed".equals(se.getMessage())) { + //ignore + } else { + throw new RuntimeException(se); + } + } catch (IOException|InterruptedException|ExecutionException e) { + throw new RuntimeException(e); + } + } + + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 0\r\n" + + "\r\n"); + } + + protected byte[] getResponseBody() { + return new byte[0]; + } + + public void stop() { + if (!serverSocket.isClosed()) { + try { + serverSocket.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + try { + serverThread.interrupt(); + } catch (Exception e) { + //ignore + } + } + + public int getServerPort() { + return serverSocket.getLocalPort(); + } + + public String getRequest() { + try { + return requestFuture.get(1, TimeUnit.MINUTES); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new AssertionError("Failed waiting for request. ", e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java new file mode 100644 index 00000000000..01392e900d8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java @@ -0,0 +1,169 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.cluster.test; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.cluster.ClusterSearcher; +import com.yahoo.search.cluster.Hasher; +import com.yahoo.search.cluster.PingableSearcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +// TODO: Author! +public class ClusterSearcherTestCase extends TestCase { + + + class TestingBackendSearcher extends PingableSearcher { + + Hit hit; + + public TestingBackendSearcher(Hit hit) { + this.hit = hit; + } + + public @Override Result search(Query query,Execution execution) { + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + } + + class BlockingBackendSearcher extends TestingBackendSearcher { + + private boolean blocking = false; + + public BlockingBackendSearcher(Hit hit) { + super(hit); + } + + public Result search(Query query,Execution execution) { + Result result = super.search(query,execution); + if(blocking) { + result.hits().addError(ErrorMessage.createUnspecifiedError("Dummy error")); + } + return result; + } + + @Override + public Pong ping(Ping ping, Execution execution) { + //Sleep an hour + Pong pong = new Pong(); + if (isBlocking()) { + + pong.addError(ErrorMessage.createTimeout("Dummy timeout")); + } + return new Pong(); + } + + public boolean isBlocking() { + return blocking; + } + + public void setBlocking(boolean blocking) { + this.blocking = blocking; + } + } + + class SimpleQuery extends Query { + int hashValue; + public SimpleQuery(int hashValue) { + this.hashValue = hashValue; + } + + @Override + public int hashCode() { + return hashValue; + } + } + + class SimpleHasher extends Hasher { + + + class SimpleNodeList extends NodeList { + public SimpleNodeList() { + super(null); + } + + public T select(int code, int trynum) { + return objects.get(code + trynum % objects.size()); + } + + public int getNodeCount() { + return objects.size(); + } + } + + List objects = new ArrayList<>(); + + @Override + public synchronized void remove(T node) { + objects.remove(node); + } + + @Override + public synchronized void add(T node) { + objects.add(node); + } + + @Override + public NodeList getNodes() { + return new SimpleNodeList(); + + } + } + + /** A cluster searcher which clusters over a set of alternative searchers (search chains would be more realistic) */ + static class SearcherClusterSearcher extends ClusterSearcher { + + public SearcherClusterSearcher(ComponentId id,List searchers,Hasher hasher) { + super(id,searchers,hasher,false); + } + + public @Override Result search(Query query,Execution execution,Searcher searcher) { + return searcher.search(query,execution); + } + + public @Override void fill(Result result,String summaryName,Execution execution,Searcher searcher) { + searcher.fill(result,summaryName,execution); + } + + public @Override Pong ping(Ping ping,Searcher searcher) { + return new Execution(searcher, Execution.Context.createContextStub()).ping(ping); + } + + } + + + public void testSimple() { + Hit blockingHit = new Hit("blocking"); + Hit nonblockingHit = new Hit("nonblocking"); + BlockingBackendSearcher blockingSearcher = new BlockingBackendSearcher(blockingHit); + List searchers=new ArrayList<>(); + searchers.add(blockingSearcher); + searchers.add(new TestingBackendSearcher(nonblockingHit)); + ClusterSearcher provider = new SearcherClusterSearcher(new ComponentId("simple"),searchers,new SimpleHasher<>()); + + Result blockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals(blockingHit,blockingResult.hits().get(0)); + Result nonblockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals(nonblockingHit,nonblockingResult.hits().get(0)); + + blockingSearcher.setBlocking(true); + + blockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals(blockingResult.hits().get(0),nonblockingHit); + nonblockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals(nonblockingResult.hits().get(0),nonblockingHit); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java new file mode 100644 index 00000000000..06686e8777a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -0,0 +1,198 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.cluster.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterSearcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; + +/** + * @author bratseth + */ +public class ClusteredConnectionTestCase extends junit.framework.TestCase { + + public void testClustering() { + Connection connection0=new Connection("0"); + Connection connection1=new Connection("1"); + Connection connection2=new Connection("2"); + List connections=new ArrayList<>(); + connections.add(connection0); + connections.add(connection1); + connections.add(connection2); + MyBackend myBackend=new MyBackend(new ComponentId("test"),connections); + + Result r; + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:2",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:1",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection2.setInService(false); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:1",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection1.setInService(false); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection0.setInService(false); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("Failed calling connection '2' in searcher 'test' for query 'NULL': Connection failed", + r.hits().getError().getDetailedMessage()); + + connection0.setInService(true); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection1.setInService(true); + connection2.setInService(true); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:2",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:1",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + } + + public void testClusteringWithPing() { + Connection connection0=new Connection("0"); + Connection connection1=new Connection("1"); + Connection connection2=new Connection("2"); + List connections=new ArrayList<>(); + connections.add(connection0); + connections.add(connection1); + connections.add(connection2); + MyBackend myBackend=new MyBackend(new ComponentId("test"),connections); + + Result r; + + // Note that we cannot make any successful queries here or we have to wait 10 seconds for + // the traffic monitor to agree that these nodes are really not responding + + connection2.setInService(false); + connection1.setInService(false); + connection0.setInService(false); + forcePing(myBackend); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("No backends in service. Try later",r.hits().getError().getMessage()); + + connection2.setInService(true); + connection1.setInService(true); + connection0.setInService(true); + forcePing(myBackend); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertNull(r.hits().getError()); + } + + private void forcePing(MyBackend myBackend) { + myBackend.getMonitor().ping(Executors.newCachedThreadPool(new DaemonThreadFactory())); + Thread.yield(); + } + + /** Represents a connection, e.g over http, in this test */ + private static class Connection { + + private String id; + + private boolean inService=true; + + public Connection(String id) { + this.id=id; + } + + /** This is used for both fill, pings and queries */ + public String getResponse() { + if (!inService) throw new RuntimeException("Connection failed"); + return id; + } + + public void setInService(boolean inservice) { + this.inService=inservice; + } + + public String toString() { + return "connection '" + id + "'"; + } + + } + + /** + * This is the kind of searcher which will be implemented by those who wish to create a searcher which is a + * client to a clustered service. + * The goal is to make writing this correctly as simple as possible. + */ + private static class MyBackend extends ClusterSearcher { + + public MyBackend(ComponentId componentId, List connections) { + super(componentId,connections,false); + } + + public @Override Result search(Query query,Execution execution,Connection connection) { + Result result=new Result(query); + result.hits().add(new Hit("from:" + connection.getResponse())); + return result; + } + + public @Override void fill(Result result,String summary,Execution execution,Connection connection) { + result.hits().get(0).fields().put("filled",connection.getResponse()); + } + + public @Override Pong ping(Ping ping,Connection connection) { + Pong pong=new Pong(); + if (connection.getResponse()==null) + pong.addError(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); + return pong; + } + + } + + /** A query with a predictable hash function */ + private static class SimpleQuery extends Query { + + int hashValue; + + public SimpleQuery(int hashValue) { + this.hashValue = hashValue; + } + + public @Override int hashCode() { + return hashValue; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java b/container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java new file mode 100644 index 00000000000..e952300e0ed --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.debug.test; + +import junit.framework.TestCase; + +import com.yahoo.search.debug.SearchChainTextRepresentation; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.test.SimpleSearchChain; + +/** + * Test of SearchChainTextRepresentation. + * @author tonytv + */ +public class SearchChainTextRepresentationTestCase extends TestCase { + + public void testTextRepresentation() { + SearchChainTextRepresentation textRepresentation = + new SearchChainTextRepresentation(SimpleSearchChain.orderedChain, new SearchChainRegistry()); + + String[] expected = { + "test [Searchchain] {", + " one [Searcher] {", + " Reason for forwarding to this search chain.", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " }", + " two [Searcher] {", + " Reason for forwarding to this search chain.", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " }", + "}" + }; + + String[] result = textRepresentation.toString().split("\n"); + assertEquals(expected.length, result.length); + + int i = 0; + for (String line : textRepresentation.toString().split("\n")) + assertEquals(expected[i++], line); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java new file mode 100644 index 00000000000..a88ef7e5e37 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.compress.CompressionType; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; + + +/** + * Tests using a dispatcher to fill a result + * + * @author bratseth + */ +public class FillTestCase { + + private MockClient client = new MockClient(); + + @Test + public void testFilling() { + Map nodes = new HashMap<>(); + nodes.put(0, client.createConnection("host0", 123)); + nodes.put(1, client.createConnection("host1", 123)); + nodes.put(2, client.createConnection("host2", 123)); + Dispatcher dispatcher = new Dispatcher(nodes, client); + + Query query = new Query(); + Result result = new Result(query); + result.hits().add(createHit(0, 0)); + result.hits().add(createHit(2, 1)); + result.hits().add(createHit(1, 2)); + result.hits().add(createHit(2, 3)); + result.hits().add(createHit(0, 4)); + + client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0)); + client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1)); + client.setDocsumReponse("host1", 2, "summaryClass1", map("field1", "s.1.2", "field2", 2)); + client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); + client.setDocsumReponse("host0", 4, "summaryClass1", map("field1", "s.0.4", "field2", 4)); + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + + assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); + assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); + assertEquals("s.1.2", result.hits().get("hit:2").getField("field1").toString()); + assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString()); + assertEquals("s.0.4", result.hits().get("hit:4").getField("field1").toString()); + assertEquals(0L, result.hits().get("hit:0").getField("field2")); + assertEquals(1L, result.hits().get("hit:1").getField("field2")); + assertEquals(2L, result.hits().get("hit:2").getField("field2")); + assertEquals(3L, result.hits().get("hit:3").getField("field2")); + assertEquals(4L, result.hits().get("hit:4").getField("field2")); + } + + @Test + public void testErrorHandling() { + client.setMalfunctioning(true); + + Map nodes = new HashMap<>(); + nodes.put(0, client.createConnection("host0", 123)); + Dispatcher dispatcher = new Dispatcher(nodes, client); + + Query query = new Query(); + Result result = new Result(query); + result.hits().add(createHit(0, 0)); + + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + + assertEquals("Malfunctioning", result.hits().getError().getDetailedMessage()); + } + + private FastHit createHit(int sourceNodeId, int hitId) { + FastHit hit = new FastHit("hit:" + hitId, 1.0); + hit.setPartId(sourceNodeId, 0); + hit.setDistributionKey(sourceNodeId); + hit.setGlobalId(client.globalIdFrom(hitId)); + return hit; + } + + private Map map(String stringKey, String stringValue, String intKey, int intValue) { + Map map = new HashMap<>(); + map.put(stringKey, stringValue); + map.put(intKey, intValue); + return map; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java new file mode 100644 index 00000000000..2a7301652b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdIdString; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class MockClient implements Client { + + private final Map> docsums = new HashMap<>(); + private final Compressor compressor = new Compressor(); + private boolean malfunctioning = false; + + /** Set to true to cause this to produce an error instead of a regular response */ + public void setMalfunctioning(boolean malfunctioning) { this.malfunctioning = malfunctioning; } + + @Override + public NodeConnection createConnection(String hostname, int port) { + return new MockNodeConnection(hostname, port); + } + + @Override + public void getDocsums(List hitsContext, NodeConnection node, CompressionType compression, + int uncompressedSize, byte[] compressedSlime, Dispatcher.GetDocsumsResponseReceiver responseReceiver, + double timeoutSeconds) { + if (malfunctioning) { + responseReceiver.receive(GetDocsumsResponseOrError.fromError("Malfunctioning")); + return; + } + + Inspector request = BinaryFormat.decode(compressor.decompress(compressedSlime, compression, uncompressedSize)).get(); + String docsumClass = request.field("class").asString(); + List> docsumsToReturn = new ArrayList<>(); + request.field("gids").traverse((ArrayTraverser)(index, gid) -> { + GlobalId docId = new GlobalId(gid.asData()); + docsumsToReturn.add(docsums.get(new DocsumKey(node.toString(), docId, docsumClass))); + }); + Slime responseSlime = new Slime(); + Cursor root = responseSlime.setObject(); + Cursor docsums = root.setArray("docsums"); + for (Map docsumFields : docsumsToReturn) { + Cursor docsumItem = docsums.addObject(); + Cursor docsum = docsumItem.setObject("docsum"); + for (Map.Entry field : docsumFields.entrySet()) { + if (field.getValue() instanceof Integer) + docsum.setLong(field.getKey(), (Integer)field.getValue()); + else if (field.getValue() instanceof String) + docsum.setString(field.getKey(), (String)field.getValue()); + else + throw new RuntimeException(); + } + } + byte[] slimeBytes = BinaryFormat.encode(responseSlime); + Compressor.Compression compressionResult = compressor.compress(compression, slimeBytes); + GetDocsumsResponse response = new GetDocsumsResponse(compressionResult.type().getCode(), slimeBytes.length, + compressionResult.data(), hitsContext); + responseReceiver.receive(GetDocsumsResponseOrError.fromResponse(response)); + } + + public void setDocsumReponse(String nodeId, int docId, String docsumClass, Map docsumValues) { + docsums.put(new DocsumKey(nodeId, globalIdFrom(docId), docsumClass), docsumValues); + } + + public GlobalId globalIdFrom(int hitId) { + return new GlobalId(new IdIdString("", "test", "", String.valueOf(hitId))); + } + + private static class MockNodeConnection implements Client.NodeConnection { + + private final String hostname; + + public MockNodeConnection(String hostname, int port) { + this.hostname = hostname; + } + + @Override + public void close() { } + + @Override + public String toString() { return hostname; } + + } + + private static class DocsumKey { + + private final String internalKey; + + public DocsumKey(String nodeId, GlobalId docId, String docsumClass) { + internalKey = docsumClass + "." + nodeId + "." + docId; + } + + @Override + public int hashCode() { return internalKey.hashCode(); } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof DocsumKey)) return false; + return ((DocsumKey)other).internalKey.equals(this.internalKey); + } + + @Override + public String toString() { return internalKey; } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java b/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java new file mode 100644 index 00000000000..37969e12399 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation; + + +/** + * @author tonytv + */ +// TODO: Fix or remove! +public class FutureWaiterTest { + +/* + + @MockClass(realClass = System.class) + public static class MockSystem { + + private static long currentTime; + private static boolean firstTime; + + private static final long startTime = 123; + + @Mock + public static synchronized long currentTimeMillis() { + if (firstTime) { + firstTime = false; + return startTime; + } + return currentTime; + } + + static synchronized void setElapsedTime(long elapsedTime) { + firstTime = true; + currentTime = elapsedTime + startTime; + } + } + + @Mocked() + FutureResult result1; + + @Mocked() + FutureResult result2; + + @Mocked() + FutureResult result3; + + @Mocked() + FutureResult result4; + + @Before + public void before() { + Mockit.setUpMock(FutureWaiterTest.MockSystem.class); + } + + @After + public void after() { + Mockit.tearDownMocks(); + } + + @Test + public void require_time_to_wait_is_adjusted_for_elapsed_time() { + MockSystem.setElapsedTime(300); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 350); + futureWaiter.waitForFutures(); + + new FullVerifications() { + { + result1.get(350 - 300, TimeUnit.MILLISECONDS); + } + }; + } + + @Test + public void require_do_not_wait_for_expired_timeouts() { + MockSystem.setElapsedTime(300); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 300); + futureWaiter.add(result2, 290); + + futureWaiter.waitForFutures(); + + new FullVerifications() { + {} + }; + } + + @Test + public void require_wait_for_largest_timeout_first() throws InterruptedException { + MockSystem.setElapsedTime(600); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 500); + futureWaiter.add(result4, 800); + futureWaiter.add(result2, 600); + futureWaiter.add(result3, 700); + + futureWaiter.waitForFutures(); + + new FullVerifications() { + { + result4.get(800 - 600, TimeUnit.MILLISECONDS); + result3.get(700 - 600, TimeUnit.MILLISECONDS); + } + }; + } + + */ +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java new file mode 100644 index 00000000000..c707702a3d3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java @@ -0,0 +1,212 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Random; +import java.util.zip.GZIPOutputStream; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.message.BasicHeader; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.text.Utf8; + +/** + * Test GZip support for the HTTP integration introduced in 4.2. + * + * @author Steinar Knutsen + */ +public class GzipDecompressingEntityTestCase { + private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000"; + private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT); + GzipDecompressingEntity testEntity; + + private static final class MockEntity implements HttpEntity { + + private final InputStream inStream; + + MockEntity(InputStream is) { + inStream = is; + } + + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public boolean isChunked() { + return false; + } + + @Override + public long getContentLength() { + return -1; + } + + @Override + public Header getContentType() { + return new BasicHeader("Content-Type", "text/plain"); + } + + @Override + public Header getContentEncoding() { + return new BasicHeader("Content-Encoding", "gzip"); + } + + @Override + public InputStream getContent() throws IOException, + IllegalStateException { + return inStream; + } + + @Override + public void writeTo(OutputStream outstream) throws IOException { + } + + @Override + public boolean isStreaming() { + return false; + } + + @Override + public void consumeContent() throws IOException { + } + } + + @Before + public void setUp() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(CONTENT_AS_BYTES); + gzip.finish(); + gzip.close(); + byte[] compressed = out.toByteArray(); + InputStream inStream = new ByteArrayInputStream(compressed); + testEntity = new GzipDecompressingEntity(new MockEntity(inStream)); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetContentLength() throws UnknownHostException { + assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength()); + } + + @Test + public final void testGetContent() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length]; + int read = in.read(buffer); + assertEquals(CONTENT_AS_BYTES.length, read); + assertArrayEquals(CONTENT_AS_BYTES, buffer); + } + + @Test + public final void testGetContentToBigArray() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + in.read(buffer); + byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); + assertArrayEquals(expected, buffer); + } + + @Test + public final void testGetContentAvailable() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + assertEquals(CONTENT_AS_BYTES.length, in.available()); + } + + @Test + public final void testLargeZip() throws IOException { + byte [] input = new byte [10000000]; + Random random = new Random(89); + for (int i = 0; i < input.length; i++) { + input[i] = (byte) random.nextInt(); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(input); + gzip.finish(); + gzip.close(); + byte[] compressed = out.toByteArray(); + assertEquals(10003073, compressed.length); + InputStream inStream = new ByteArrayInputStream(compressed); + GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream)); + assertEquals(input.length, gunzipper.getContentLength()); + byte[] buffer = new byte[input.length]; + InputStream content = gunzipper.getContent(); + assertEquals(input.length, content.available()); + int read = content.read(buffer); + assertEquals(input.length, read); + assertArrayEquals(input, buffer); + } + + @Test + public final void testGetContentReadByte() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + int i = 0; + while (i < buffer.length) { + int r = in.read(); + if (r == -1) { + break; + } else { + buffer[i++] = (byte) r; + } + } + byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); + assertEquals(CONTENT_AS_BYTES.length, i); + assertArrayEquals(expected, buffer); + } + + @Test + public final void testGetContentReadWithOffset() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length); + assertEquals(CONTENT_AS_BYTES.length, read); + byte[] expected = new byte[CONTENT_AS_BYTES.length * 2]; + for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) { + expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i]; + } + assertArrayEquals(expected, buffer); + read = in.read(buffer, 0, CONTENT_AS_BYTES.length); + assertEquals(-1, read); + } + + @Test + public final void testGetContentSkip() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + final long n = 5L; + long skipped = in.skip(n); + assertEquals(n, skipped); + int read = in.read(); + assertEquals(CONTENT_AS_BYTES[(int) n], read); + skipped = in.skip(5000); + assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped); + assertEquals(-1L, in.skip(1L)); + } + + + @Test + public final void testWriteToOutputStream() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + testEntity.writeTo(out); + assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java new file mode 100644 index 00000000000..c3bd2ada260 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java @@ -0,0 +1,238 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.search.federation.ProviderConfig; +import org.junit.Test; + +import static com.yahoo.search.federation.ProviderConfig.Yca; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + * @author Steinar Knutsen + */ +public class HttpParametersTest { + + @Test + public void create_from_config() throws Exception { + ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder() + .connectionTimeout(1.0) + .maxConnectionPerRoute(2) + .maxConnections(3) + .path("myPath") + .readTimeout(4) + .socketBufferBytes(5) + .yca(new Yca.Builder() + .applicationId("myId") + .host("myYcaHost") + .port(7) + .retry(8) + .ttl(9) + .useProxy(true))); + + HTTPParameters httpParameters = new HTTPParameters(config); + + // Written to configuredConnectionTimeout, but it is not accessible!? + //assertThat(httpParameters.getConnectionTimeout(), is(1000)); + + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2)); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getMaxTotalConnections(), is(3)); + + assertThat(httpParameters.getPath(), is("/myPath")); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getReadTimeout(), is(4)); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5)); + + + assertThat(httpParameters.getYcaUseProxy(), is(true)); + assertThat(httpParameters.getYcaApplicationId(), is("myId")); + assertThat(httpParameters.getYcaProxy(), is("myYcaHost")); + assertThat(httpParameters.getYcaPort(), is(7)); + assertThat(httpParameters.getYcaRetry(), is(8000L)); + assertThat(httpParameters.getYcaTtl(), is(9000L)); + } + + @Test + public void requireFreezeWorksForAccessors() { + HTTPParameters p = new HTTPParameters(); + boolean caught = false; + final int expected = 37; + p.setConnectionTimeout(expected); + assertEquals(expected, p.getConnectionTimeout()); + p.freeze(); + try { + p.setConnectionTimeout(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setReadTimeout(expected); + assertEquals(expected, p.getReadTimeout()); + p.freeze(); + try { + p.setReadTimeout(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setPersistentConnections(true); + assertTrue(p.getPersistentConnections()); + p.freeze(); + try { + p.setPersistentConnections(false); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + assertEquals("http", p.getProxyType()); + + p = new HTTPParameters(); + caught = false; + p.setEnableProxy(true); + assertTrue(p.getEnableProxy()); + p.freeze(); + try { + p.setEnableProxy(false); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setProxyHost("nalle"); + assertEquals("nalle", p.getProxyHost()); + p.freeze(); + try { + p.setProxyHost("jappe"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setProxyPort(expected); + assertEquals(expected, p.getProxyPort()); + p.freeze(); + try { + p.setProxyPort(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMethod("POST"); + assertEquals("POST", p.getMethod()); + p.freeze(); + try { + p.setMethod("GET"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setSchema("gopher"); + assertEquals("gopher", p.getSchema()); + p.freeze(); + try { + p.setSchema("http"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setInputEncoding("iso-8859-15"); + assertEquals("iso-8859-15", p.getInputEncoding()); + p.freeze(); + try { + p.setInputEncoding("shift-jis"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setOutputEncoding("iso-8859-15"); + assertEquals("iso-8859-15", p.getOutputEncoding()); + p.freeze(); + try { + p.setOutputEncoding("shift-jis"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMaxTotalConnections(expected); + assertEquals(expected, p.getMaxTotalConnections()); + p.freeze(); + try { + p.setMaxTotalConnections(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMaxConnectionsPerRoute(expected); + assertEquals(expected, p.getMaxConnectionsPerRoute()); + p.freeze(); + try { + p.setMaxConnectionsPerRoute(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setSocketBufferSizeBytes(expected); + assertEquals(expected, p.getSocketBufferSizeBytes()); + p.freeze(); + try { + p.setSocketBufferSizeBytes(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setRetries(expected); + assertEquals(expected, p.getRetries()); + p.freeze(); + try { + p.setRetries(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java new file mode 100644 index 00000000000..8edc1ca8dd8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.federation.ProviderConfig.PingOption; +import com.yahoo.search.federation.http.Connection; +import com.yahoo.search.federation.http.HTTPProviderSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; +import org.junit.Test; + +import java.io.InputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.assertThat; + +/** + * See bug #3234696. + * + * @author Einar M R Rosenvinge + */ +public class HttpPostTestCase { + + @Test + public void testPostingSearcher() throws Exception { + StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer(); + server.start(); + + TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"), + Arrays.asList(new Connection("localhost", server.getServerPort())), + "/"); + Query q = new Query(""); + q.setTimeout(10000000L); + Execution e = new Execution(searcher, Execution.Context.createContextStub()); + + searcher.search(q, e); + + assertThat(server.getRequest(), containsString("My POST body")); + server.stop(); + } + + private static class TestPostSearcher extends HTTPProviderSearcher { + public TestPostSearcher(ComponentId id, List connections, String path) { + super(id, connections, httpParameters(path), Statistics.nullImplementation); + } + + private static HTTPParameters httpParameters(String path) { + HTTPParameters httpParameters = new HTTPParameters(path); + httpParameters.setPingOption(PingOption.Enum.DISABLE); + return httpParameters; + } + + @Override + protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) { + HttpPost request = new HttpPost(uri); + request.setEntity(entity); + return request; + } + + @Override + protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { + try { + return new StringEntity("My POST body"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + @Override + public Map getCacheKey(Query q) { + return new HashMap<>(0); + } + + @Override + public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { + // do nothing with the result + } + + @Override + protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { + //Empty + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java new file mode 100644 index 00000000000..c59dffb9cb7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.text.Utf8; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Rudimentary http searcher test. + * + * @author Jon Bratseth + */ +public class HttpTestCase extends junit.framework.TestCase { + + private StupidSingleThreadedHttpServer httpServer; + private TestHTTPClientSearcher searcher; + + public void testSearcher() throws JAXBException { + Result result = searchUsingLocalhost(); + + assertEquals("ok", result.getQuery().properties().get("gotResponse")); + assertEquals(0, result.getQuery().errors().size()); + } + + private Result searchUsingLocalhost() { + searcher = new TestHTTPClientSearcher("test","localhost",getPort()); + Query query = new Query("/?query=test"); + + query.setWindow(0,10); + return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub())); + } + + public void test_that_ip_address_set_on_meta_hit() { + Result result = searchUsingLocalhost(); + Hit metaHit = getFirstMetaHit(result.hits()); + String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS); + + assertEquals(ip, "127.0.0.1"); + } + + private Hit getFirstMetaHit(HitGroup hits) { + for (Iterator i = hits.unorderedDeepIterator(); i.hasNext();) { + Hit hit = i.next(); + if (hit.isMeta()) + return hit; + } + return null; + } + + @Override + public void setUp() throws Exception { + httpServer = new StupidSingleThreadedHttpServer(0, 0) { + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 5\r\n" + + "\r\n" + + "hello"); + } + }; + httpServer.start(); + } + + private int getPort() { + return httpServer.getServerPort(); + } + + @Override + public void tearDown() throws Exception { + httpServer.stop(); + if (searcher != null) { + searcher.shutdownConnectionManagers(); + } + } + + private static class TestHTTPClientSearcher extends HTTPClientSearcher { + + public TestHTTPClientSearcher(String id, String hostName, int port) { + super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); + } + + private static List toConnections(String hostName,int port) { + List connections=new ArrayList<>(); + connections.add(new Connection(hostName,port)); + return connections; + } + + @Override + public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { + query.properties().set("gotResponse","ok"); + return query; + } + + @Override + public Map getCacheKey(Query q) { + return null; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java new file mode 100644 index 00000000000..34791168db4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java @@ -0,0 +1,278 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; +import org.apache.http.HttpEntity; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Check for different keep-alive scenarios. What we really want to test + * is the server does not hang. + * + * @author Steinar Knutsen + */ +public class PingTestCase extends junit.framework.TestCase { + static final int TIMEOUT_MS = 60000; + public void testNiceCase() throws Exception { + NiceStupidServer server = new NiceStupidServer(); + server.start(); + checkSearchAndPing(true, true, true, server.getServerPort()); + server.stop(); + } + + private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) { + String resultThing; + String comment; + TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test", + "localhost", port); + try { + + Query query = new Query("/?query=test"); + + query.setWindow(0, 10); + // high timeout to allow for overloaded test machine + query.setTimeout(TIMEOUT_MS); + Ping ping = new Ping(TIMEOUT_MS); + + long start = System.currentTimeMillis(); + Execution exe = new Execution(searcher, Execution.Context.createContextStub()); + exe.search(query); + + resultThing = firstSearch ? "ok" : null; + comment = firstSearch ? "First search should have succeeded." : "First search should fail."; + assertEquals(comment, resultThing, query.properties().get("gotResponse")); + Pong pong = searcher.ping(ping, searcher.getConnection()); + if (pongCheck) { + assertEquals("Ping should not have failed.", 0, pong.getErrorSize()); + } else { + assertEquals("Ping should have failed.", 1, pong.getErrorSize()); + } + exe = new Execution(searcher, Execution.Context.createContextStub()); + exe.search(query); + + resultThing = secondSearch ? "ok" : null; + comment = secondSearch ? "Second search should have succeeded." : "Second search should fail."; + + assertEquals(resultThing, query.properties().get("gotResponse")); + long duration = System.currentTimeMillis() - start; + // target for duration based on the timeout values + some slack + assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000); + searcher.shutdownConnectionManagers(); + } finally { + searcher.deconstruct(); + } + } + + public void testUselessCase() throws Exception { + UselessStupidServer server = new UselessStupidServer(); + server.start(); + checkSearchAndPing(false, true, false, server.getServerPort()); + server.stop(); + } + + public void testGrumpyCase() throws Exception { + GrumpyStupidServer server = new GrumpyStupidServer(); + server.start(); + checkSearchAndPing(false, false, false, server.getServerPort()); + server.stop(); + } + + public void testPassiveAggressiveCase() throws Exception { + PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer(); + server.start(); + checkSearchAndPing(true, false, true, server.getServerPort()); + server.stop(); + } + + // OK on ping and search + private static class NiceStupidServer extends StupidSingleThreadedHttpServer { + private NiceStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } + } + + // rejects ping and accepts search + private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer { + + private PassiveAggressiveStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + if (request.contains("/ping")) { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } else { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } + } + } + + // accepts ping and rejects search + private static class UselessStupidServer extends StupidSingleThreadedHttpServer { + private UselessStupidServer() throws IOException { + super(0, 0); + } + + + @Override + protected byte[] getResponse(String request) { + if (request.contains("/ping")) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } else { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } + } + } + + // rejects ping and search + private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer { + private GrumpyStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } + } + + private static class TestHTTPClientSearcher extends HTTPClientSearcher { + + public TestHTTPClientSearcher(String id, String hostName, int port) { + super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); + } + + private static List toConnections(String hostName,int port) { + List connections=new ArrayList<>(); + connections.add(new Connection(hostName,port)); + return connections; + } + + @Override + public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { + query.properties().set("gotResponse","ok"); + return query; + } + + @Override + public Result search(Query query, Execution execution, + Connection connection) { + URI uri; + try { + uri = new URL("http", connection.getHost(), connection + .getPort(), "/search").toURI(); + } catch (MalformedURLException e) { + query.errors().add(createMalformedUrlError(query, e)); + return execution.search(query); + } catch (URISyntaxException e) { + query.errors().add(createMalformedUrlError(query, e)); + return execution.search(query); + } + + HttpEntity entity; + try { + entity = getEntity(uri, query); + } catch (IOException e) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in " + + this + " using " + connection + + " for " + query + ": " + + Exceptions.toMessageString(e))); + return execution.search(query); + } catch (TimeoutException e) { + query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in " + + this + + " for " + query + ": " + e.getMessage())); + return execution.search(query); + } + if (entity == null) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in " + + this + " using " + connection + " for " + query)); + return execution.search(query); + } + + try { + query = handleResponse(entity, query); + } catch (IOException e) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("Error when trying to consume input in " + + this + ": " + Exceptions.toMessageString(e))); + } finally { + cleanupHttpEntity(entity); + } + return execution.search(query); + } + + @Override + public Map getCacheKey(Query q) { + return null; + } + + @Override + protected URI getPingURI(Connection connection) + throws MalformedURLException, URISyntaxException { + return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI(); + } + + Connection getConnection() { + return getHasher().getNodes().select(0, 0); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java new file mode 100644 index 00000000000..baeb9fd0a41 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Tests that source and backend specific parameters from the query are added correctly to the backend requests + * + * @author Jon Bratseth + */ +public class QueryParametersTestCase extends junit.framework.TestCase { + + public void testQueryParameters() { + Query query=new Query(); + query.properties().set("a","a-value"); + query.properties().set("b.c","b.c-value"); + query.properties().set("source.otherSource.d","d-value"); + query.properties().set("source.testSource.e","e-value"); + query.properties().set("source.testSource.f.g","f.g-value"); + query.properties().set("provider.testProvider.h","h-value"); + query.properties().set("provider.testProvider.i.j","i.j-value"); + + query.properties().set("sourceName","testSource"); // Done by federation searcher + query.properties().set("providerName","testProvider"); // Done by federation searcher + + TestHttpProvider searcher=new TestHttpProvider(); + Map parameters=searcher.getQueryMap(query); + searcher.deconstruct(); + + assertEquals(4,parameters.size()); // the appropriate 4 of the above + assertEquals(parameters.get("e"),"e-value"); + assertEquals(parameters.get("f.g"),"f.g-value"); + assertEquals(parameters.get("h"),"h-value"); + assertEquals(parameters.get("i.j"),"i.j-value"); + } + + public static class TestHttpProvider extends HTTPProviderSearcher { + + public TestHttpProvider() { + super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation); + } + + @Override + public Map getCacheKey(Query q) { + return Collections.singletonMap("nocaching", String.valueOf(Math.random())); + } + + @Override + protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { + } + + } + +} + diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java new file mode 100644 index 00000000000..e874c89b918 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.processing.request.properties.PropertyMap; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.federation.sourceref.Target; +import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; + +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; + +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class SearchChainResolverTestCase { + + private static final FederationOptions federationOptions = + new FederationOptions().setTimeoutInMilliseconds(3000).setOptional(true); + + private static final ComponentId searchChainId = ComponentId.fromString("search-chain"); + private static final ComponentId providerId = ComponentId.fromString("provider"); + private static final ComponentId provider2Id = ComponentId.fromString("provider2"); + + private static final ComponentId sourceId = ComponentId.fromString("source"); + private static final ComponentId sourceChainInProviderId = + ComponentId.fromString("source-chain").nestInNamespace(providerId); + private static final ComponentId sourceChainInProvider2Id = + ComponentId.fromString("source-chain").nestInNamespace(provider2Id); + + private static final SearchChainResolver searchChainResolver; + + static { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + builder.addSearchChain(searchChainId, federationOptions.setUseByDefault(true), Collections.emptyList()); + builder.addSearchChain(providerId, federationOptions.setUseByDefault(false), Collections.emptyList()); + builder.addSourceForProvider(sourceId, providerId, sourceChainInProviderId, true, + federationOptions.setUseByDefault(true), Collections.emptyList()); + builder.addSourceForProvider(sourceId, provider2Id, sourceChainInProvider2Id, false, + federationOptions.setUseByDefault(false), Collections.emptyList()); + + searchChainResolver = builder.build(); + } + + @Test + public void check_default_search_chains() { + assertThat(searchChainResolver.defaultTargets().size(), is(2)); + + Iterator iterator = searchChainResolver.defaultTargets().iterator(); + assertThat(iterator.next().searchRefDescription(), is(searchChainId.toString())); + assertThat(iterator.next().searchRefDescription(), is(sourceChainInProviderId.toString())); + } + + @Test + public void require_error_message_for_invalid_source() { + try { + resolve("no-such-source"); + fail("Expected exception."); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Could not resolve source ref 'no-such-source'.")); + } + } + + @Test + public void lookup_search_chain() throws Exception { + SearchChainInvocationSpec res = resolve(searchChainId.getName()); + assertThat(res.searchChainId, is(searchChainId)); + } + + //TODO: TVT: @Test() + public void lookup_provider() throws Exception { + SearchChainInvocationSpec res = resolve(providerId.getName()); + assertThat(res.provider, is(providerId)); + assertNull(res.source); + assertThat(res.searchChainId, is(providerId)); + } + + @Test + public void lookup_source() throws Exception { + SearchChainInvocationSpec res = resolve(sourceId.getName()); + assertIsSourceInProvider(res); + } + + @Test + public void lookup_source_search_chain_directly() throws Exception { + SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()); + assertIsSourceInProvider(res); + } + + private void assertIsSourceInProvider(SearchChainInvocationSpec res) { + assertThat(res.provider, is(providerId)); + assertThat(res.source, is(sourceId)); + assertThat(res.searchChainId, is(sourceChainInProviderId)); + } + + @Test + public void lookup_source_for_provider2() throws Exception { + SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()); + assertThat(res.provider, is(provider2Id)); + assertThat(res.source, is(sourceId)); + assertThat(res.searchChainId, is(sourceChainInProvider2Id)); + } + + @Test + public void lists_source_ref_description_for_top_level_targets() { + SortedSet topLevelTargets = searchChainResolver.allTopLevelTargets(); + assertThat(topLevelTargets.size(), is(3)); + + Iterator i = topLevelTargets.iterator(); + assertSearchRefDescriptionIs(i.next(), providerId.toString()); + assertSearchRefDescriptionIs(i.next(), searchChainId.toString()); + assertSearchRefDescriptionIs(i.next(), "source[provider = provider, provider2]"); + } + + private void assertSearchRefDescriptionIs(Target target, String expected) { + assertThat(target.searchRefDescription(), is(expected)); + } + + static Properties emptySourceToProviderMap() { + return new PropertyMap(); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification) throws UnresolvedSearchChainException { + return resolve(sourceSpecification, emptySourceToProviderMap()); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification, String providerSpecification) + throws UnresolvedSearchChainException { + Properties sourceToProviderMap = emptySourceToProviderMap(); + sourceToProviderMap.set("source." + sourceSpecification + ".provider", providerSpecification); + return resolve(sourceSpecification, sourceToProviderMap); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification, Properties sourceToProviderMap) + throws UnresolvedSearchChainException { + SearchChainInvocationSpec res = searchChainResolver.resolve( + ComponentSpecification.fromString(sourceSpecification), sourceToProviderMap); + assertThat(res.federationOptions, is(federationOptions)); + return res; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java new file mode 100644 index 00000000000..f8559745358 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.federation.sourceref.SourceRefResolver; +import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import static com.yahoo.search.federation.sourceref.test.SearchChainResolverTestCase.emptySourceToProviderMap; +import static junit.framework.Assert.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.matchers.JUnitMatchers.hasItems; + + +/** + * Test for SourceRefResolver. + * @author tonytv + */ +public class SourceRefResolverTestCase { + private static final String cluster1 = "cluster1"; + private static final String cluster2 = "cluster2"; + private static final String cluster3 = "cluster3"; + private static IndexFacts indexFacts; + + private static final SourceRefResolver sourceRefResolver = createSourceRefResolver(); + + static { + setupIndexFacts(); + } + + private static SourceRefResolver createSourceRefResolver() { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + builder.addSearchChain(ComponentId.fromString(cluster1), new FederationOptions().setUseByDefault(true), + Collections.emptyList()); + builder.addSearchChain(ComponentId.fromString(cluster2), new FederationOptions().setUseByDefault(true), + Collections.emptyList()); + + return new SourceRefResolver(builder.build()); + } + + private static void setupIndexFacts() { + TreeMap> masterClusters = new TreeMap<>(); + masterClusters.put(cluster1, Arrays.asList("document1", "document2")); + masterClusters.put(cluster2, Arrays.asList("document1")); + masterClusters.put(cluster3, Arrays.asList("document3")); + indexFacts = new IndexFacts(new IndexModel(masterClusters, null, null)); + } + + @Test + public void check_test_assumptions() { + assertThat(indexFacts.clustersHavingSearchDefinition("document1"), hasItems("cluster1", "cluster2")); + } + + @Test + public void lookup_search_chain() throws Exception { + Set searchChains = resolve(cluster1); + assertThat(searchChains.size(), is(1)); + assertThat(searchChainIds(searchChains), hasItems(cluster1)); + } + + @Test + public void lookup_search_chains_for_document1() throws Exception { + Set searchChains = resolve("document1"); + assertThat(searchChains.size(), is(2)); + assertThat(searchChainIds(searchChains), hasItems(cluster1, cluster2)); + } + + @Test + public void error_when_document_gives_cluster_without_matching_search_chain() { + try { + resolve("document3"); + fail("Expected exception"); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Failed to resolve cluster search chain 'cluster3' " + + "when using source ref 'document3' as a document name.")); + } + } + + @Test + public void error_when_no_document_or_search_chain() { + try { + resolve("document4"); + fail("Expected exception"); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Could not resolve source ref 'document4'.")); + } + } + + private List searchChainIds(Set searchChains) { + List names = new ArrayList<>(); + for (SearchChainInvocationSpec searchChain : searchChains) { + names.add(searchChain.searchChainId.stringValue()); + } + return names; + } + + private Set resolve(String documentName) throws UnresolvedSearchChainException { + return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap(), indexFacts); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java new file mode 100644 index 00000000000..40786ee89a9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +public class AddHitsWithRelevanceSearcher extends Searcher { + public static final int numHitsAdded = 5; + + private final String chainName; + private final int relevanceMultiplier; + + public AddHitsWithRelevanceSearcher(String chainName, int rankMultiplier) { + this.chainName = chainName; + this.relevanceMultiplier = rankMultiplier; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + for (int i = 1; i <= numHitsAdded; ++i) { + result.hits().add(createHit(i)); + } + return result; + } + + private Hit createHit(int i) { + int relevance = i * relevanceMultiplier; + return new Hit(chainName + "-" + relevance, relevance); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java new file mode 100644 index 00000000000..dcecf36f2ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +public class BlockingSearcher extends Searcher { + @Override + public synchronized Result search(Query query, Execution execution) { + try { + while (true) + wait(); + } catch (InterruptedException e) { + } + return execution.search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java new file mode 100644 index 00000000000..dba0deb607a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java @@ -0,0 +1,306 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import java.util.Optional; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.net.URI; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.selection.FederationTarget; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.Execution.Context; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author tonytv + */ +public class FederationSearcherTest { + private static final String hasBeenFilled = "hasBeenFilled"; + + private static class AddHitSearcher extends Searcher { + protected Hit hit = createHit(); + + private Hit createHit() { + Hit hit = new Hit("dummy"); + hit.setFillable(); + return hit; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + if (firstHit(result) != hit) { + throw new RuntimeException("Unknown hit"); + } + firstHit(result).setField(hasBeenFilled, true); + } + } + + private static class ModifyQueryAndAddHitSearcher extends AddHitSearcher { + private final String marker; + + ModifyQueryAndAddHitSearcher(String marker) { + super(); + this.marker = marker; + } + + @Override + public Result search(Query query, Execution execution) { + query.getModel().getQueryTree().setRoot(new WordItem(marker)); + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + + } + + @Test + public void require_that_hits_are_not_automatically_filled() { + Result result = federationToSingleAddHitSearcher().search(); + assertNotFilled(firstHitInFirstGroup(result)); + } + + @Test + public void require_that_hits_can_be_filled() { + Result result = federationToSingleAddHitSearcher().searchAndFill(); + assertFilled(firstHitInFirstGroup(result)); + } + + @Test + public void require_that_hits_can_be_filled_when_moved() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addSearchChain("chain2", new AddHitSearcher()); + + Result result = tester.search(); + + Result reorganizedResult = new Result(result.getQuery()); + HitGroup hit1 = new HitGroup(); + HitGroup nestedHitGroup = new HitGroup(); + + hit1.add(nestedHitGroup); + reorganizedResult.hits().add(hit1); + + HitGroup chain1Group = (HitGroup) result.hits().get(0); + HitGroup chain2Group = (HitGroup) result.hits().get(1); + + nestedHitGroup.add(chain1Group.get(0)); + reorganizedResult.hits().add(chain2Group.get(0)); + reorganizedResult.hits().add(nestedHitGroup); + + tester.fill(reorganizedResult); + assertFilled(nestedHitGroup.get(0)); + assertFilled(chain2Group.get(0)); + + } + + @Test + public void require_that_hits_can_be_filled_for_multiple_chains_and_queries() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addSearchChain("chain2", new ModifyQueryAndAddHitSearcher("modified1")); + tester.addSearchChain("chain3", new ModifyQueryAndAddHitSearcher("modified2")); + + Result result = tester.search(); + tester.fill(result); + for (Iterator i = result.hits().deepIterator(); i.hasNext();) { + Hit h = i.next(); + assertFilled(h); + } + assertEquals(3, result.hits().getConcreteSize()); + } + + + @Test + public void require_that_optional_search_chains_does_not_delay_federation() { + BlockingSearcher blockingSearcher = new BlockingSearcher(); + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addOptionalSearchChain("chain2", blockingSearcher); + + Result result = tester.searchAndFill(); + assertThat(getNonErrorHits(result).size(), is(1)); + assertFilled(getFirstHit(getNonErrorHits(result).get(0))); + assertNotNull(result.hits().getError()); + } + + @Test + public void require_that_calling_a_single_slow_source_with_long_timeout_does_not_delay_federation() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", + new FederationOptions().setUseByDefault(true).setRequestTimeoutInMilliseconds(3600 * 1000), + new BlockingSearcher() ); + + Query query = new Query(); + query.setTimeout(2); // make the test run faster + Result result = tester.search(query); + assertThat(getNonErrorHits(result).size(), is(0)); + assertNotNull(result.hits().getError()); + } + + private Hit getFirstHit(Hit hitGroup) { + if (hitGroup instanceof HitGroup) + return ((HitGroup) hitGroup).get(0); + else + throw new IllegalArgumentException("Expected HitGroup"); + } + + private List getNonErrorHits(Result result) { + List nonErrorHits = new ArrayList<>(); + for (Hit hit : result.hits()) { + if (!(hit instanceof ErrorHit)) + nonErrorHits.add(hit); + } + + return nonErrorHits; + } + private static void assertFilled(Hit hit) { + assertTrue((Boolean)hit.getField(hasBeenFilled)); + } + + private static void assertNotFilled(Hit hit) { + assertNull(hit.getField(hasBeenFilled)); + } + + private FederationTester federationToSingleAddHitSearcher() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + return tester; + } + + private static Hit firstHit(Result result) { + return result.hits().get(0); + } + + private static Hit firstHitInFirstGroup(Result result) { + return ((HitGroup)firstHit(result)).get(0); + } + + @Test + public void custom_federation_target() { + ComponentId targetSelectorId = ComponentId.fromString("TargetSelector"); + ComponentRegistry targetSelectors = new ComponentRegistry<>(); + targetSelectors.register(targetSelectorId, new TestTargetSelector()); + + FederationSearcher searcher = new FederationSearcher( + new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + new StrictContractsConfig(new StrictContractsConfig.Builder()), + targetSelectors); + + Result result = new Execution(searcher, Context.createContextStub()).search(new Query()); + HitGroup myChainGroup = (HitGroup) result.hits().get(0); + assertThat(myChainGroup.getId(), is(new URI("source:myChain"))); + assertThat(myChainGroup.get(0).getId(), is(new URI("myHit"))); + } + + static class TestTargetSelector implements TargetSelector { + String keyName = getClass().getName(); + + @Override + public Collection> getTargets(Query query, ChainRegistry searcherChainRegistry) { + return Arrays.asList( + new FederationTarget<>(new Chain<>("myChain", Collections.emptyList()), new FederationOptions(), "hello")); + } + + @Override + public void modifyTargetQuery(FederationTarget target, Query query) { + checkTarget(target); + query.properties().set(keyName, "called"); + } + + @Override + public void modifyTargetResult(FederationTarget target, Result result) { + checkTarget(target); + assertThat(result.getQuery().properties().getString(keyName), is("called")); + result.hits().add(new Hit("myHit")); + } + + private void checkTarget(FederationTarget target) { + assertThat(target.getCustomData(), is("hello")); + assertThat(target.getChain().getId(), is(ComponentId.fromString("myChain"))); + } + } + + static class TestMultipleTargetSelector implements TargetSelector { + String keyName = getClass().getName(); + + @Override + public Collection> getTargets(Query query, ChainRegistry searcherChainRegistry) { + return Arrays.asList(createTarget(1), createTarget(2)); + } + + private FederationTarget createTarget(int number) { + return new FederationTarget<>(new Chain<>("chain" + number, Collections.emptyList()), + new FederationOptions(), + "custom-data:" + number); + } + + @Override + public void modifyTargetQuery(FederationTarget target, Query query) { + query.properties().set(keyName, "modifyTargetQuery:" + target.getCustomData()); + } + + @Override + public void modifyTargetResult(FederationTarget target, Result result) { + Hit hit = new Hit("MyHit" + target.getCustomData()); + hit.setField("data", result.getQuery().properties().get(keyName)); + result.hits().add(hit); + } + } + + @Test + public void target_selectors_can_have_multiple_targets() { + ComponentId targetSelectorId = ComponentId.fromString("TestMultipleTargetSelector"); + ComponentRegistry targetSelectors = new ComponentRegistry<>(); + targetSelectors.register(targetSelectorId, new TestMultipleTargetSelector()); + + FederationSearcher searcher = new FederationSearcher( + new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + new StrictContractsConfig(new StrictContractsConfig.Builder()), + targetSelectors); + + Result result = new Execution(searcher, Context.createContextStub()).search(new Query()); + + Iterator hitsIterator = result.hits().deepIterator(); + Hit hit1 = hitsIterator.next(); + Hit hit2 = hitsIterator.next(); + + assertThat(hit1.getSource(), is("chain1")); + assertThat(hit2.getSource(), is("chain2")); + + assertThat((String)hit1.getField("data"), is("modifyTargetQuery:custom-data:1")); + assertThat((String)hit2.getField("data"), is("modifyTargetQuery:custom-data:2")); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java new file mode 100644 index 00000000000..bc00890624b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java @@ -0,0 +1,411 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.test.QueryTestCase; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.*; +import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties; + +/** + * Test for federation searcher. The searcher is also tested in + * com.yahoo.prelude.searcher.test.BlendingSearcherTestCase. + * + * @author Arne Bergene Fossaa + */ +@SuppressWarnings("deprecation") +public class FederationSearcherTestCase { + + static final String SOURCE1 = "source1"; + static final String SOURCE2 = "source2"; + + public static class TwoSourceChecker extends TraceVisitor { + public boolean traceFromSource1 = false; + public boolean traceFromSource2 = false; + + @Override + public void visit(TraceNode node) { + if (SOURCE1.equals(node.payload())) { + traceFromSource1 = true; + } else if (SOURCE2.equals(node.payload())) { + traceFromSource2 = true; + } + } + + } + + private FederationConfig.Builder builder; + private SearchChainRegistry chainRegistry; + + @Before + public void setUp() throws Exception { + builder = new FederationConfig.Builder(); + chainRegistry = new SearchChainRegistry(); + } + + @After + public void tearDown() throws Exception { + builder = null; + chainRegistry = null; + } + + private void addChained(final Searcher searcher, final String sourceName) { + builder.target(new FederationConfig.Target.Builder(). + id(sourceName). + searchChain(new FederationConfig.Target.SearchChain.Builder(). + searchChainId(sourceName). + timeoutMillis(10000). + useByDefault(true)) + ); + chainRegistry.register(new ComponentId(sourceName), + createSearchChain(new ComponentId(sourceName), searcher)); + } + + private Searcher createFederationSearcher() { + return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder())); + } + + private Searcher createFederationSearcher(PropagateSourceProperties.Enum propagateSourceProperties) { + return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder().propagateSourceProperties(propagateSourceProperties))); + } + + private Searcher createStrictFederationSearcher() { + StrictContractsConfig.Builder builder = new StrictContractsConfig.Builder(); + builder.searchchains(true); + final StrictContractsConfig contracts = new StrictContractsConfig(builder); + return buildFederation(contracts); + } + + private Searcher buildFederation(final StrictContractsConfig contracts) + throws RuntimeException { + + return new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry()); + } + + private SearchChain createSearchChain(final ComponentId chainId, + final Searcher searcher) { + return new SearchChain(chainId, searcher); + } + + @Test + public void testQueryProfileNestedReferencing() { + addChained(new MockSearcher(), "mySource1"); + addChained(new MockSearcher(), "mySource2"); + Chain mainChain = new Chain<>("default", createFederationSearcher()); + + QueryProfile defaultProfile = new QueryProfile("default"); + defaultProfile.set("source.mySource1.hits", "%{hits}", (QueryProfileRegistry)null); + defaultProfile.freeze(); + Query q = new Query(QueryTestCase.httpEncode("?query=test"), defaultProfile.compile(null)); + + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + } + + @Test + public void testTraceTwoSources() { + final Chain mainChain = twoTracingSources(false); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertTrue(lookForTraces.traceFromSource2); + } + + private Chain twoTracingSources(boolean strictContracts) { + addChained(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + query.trace(SOURCE1, 1); + return execution.search(query); + } + + }, SOURCE1); + + addChained(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + query.trace(SOURCE2, 1); + return execution.search(query); + } + + }, SOURCE2); + + final Chain mainChain = new Chain<>("default", + new FederationSearcher(new FederationConfig(builder), + new StrictContractsConfig( + new StrictContractsConfig.Builder().searchchains(strictContracts)), + new ComponentRegistry<>())); + return mainChain; + } + + @Test + public void testTraceOneSourceNoCloning() { + final Chain mainChain = twoTracingSources(true); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertFalse(lookForTraces.traceFromSource2); + } + + @Test + public void testTraceOneSourceWithCloning() { + final Chain mainChain = twoTracingSources(false); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertFalse(lookForTraces.traceFromSource2); + + } + + + @Test + public void testPropertyPropagation() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.ALL); + + assertEquals("source:mySource1", result.hits().get(0).getId() + .stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId() + .stringValue()); + assertEquals("nalle", result.hits().get(0).getQuery().getPresentation() + .getSummary()); + assertNull(result.hits().get(1).getQuery().getPresentation() + .getSummary()); + + } + + private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) { + addChained(new MockSearcher(), "mySource1"); + addChained(new MockSearcher(), "mySource2"); + final Chain mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties)); + + final Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle")); + + final Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + return result; + } + + @Test + public void testDisablePropertyPropagation() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.NONE); + + assertNull(result.hits().get(0).getQuery().getPresentation() + .getSummary()); + } + + @Test + public void testNoCloning() { + final String sourceName = "cloningcheck"; + Query query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName)); + addChained(new QueryCheckSearcher(query), sourceName); + addChained(new MockSearcher(), "mySource1"); + Chain mainChain = new Chain<>("default", createStrictFederationSearcher()); + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + HitGroup h = (HitGroup) result.hits().get(0); + assertNull(h.getErrorHit()); + assertSame(QueryCheckSearcher.OK, h.get(0).getField(QueryCheckSearcher.STATUS)); + + mainChain = new Chain<>("default", createFederationSearcher()); + result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + h = (HitGroup) result.hits().get(0); + assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, + h.getError().getDetailedMessage()); + + query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName + ",mySource1")); + addChained(new QueryCheckSearcher(query), sourceName); + result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + h = (HitGroup) result.hits().get(0); + assertEquals("source:" + sourceName, h.getId().stringValue()); + assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, + h.getError().getDetailedMessage()); + assertEquals("source:mySource1", result.hits().get(1).getId() + .stringValue()); + } + + @Test + public void testTopLevelHitGroupFieldPropagation() { + addChained(new MockSearcher(), "mySource1"); + addChained(new AnotherMockSearcher(), "mySource2"); + Chain mainChain = new Chain<>("default", createFederationSearcher()); + + Query q = new Query("?query=test"); + + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + assertEquals( + AnotherMockSearcher.IS_THIS_PROPAGATED, + result.hits().get(1).getField(AnotherMockSearcher.PROPAGATION_KEY)); + } + + private static class MockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + String sourceName = query.properties().getString("sourceName", "unknown"); + Result result = new Result(query); + for (int i = 1; i <= query.getHits(); i++) { + final Hit hit = new Hit(sourceName + ":" + i, 1d / i); + hit.setSource(sourceName); + result.hits().add(hit); + } + return result; + } + + } + + private static class SleepingMockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + try { + Thread.sleep(100); + } catch (Exception e) { + throw new RuntimeException(e); + } + return execution.search(query); + } + } + + + private static class AnotherMockSearcher extends Searcher { + + private static final String PROPAGATION_KEY = "hello"; + private static final String IS_THIS_PROPAGATED = "is this propagated?"; + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + result.hits().setField(PROPAGATION_KEY, IS_THIS_PROPAGATED); + return result; + } + } + + @Test + public void testProviderSelectionFromQueryProperties() { + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(new Chain<>("provider1", new MockProvider("provider1"))); + registry.register(new Chain<>("provider2", new MockProvider("provider2"))); + registry.register(new Chain<>("default", createMultiProviderFederationSearcher())); + assertSelects("provider1", registry); + assertSelects("provider2", registry); + } + + private void assertSelects(String providerName, SearchChainRegistry registry) { + QueryProfile profile = new QueryProfile("test"); + profile.set("source.news.provider", providerName, (QueryProfileRegistry)null); + Query query = new Query(QueryTestCase.httpEncode("?query=test&model.sources=news"), profile.compile(null)); + Result result = new Execution(registry.getComponent("default"), Execution.Context.createContextStub(registry, null)).search(query); + assertEquals(1, result.hits().size()); + assertNotNull(result.hits().get(providerName + ":1")); + } + + private FederationSearcher createMultiProviderFederationSearcher() { + final FederationOptions options = new FederationOptions(); + final SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + + final ComponentId provider1 = new ComponentId("provider1"); + final ComponentId provider2 = new ComponentId("provider2"); + final ComponentId news = new ComponentId("news"); + builder.addSearchChain(provider1, options, + Collections. emptyList()); + builder.addSearchChain(provider2, options, + Collections. emptyList()); + builder.addSourceForProvider(news, provider1, provider1, true, options, + Collections. emptyList()); + builder.addSourceForProvider(news, provider2, provider2, false, + options, Collections. emptyList()); + + return new FederationSearcher(new ComponentId("federation"), builder.build()); + } + + private static class MockProvider extends Searcher { + + private final String name; + + public MockProvider(final String name) { + this.name = name; + } + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + result.hits().add(new Hit(name + ":1")); + return result; + } + + } + + private static class QueryCheckSearcher extends Searcher { + private static final String STATUS = "status"; + public static final String FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY = "FederationSearcher has cloned the query."; + public static final String OK = "Got the correct query."; + private final Query query; + + QueryCheckSearcher(final Query query) { + this.query = query; + } + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + if (query != this.query) { + result.hits().addError(ErrorMessage + .createErrorInPluginSearcher(FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY)); + } else { + final Hit h = new Hit("QueryCheckSearcher status hit"); + h.setField(STATUS, OK); + result.hits().add(h); + } + return result; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java new file mode 100644 index 00000000000..7b0451a01ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +import java.util.Collections; + +/** +* @author tonytv +*/ +class FederationTester { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + SearchChainRegistry registry = new SearchChainRegistry(); + + Execution execution; + + void addSearchChain(String id, Searcher... searchers) { + addSearchChain(id, federationOptions(), searchers); + } + + void addSearchChain(String id, FederationOptions federationOptions, Searcher... searchers) { + ComponentId searchChainId = ComponentId.fromString(id); + + builder.addSearchChain(searchChainId, federationOptions, Collections.emptyList()); + + Chain chain = new Chain<>(searchChainId, searchers); + registry.register(chain); + } + + public void addOptionalSearchChain(String id, Searcher... searchers) { + addSearchChain(id, federationOptions().setOptional(true), searchers); + } + + private FederationOptions federationOptions() { + int preventTimeout = 24 * 60 * 60 * 1000; + return new FederationOptions().setUseByDefault(true).setTimeoutInMilliseconds(preventTimeout); + } + + FederationSearcher buildFederationSearcher() { + return new FederationSearcher(ComponentId.fromString("federation"), builder.build()); + } + + public Result search() { + return search(new Query()); + } + + public Result search(Query query) { + execution = createExecution(); + return execution.search(query); + } + + public Result searchAndFill() { + Result result = search(); + fill(result); + return result; + } + + private Execution createExecution() { + registry.freeze(); + return new Execution(new Chain(buildFederationSearcher()), Execution.Context.createContextStub(registry, null)); + } + + public void fill(Result result) { + execution.fill(result, "default"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java new file mode 100644 index 00000000000..dcbbb217c7d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringStartsWith.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class HitCountTestCase { + + @Test + public void require_that_offset_and_hits_are_adjusted_when_federating() { + final int chain1RelevanceMultiplier = 1; + final int chain2RelevanceMultiplier = 10; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitsWithRelevanceSearcher("chain1", chain1RelevanceMultiplier)); + tester.addSearchChain("chain2", new AddHitsWithRelevanceSearcher("chain2", chain2RelevanceMultiplier)); + + Query query = new Query(); + query.setHits(5); + + query.setOffset(0); + assertAllHitsFrom("chain2", flattenAndTrim(tester.search(query))); + + query.setOffset(5); + assertAllHitsFrom("chain1", flattenAndTrim(tester.search(query))); + } + + @Test + public void require_that_hit_counts_are_merged() { + final long chain1TotalHitCount = 3; + final long chain1DeepHitCount = 5; + + final long chain2TotalHitCount = 7; + final long chain2DeepHitCount = 11; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new SetHitCountsSearcher(chain1TotalHitCount, chain1DeepHitCount)); + tester.addSearchChain("chain2", new SetHitCountsSearcher(chain2TotalHitCount, chain2DeepHitCount)); + + Result result = tester.searchAndFill(); + + assertThat(result.getTotalHitCount(), is(chain1TotalHitCount + chain2TotalHitCount)); + assertThat(result.getDeepHitCount(), is(chain1DeepHitCount + chain2DeepHitCount)); + } + + @Test + public void require_that_logging_hit_is_populated_with_result_count() { + final long chain1TotalHitCount = 9; + final long chain1DeepHitCount = 14; + + final long chain2TotalHitCount = 11; + final long chain2DeepHitCount = 15; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", + new SetHitCountsSearcher(chain1TotalHitCount, chain1DeepHitCount)); + + tester.addSearchChain("chain2", + new SetHitCountsSearcher(chain2TotalHitCount, chain2DeepHitCount), + new AddHitsWithRelevanceSearcher("chain1", 2)); + + Query query = new Query(); + query.setOffset(2); + query.setHits(7); + Result result = tester.search(); + List metaHits = getFirstMetaHitInEachGroup(result); + + Hit first = metaHits.get(0); + assertEquals(chain1TotalHitCount, first.getField("count_total")); + assertEquals(chain1TotalHitCount, first.getField("count_total")); + assertEquals(1, first.getField("count_first")); + assertEquals(0, first.getField("count_last")); + + Hit second = metaHits.get(1); + assertEquals(chain2TotalHitCount, second.getField("count_total")); + assertEquals(chain2TotalHitCount, second.getField("count_total")); + assertEquals(1, second.getField("count_first")); + assertEquals(AddHitsWithRelevanceSearcher.numHitsAdded, second.getField("count_last")); + + } + + private List getFirstMetaHitInEachGroup(Result result) { + List metaHits = new ArrayList<>(); + for (Hit topLevelHit : result.hits()) { + if (topLevelHit instanceof HitGroup) { + for (Hit hit : (HitGroup)topLevelHit) { + if (hit.isMeta()) { + metaHits.add(hit); + break; + } + } + } + } + return metaHits; + } + + private void assertAllHitsFrom(String chainName, HitGroup flattenedHits) { + for (Hit hit : flattenedHits) { + assertThat(hit.getId().toString(), startsWith(chainName)); + } + } + + private HitGroup flattenAndTrim(Result result) { + HitGroup flattenedHits = new HitGroup(); + result.setQuery(result.getQuery()); + flatten(result.hits(), flattenedHits); + + flattenedHits.trim(result.getQuery().getOffset(), result.getQuery().getHits()); + return flattenedHits; + } + + private void flatten(HitGroup hits, HitGroup flattenedHits) { + for (Hit hit : hits) { + if (hit instanceof HitGroup) { + flatten((HitGroup) hit, flattenedHits); + } else { + flattenedHits.add(hit); + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java new file mode 100644 index 00000000000..81a3007735c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +class SetHitCountsSearcher extends Searcher { + + private final long totalHitCount; + private final long deepHitCount; + + public SetHitCountsSearcher(long totalHitCount, long deepHitCount) { + this.totalHitCount = totalHitCount; + this.deepHitCount = deepHitCount; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(createLoggingHit()); + + result.setTotalHitCount(totalHitCount); + result.setDeepHitCount(deepHitCount); + return result; + } + + private Hit createLoggingHit() { + Hit hit = new Hit("SetHitCountSearcher"); + hit.setMeta(true); + hit.types().add("logging"); + return hit; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java new file mode 100644 index 00000000000..2868d69457b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.QueryMarshaller; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QueryMarshallerTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + + @Test + public void testCommonCommonCase() { + AndItem root = new AndItem(); + addThreeWords(root); + assertEquals("a AND b AND c", new QueryMarshaller().marshal(root)); + } + + @Test + public void testPhrase() { + PhraseItem root = new PhraseItem(); + root.setIndexName("habla"); + addThreeWords(root); + assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root)); + } + + @Test + public void testPhraseDefaultIndex() { + PhraseItem root = new PhraseItem(); + addThreeWords(root); + assertEquals("\"a b c\"", new QueryMarshaller().marshal(root)); + } + + @Test + public void testLittleMoreComplex() { + AndItem root = new AndItem(); + addThreeWords(root); + OrItem ambig = new OrItem(); + root.addItem(ambig); + addThreeWords(ambig); + AndItem but = new AndItem(); + addThreeWords(but); + ambig.addItem(but); + assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )", + new QueryMarshaller().marshal(root)); + } + + @Test + public void testRank() { + RankItem root = new RankItem(); + addThreeWords(root); + assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root)); + } + + @Test + public void testNear() { + NearItem near = new NearItem(3); + addThreeWords(near); + assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near)); + } + + @Test + public void testONear() { + ONearItem oNear = new ONearItem(3); + addThreeWords(oNear); + assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear)); + } + + private void addThreeWords(CompositeItem root) { + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("c")); + } + + @Test + public void testNegativeGroupedTerms() { + testQueryString(new QueryMarshaller(), "a -(b c) -(d e)", + "a ANDNOT ( b AND c ) ANDNOT ( d AND e )"); + } + + @Test + public void testPositiveGroupedTerms() { + testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )"); + } + + @Test + public void testInt() { + testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123"); + } + + @Test + public void testCJKOneWord() { + testQueryString(new QueryMarshaller(), "天é¾äºº"); + } + + @Test + public void testTwoWords() { + testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics()); + } + + @Test + public void testTwoWordsInPhrase() { + testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics()); + } + + @Test + public void testCJKTwoSentences() { + testQueryString(new QueryMarshaller(), "是ä¸æ˜¯é€™æ¨£çš„夜晚 ä½ æ‰æœƒé€™æ¨£åœ°æƒ³èµ·æˆ‘", "是ä¸æ˜¯é€™æ¨£çš„夜晚 AND ä½ æ‰æœƒé€™æ¨£åœ°æƒ³èµ·æˆ‘"); + } + + @Test + public void testCJKTwoSentencesWithLanguage() { + testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2è¬", "助妳好孕 AND 生1胎北市發2è¬", "zh-Hant"); + } + + @Test + public void testCJKTwoSentencesInPhrase() { + QueryMarshaller marshaller = new QueryMarshaller(); + testQueryString(marshaller, "\"助妳好孕 生1胎北市發2è¬\"", "\"助妳好孕 生1胎北市發2è¬\"", "zh-Hant"); + testQueryString(marshaller, "\"是ä¸æ˜¯é€™æ¨£çš„夜晚 ä½ æ‰æœƒé€™æ¨£åœ°æƒ³èµ·æˆ‘\"", "\"是ä¸æ˜¯é€™æ¨£çš„夜晚 ä½ æ‰æœƒé€™æ¨£åœ°æƒ³èµ·æˆ‘\""); + } + + @Test + public void testCJKMultipleSentences() { + testQueryString(new QueryMarshaller(), "염부장님과 í•¨ê»˜í–ˆë˜ ì¢‹ì€ ì¶”ì–µë“¤ì€", "염부장님과 AND í•¨ê»˜í–ˆë˜ AND ì¢‹ì€ AND 추억들ì€"); + } + + @Test + public void testIndexRestriction() { + /** ticket 3707606, comment #29 */ + testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\""); + } + + private void testQueryString(QueryMarshaller marshaller, String uq) { + testQueryString(marshaller, uq, uq, null); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq) { + testQueryString(marshaller, uq, mq, null); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) { + testQueryString(marshaller, uq, mq, lang, linguistics); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) { + Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : "")); + query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics))); + assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot())); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java new file mode 100644 index 00000000000..9135984b26b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.VespaSearcher; +import java.util.Map; + +/** + * Tests that source and backend specific parameters from the query are added correctly to the backend requests + * + * @author Jon Bratseth + */ +public class QueryParametersTestCase extends junit.framework.TestCase { + + public void testQueryParameters() { + Query query=new Query(); + query.properties().set("a","a-value"); + query.properties().set("b.c","b.c-value"); + query.properties().set("source.otherSource.d","d-value"); + query.properties().set("source.testSource.e","e-value"); + query.properties().set("source.testSource.f.g","f.g-value"); + query.properties().set("provider.testProvider.h","h-value"); + query.properties().set("provider.testProvider.i.j","i.j-value"); + + query.properties().set("sourceName","testSource"); // Done by federation searcher + query.properties().set("providerName","testProvider"); // Done by federation searcher + + VespaSearcher searcher=new VespaSearcher("testProvider","",0,""); + Map parameters=searcher.getQueryMap(query); + searcher.deconstruct(); + + assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above + assertEquals(parameters.get("e"),"e-value"); + assertEquals(parameters.get("f.g"),"f.g-value"); + assertEquals(parameters.get("h"),"h-value"); + assertEquals(parameters.get("i.j"),"i.j-value"); + } + +} + diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java new file mode 100644 index 00000000000..8cec6c64554 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import java.util.Iterator; + +import junit.framework.TestCase; + +import com.yahoo.net.URI; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.federation.vespa.ResultBuilder; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; + +/** + * Test XML parsing of results. + * + * @author Steinar Knutsen + */ +@SuppressWarnings("deprecation") +public class ResultBuilderTestCase extends TestCase { + + public ResultBuilderTestCase (String name) { + super(name); + } + + private boolean quickCompare(double a, double b) { + double z = Math.min(Math.abs(a), Math.abs(b)); + if (Math.abs((a - b)) < (z / 1e14)) { + return true; + } else { + return false; + } + } + + public void testSimpleResult() { + boolean gotErrorDetails = false; + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a")); + assertEquals(3, res.getConcreteHitCount()); + assertEquals(4, res.getHitCount()); + ErrorHit e = (ErrorHit) res.hits().get(0); + // known problem, if the same error is the main error is + // in details, it'll be added twice. Not sure how to fix that, + // because old Vespa systems give no error details, and there + // is no way of nuking an existing error if the details exist. + for (Iterator i = e.errorIterator(); i.hasNext();) { + ErrorMessage err = (ErrorMessage) i.next(); + assertEquals(5, err.getCode()); + String details = err.getDetailedMessage(); + if (details != null) { + gotErrorDetails = true; + assertEquals("An error as ordered", details.trim()); + } + } + assertTrue("Error details are missing", gotErrorDetails); + assertEquals(new URI("http://def"), res.hits().get(1).getId()); + assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category")); + assertEquals("habla" + + "blbl
<>&fdlkkgj</field>;lk" + + "", res.hits().get(3).getField("annoying").toString()); + } + + public void testNestedResult() { + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a")); + assertNull(res.hits().getError()); + assertEquals(3, res.hits().size()); + assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString()); + HitGroup g1 = (HitGroup) res.hits().get(1); + HitGroup g2 = (HitGroup) res.hits().get(2); + assertEquals(15, g1.size()); + assertEquals("reward_for_thumb", g1.get(1).getField("id").toString()); + assertEquals(10, g2.size()); + HitGroup g3 = (HitGroup) g2.get(3); + assertEquals("badge", g3.getTypeString()); + assertEquals(2, g3.size()); + assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString()); + } + + public void testWeirdDocumentID() { + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a")); + assertNull(res.hits().getError()); + assertEquals(3, res.hits().size()); + assertEquals(new URI("nalle"), res.hits().get(0).getId()); + assertEquals(new URI("tralle"), res.hits().get(1).getId()); + assertEquals(new URI("kalle"), res.hits().get(2).getId()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java new file mode 100644 index 00000000000..a1c3529e2e4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class VespaIntegrationTestCase extends junit.framework.TestCase { + + // TODO: Setup the answering vespa searcher from this test.... + public void testIt() { + if (System.currentTimeMillis() > 0) return; + Chain chain=new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,"")); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test")); + assertEquals(23,result.hits().size()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java new file mode 100644 index 00000000000..63da6adca77 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import junit.framework.TestCase; +import org.apache.http.HttpEntity; +import java.io.IOException; +import java.net.URI; + +/** + * Check query marshaling in VespaSearcher works... and stuff... + * + * @author Steinar Knutsen + */ +public class VespaSearcherTestCase extends TestCase { + + // TODO: More tests + + private VespaSearcher searcher; + + protected @Override void setUp() { + searcher = new VespaSearcher("cache1","",0,""); + } + + protected @Override void tearDown() { + searcher.deconstruct(); + } + + public void testMarshalQuery() { + RankItem root = new RankItem(); + QueryTree r = new QueryTree(root); + AndItem recall = new AndItem(); + PhraseItem usual = new PhraseItem(); + PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); + WordItem filterWord = new WordItem("silly"); + + filterPhrase.setFilter(true); + filterWord.setFilter(true); + + root.addItem(recall); + usual.addItem(new WordItem("new")); + usual.addItem(new WordItem("york")); + recall.addItem(usual); + recall.addItem(new WordItem("shoes")); + root.addItem(new WordItem("nike")); + root.addItem(new WordItem("adidas")); + root.addItem(filterPhrase); + recall.addItem(filterWord); + + assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r)); + } + + public void testMarshalQuerySmallTree() { + RankItem root = new RankItem(); + QueryTree r = new QueryTree(root); + AndItem recall = new AndItem(); + PhraseItem usual = new PhraseItem(); + PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); + WordItem filterWord = new WordItem("silly"); + + filterPhrase.setFilter(true); + filterWord.setFilter(true); + + root.addItem(recall); + usual.addItem(new WordItem("new")); + usual.addItem(new WordItem("york")); + recall.addItem(usual); + recall.addItem(new WordItem("shoes")); + root.addItem(filterPhrase); + recall.addItem(filterWord); + + assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r)); + // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly + // assertMarshals(root) + } + + public void testWandMarshalling() { + WeakAndItem root = new WeakAndItem(); + root.setN(32); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("c")); + assertMarshals(root); + } + + public void testWandMarshalling2() { + // AND (WAND(10) a!1 the!10) source:yahoonews + AndItem root = new AndItem(); + WeakAndItem wand = new WeakAndItem(10); + wand.addItem(newWeightedWordItem("a",1)); + wand.addItem(newWeightedWordItem("the",10)); + root.addItem(wand); + root.addItem(new WordItem("yahoonews","source")); + assertMarshals(root); + } + + private WordItem newWeightedWordItem(String word,int weight) { + WordItem wordItem=new WordItem(word); + wordItem.setWeight(weight); + return wordItem; + } + + private void assertMarshals(Item root) { + QueryTree r = new QueryTree(root); + String marshalledQuery=searcher.marshalQuery(r); + assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original", + r,parseQuery(marshalledQuery,"")); + } + + private static Item parseQuery(String query, String filter) { + Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment()); + return parser.parse(new Parsable().setQuery(query).setFilter(filter)); + } + + public void testSourceProviderProperties() throws Exception { + /* TODO: update test + Server httpServer = new Server(); + try { + SocketConnector listener = new SocketConnector(); + listener.setHost("0.0.0.0"); + httpServer.addConnector(listener); + httpServer.setHandler(new DummyHandler()); + httpServer.start(); + + int port=httpServer.getConnectors()[0].getLocalPort(); + + List sourcesConfig = new ArrayList(); + SourcesConfig.Source sourceConfig = new SourcesConfig.Source(); + sourceConfig.chain.setValue("news"); + sourceConfig.provider.setValue("news"); + sourceConfig.id.setValue("news"); + sourceConfig.timelimit.value = 10000; + sourcesConfig.add(sourceConfig); + FederationSearcher federator = + new FederationSearcher(ComponentId.createAnonymousComponentId(), + new ArrayList(sourcesConfig)); + SearchChain mainChain=new OrderedSearchChain(federator); + + SearchChainRegistry registry=new SearchChainRegistry(); + SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,"")); + registry.register(sourceChain); + Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1"); + Result result=new Execution(mainChain,registry).search(query); + assertNull(result.hits().getError()); + Hit testHit=result.hits().get("testHit"); + assertNotNull(testHit); + assertEquals("testValue",testHit.fields().get("testField")); + assertEquals("a1",testHit.fields().get("a")); + assertEquals("b1",testHit.fields().get("b")); + } + finally { + httpServer.stop(); + } + */ + } + + public void testVespaSearcher() { + VespaSearcher v=new VespaSearcherValidatingSubclass(); + new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter"))); + } + + private class VespaSearcherValidatingSubclass extends VespaSearcher { + + public VespaSearcherValidatingSubclass() { + super("configId","host",80,"path"); + } + + @Override + protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { + assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString()); + return super.getEntity(uri,requestMeta,query); + } + + } + + // used by the old testSourceProviderProperties() +// private class DummyHandler extends AbstractHandler { +// public void handle(String s, Request request, HttpServletRequest httpServletRequest, +// HttpServletResponse httpServletResponse) throws IOException, ServletException { +// +// try { +// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse(); +// +// httpResponse.setStatus(HttpStatus.OK_200); +// httpResponse.setContentType("text/xml"); +// httpResponse.setCharacterEncoding("UTF-8"); +// Result r=new Result(new Query()); +// Hit testHit=new Hit("testHit"); +// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate... +// testHit.setField("testField","testValue"); +// // Write back all incoming properties: +// for (Object e : httpServletRequest.getParameterMap().entrySet()) { +// Map.Entry entry=(Map.Entry)e; +// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue())); +// } +// +// r.hits().add(testHit); +// +// //StringWriter sw=new StringWriter(); +// //r.render(sw); +// //System.out.println(sw.toString()); +// +// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r); +// httpResponse.complete(); +// } +// catch (Exception e) { +// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e)); +// e.printStackTrace(); +// } +// } +// +// private String getFirstValue(Object entry) { +// if (entry instanceof String[]) +// return ((String[])entry)[0].toString(); +// else +// return entry.toString(); +// } +// } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml new file mode 100644 index 00000000000..b4b5c072eca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml @@ -0,0 +1,23 @@ + + + + + nalle + 75 + 0 + + + tralle + 73 + 0 + test/stuff\tsome/other + dklf øæå sdf > & < +Ipsum, etc. + + + kalle + 75 + 0 + hablablbl
&fdlkkgj]]>;lk + + diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml new file mode 100644 index 00000000000..c935f16528f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml @@ -0,0 +1,318 @@ + + + + +ABCDEFGHIJKLMNOPQRSTUVWXYZ +zero +0 +1287600988 +1287600988 + + + +thumb +1287600992 +1287600992 +0 +zero +1 + + +reward_for_thumb +1287600992 +1287600992 +0 +zero +1 + + +undo_thumb +1287600992 +1287600992 +0 +zero +1 + + + +buzz +1287600989 +1287600989 +0 +zero +1 + + + +undo_reward_for_thumb +1287600992 +1287600992 +0 +zero +1 + + + +vote +1287600989 +1287600989 +0 +zero +1 + + + +report_abuse +1287600992 +1287600992 +0 +zero +1 + + + +reward_for_vote +1287600989 +1287600989 +0 +zero +1 + + + +signup +1287600993 +1287600993 +0 +zero +1 + + + +registered +1287600989 +1287600989 +0 +zero +1 + + + +get_points +1287600989 +1287600989 +0 +zero +1 + + + +contrib_SignedUp +1287600993 +1287600993 +0 +zero +1 + + + +contrib_AgreedToTos +1287600993 +1287600993 +500 +zero +1 + + + +Create Feature + + +0 + +1 + + + +add_theme + + +0 + +1 + + + + + + + + +badge +badge/First Feature +You’ve created your First Feature! +active +http://example.yahoo.com/1stfeature.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/25th Feature +You’ve created your 25th Feature! +active +http://example.yahoo.com/25thfeature.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/50th Feature +You’ve created your 50th Feature! +active +http://example.yahoo.com/10thfeature.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Topic Explorer 5 +You’ve added a Feature to your 5th Topic Page! +active +http://example.yahoo.com/5thtopic.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Topic Explorer 15 +You’ve added a Feature to your 15th Topic Page! +active +http://example.yahoo.com/15thtopic.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Topic Explorer 30 +You’ve added a Feature to your 30th Topic Page! +active +http://example.yahoo.com/30thtopic.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Pollster +You’ve created your 5th Poll Feature. +active +http://example.yahoo.com/pollster.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + +badge +badge/Reporter +You’ve created your 5th Article Feature. +active +http://example.yahoo.com/newsreporter.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + +badge +badge/Paparazzi +You’ve created your 5th Image Feature. +active +http://example.yahoo.com/paparazzi.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + +badge +badge/Video Reporter +You’ve created your 5th Video Feature. +active +http://example.yahoo.com/director.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java new file mode 100644 index 00000000000..bc0d69f4bf7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author Simon Thoresen + */ +public class ContinuationTestCase { + + private static final String KNOWN_CONTINUATION = "BCBCBCBEBGBCBKCBACBKCCK"; + + @Test + public void requireThatToStringCanBeParsedByFromString() { + Continuation cnt = Continuation.fromString(KNOWN_CONTINUATION); + assertNotNull(cnt); + assertEquals(KNOWN_CONTINUATION, cnt.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java new file mode 100644 index 00000000000..19723dcd51a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.search.Query; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.searchchain.Execution; + +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.TimeZone; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class GroupingQueryParserTestCase { + + @Test + public void requireThatNoRequestIsSkipped() { + assertEquals(Collections.emptyList(), executeQuery(null, null, null)); + } + + @Test + public void requireThatEmptyRequestIsSkipped() { + assertEquals(Collections.emptyList(), executeQuery("", null, null)); + } + + @Test + public void requireThatRequestIsParsed() { + List lst = executeQuery("all(group(foo) each(output(max(bar))))", null, null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + assertNotNull(req.getRootOperation()); + } + + @Test + public void requireThatRequestListIsParsed() { + List lst = executeQuery("all();each()", null, null); + assertNotNull(lst); + assertEquals(2, lst.size()); + assertTrue(lst.get(0).getRootOperation() instanceof AllOperation); + assertTrue(lst.get(1).getRootOperation() instanceof EachOperation); + } + + @Test + public void requireThatEachRightBelowAllParses() { + List lst = executeQuery("all(each(output(summary(bar))))", + null, null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + final GroupingOperation rootOperation = req.getRootOperation(); + assertNotNull(rootOperation); + assertSame(AllOperation.class, rootOperation.getClass()); + assertSame(EachOperation.class, rootOperation.getChildren().get(0).getClass()); + } + + @Test + public void requireThatContinuationListIsParsed() { + List lst = executeQuery("all(group(foo) each(output(max(bar))))", + "BCBCBCBEBGBCBKCBACBKCCK BCBBBBBDBF", null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + assertNotNull(req.getRootOperation()); + assertEquals(2, req.continuations().size()); + } + + @Test + public void requireThatTimeZoneIsParsed() { + List lst = executeQuery("all(group(foo) each(output(max(bar))))", null, "cet"); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + TimeZone time = req.getTimeZone(); + assertNotNull(time); + assertEquals(TimeZone.getTimeZone("cet"), time); + } + + @Test + public void requireThatTimeZoneHasUtcDefault() { + List lst = executeQuery("all(group(foo) each(output(max(bar))))", null, null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + TimeZone time = req.getTimeZone(); + assertNotNull(time); + assertEquals(TimeZone.getTimeZone("utc"), time); + } + + private static List executeQuery(String request, String continuation, String timeZone) { + Query query = new Query(); + query.properties().set(GroupingQueryParser.PARAM_REQUEST, request); + query.properties().set(GroupingQueryParser.PARAM_CONTINUE, continuation); + query.properties().set(GroupingQueryParser.PARAM_TIMEZONE, timeZone); + new Execution(new GroupingQueryParser(), Execution.Context.createContextStub()).search(query); + return GroupingRequest.getRequests(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java new file mode 100644 index 00000000000..38e94092644 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.result.Hit; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class GroupingRequestTestCase { + + @Test + public void requireThatContinuationListIsMutable() { + GroupingRequest req = GroupingRequest.newInstance(new Query()); + assertTrue(req.continuations().isEmpty()); + + Continuation foo = new Continuation() { + + }; + req.continuations().add(foo); + assertEquals(Arrays.asList(foo), req.continuations()); + + req.continuations().clear(); + assertTrue(req.continuations().isEmpty()); + } + + @Test + public void requireThatResultIsFound() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + Result res = new Result(query); + + res.hits().add(new Hit("foo")); + RootGroup bar = newRootGroup(0); + req.setResultGroup(bar); + res.hits().add(bar); + res.hits().add(new Hit("baz")); + + Group grp = req.getResultGroup(res); + assertNotNull(grp); + assertSame(bar, grp); + } + + @Test + public void requireThatParallelRequestsAreSupported() { + Query query = new Query(); + Result res = new Result(query); + + GroupingRequest reqA = GroupingRequest.newInstance(query); + RootGroup grpA = newRootGroup(0); + reqA.setResultGroup(grpA); + res.hits().add(grpA); + + GroupingRequest reqB = GroupingRequest.newInstance(query); + RootGroup grpB = newRootGroup(1); + reqB.setResultGroup(grpB); + res.hits().add(grpB); + + Group grp = reqA.getResultGroup(res); + assertNotNull(grp); + assertSame(grpA, grp); + + assertNotNull(grp = reqB.getResultGroup(res)); + assertSame(grpB, grp); + } + + @Test + public void requireThatRemovedResultIsNull() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + Result res = new Result(query); + + res.hits().add(new Hit("foo")); + RootGroup bar = newRootGroup(0); + req.setResultGroup(bar); + res.hits().add(new Hit("baz")); + + assertNull(req.getResultGroup(res)); + } + + @Test + public void requireThatNonGroupResultIsNull() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + Result res = new Result(query); + + RootGroup grp = newRootGroup(0); + req.setResultGroup(grp); + res.hits().add(new Hit(grp.getId().toString())); + + assertNull(req.getResultGroup(res)); + } + + @Test + public void requireThatGetRequestsReturnsAllRequests() { + Query query = new Query(); + assertEquals(Collections.emptyList(), GroupingRequest.getRequests(query)); + + GroupingRequest foo = GroupingRequest.newInstance(query); + assertEquals(Arrays.asList(foo), GroupingRequest.getRequests(query)); + + GroupingRequest bar = GroupingRequest.newInstance(query); + assertEquals(Arrays.asList(foo, bar), GroupingRequest.getRequests(query)); + } + + @Test + public void requireThatGetRequestThrowsIllegalArgumentOnBadProperty() throws Exception { + Query query = new Query(); + Field propName = GroupingRequest.class.getDeclaredField("PROP_REQUEST"); + propName.setAccessible(true); + query.properties().set((CompoundName)propName.get(null), new Object()); + try { + GroupingRequest.getRequests(query); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + private static RootGroup newRootGroup(int id) { + return new RootGroup(id, new Continuation() { + + }); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java new file mode 100644 index 00000000000..38248bad6cf --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class GroupingValidatorTestCase { + + @Test + public void requireThatAvailableAttributesDoNotThrow() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + validateGrouping("myCluster", Arrays.asList("foo", "bar"), query); + } + + @Test + public void requireThatUnavailableAttributesThrow() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + try { + validateGrouping("myCluster", Arrays.asList("foo"), query); + fail("Validator should throw exception because attribute 'bar' is unavailable."); + } catch (UnavailableAttributeException e) { + assertEquals("myCluster", e.getClusterName()); + assertEquals("bar", e.getAttributeName()); + } + } + + @Test + public void requireThatEnableFlagPreventsThrow() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + query.properties().set(GroupingValidator.PARAM_ENABLED, "false"); + validateGrouping("myCluster", Arrays.asList("foo"), query); + } + + private static void validateGrouping(String clusterName, Collection attributeNames, Query query) { + QrSearchersConfig.Builder qrsConfig = new QrSearchersConfig.Builder().searchcluster( + new QrSearchersConfig.Searchcluster.Builder() + .indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.REALTIME) + .name(clusterName)); + ClusterConfig.Builder clusterConfig = new ClusterConfig.Builder(). + clusterId(0). + clusterName("test"); + AttributesConfig.Builder attributesConfig = new AttributesConfig.Builder(); + for (String attributeName : attributeNames) { + attributesConfig.attribute(new AttributesConfig.Attribute.Builder() + .name(attributeName)); + } + new Execution( + new GroupingValidator(new QrSearchersConfig(qrsConfig), + new ClusterConfig(clusterConfig), + new AttributesConfig(attributesConfig)), + Execution.Context.createContextStub()).search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java new file mode 100644 index 00000000000..c674a8a0755 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java @@ -0,0 +1,219 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.HitList; +import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.grouping.result.StringId; +import com.yahoo.search.query.Sorting; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author Andreas Eriksen + */ +public class UniqueGroupingSearcherTestCase { + + @Test + public void testSkipGroupingBasedDedup() throws Exception { + Result result = search("?query=foo", + new MockResultProvider(0, false)); + assertEquals(0, result.hits().size()); + } + + @Test + public void testSkipGroupingBasedDedupIfMultiLevelSorting() throws Exception { + Result result = search("?query=foo&unique=fingerprint&sorting=-pubdate%20-[rank]", + new MockResultProvider(0, false)); + assertEquals(0, result.hits().size()); + } + @Test + public void testIllegalSortingSpec() { + try { + search("?query=foo&unique=fingerprint&sorting=-1", + new MockResultProvider(0, true).addGroupList(new GroupList("fingerprint"))); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString( + "Invalid request parameter: Could not set 'ranking.sorting' to '-1': " + + "Illegal attribute name '1' for sorting. Requires '[\\[]*[a-zA-Z_][\\.a-zA-Z0-9_-]*[\\]]*'")); + } + } + + @Test + public void testGroupingBasedDedupNoGroupingHits() throws Exception { + Result result = search("?query=foo&unique=fingerprint", + new MockResultProvider(0, true)); + assertEquals(0, result.hits().size()); + } + + @Test + public void testGroupingBasedDedupWithEmptyGroupingHitsList() throws Exception { + Result result = search("?query=foo&unique=fingerprint", + new MockResultProvider(0, true).addGroupList(new GroupList("fingerprint"))); + assertEquals(0, result.hits().size()); + assertEquals(0, result.getTotalHitCount()); + } + + @Test + public void testGroupingBasedDedupWithNullGroupingResult() throws Exception { + try { + search("?query=foo&unique=fingerprint", + new MockResultProvider(0, false)); + fail(); + } catch (IllegalStateException e) { + assertEquals("Failed to produce deduped result set.", e.getMessage()); + } + } + + @Test + public void testGroupingBasedDedupWithGroupingHits() throws Exception { + GroupList fingerprint = new GroupList("fingerprint"); + fingerprint.add(makeHitGroup("1")); + fingerprint.add(makeHitGroup("2")); + fingerprint.add(makeHitGroup("3")); + fingerprint.add(makeHitGroup("4")); + fingerprint.add(makeHitGroup("5")); + fingerprint.add(makeHitGroup("6")); + fingerprint.add(makeHitGroup("7")); + + MockResultProvider mockResultProvider = new MockResultProvider(15, true); + mockResultProvider.addGroupList(fingerprint); + mockResultProvider.resultGroup.setField(UniqueGroupingSearcher.LABEL_COUNT, 42l); + Result result = search("?query=foo&unique=fingerprint&hits=5&offset=1", mockResultProvider); + assertEquals(5, result.hits().size()); + assertEquals("2", result.hits().get(0).getId().toString()); + assertEquals("3", result.hits().get(1).getId().toString()); + assertEquals("4", result.hits().get(2).getId().toString()); + assertEquals("5", result.hits().get(3).getId().toString()); + assertEquals("6", result.hits().get(4).getId().toString()); + assertEquals(42, result.getTotalHitCount()); + } + + @Test + public void testGroupingBasedDedupWithGroupingHitsAndSorting() throws Exception { + GroupList fingerprint = new GroupList("fingerprint"); + fingerprint.add(makeSortingHitGroup("1")); + fingerprint.add(makeSortingHitGroup("2")); + fingerprint.add(makeSortingHitGroup("3")); + fingerprint.add(makeSortingHitGroup("4")); + fingerprint.add(makeSortingHitGroup("5")); + fingerprint.add(makeSortingHitGroup("6")); + fingerprint.add(makeSortingHitGroup("7")); + + MockResultProvider mockResultProvider = new MockResultProvider(100, true); + mockResultProvider.addGroupList(fingerprint); + mockResultProvider.resultGroup.setField(UniqueGroupingSearcher.LABEL_COUNT, 1337l); + + Result result = search("?query=foo&unique=fingerprint&hits=5&offset=1&sorting=-expdate", mockResultProvider); + assertEquals(5, result.hits().size()); + assertEquals("2", result.hits().get(0).getId().toString()); + assertEquals("3", result.hits().get(1).getId().toString()); + assertEquals("4", result.hits().get(2).getId().toString()); + assertEquals("5", result.hits().get(3).getId().toString()); + assertEquals("6", result.hits().get(4).getId().toString()); + assertEquals(1337, result.getTotalHitCount()); + } + + @Test + public void testBuildGroupingExpression() throws Exception { + assertEquals("all(group(title) max(11) output(count() as(uniqueCount)) each(max(1) each(output(summary())) " + + "as(uniqueHits)))", + UniqueGroupingSearcher + .buildGroupingExpression("title", 11, null, null) + .toString()); + assertEquals("all(group(fingerprint) max(5) output(count() as(uniqueCount)) each(max(1) " + + "each(output(summary(attributeprefetch))) as(uniqueHits)))", + UniqueGroupingSearcher + .buildGroupingExpression("fingerprint", 5, "attributeprefetch", null) + .toString()); + assertEquals("all(group(fingerprint) max(5) order(neg(max(pubdate))) output(count() as(uniqueCount)) each(" + + "all(group(neg(pubdate)) max(1) order(neg(max(pubdate))) each(each(output(summary())) " + + "as(uniqueHits)) as(uniqueGroups))))", + UniqueGroupingSearcher + .buildGroupingExpression("fingerprint", 5, null, new Sorting("-pubdate")) + .toString()); + assertEquals("all(group(fingerprint) max(5) order(min(pubdate)) output(count() as(uniqueCount)) each(" + + "all(group(pubdate) max(1) order(min(pubdate)) each(each(output(summary(attributeprefetch))) " + + "as(uniqueHits)) as(uniqueGroups))))", + UniqueGroupingSearcher + .buildGroupingExpression("fingerprint", 5, "attributeprefetch", new Sorting("+pubdate")) + .toString()); + } + + private static Group makeHitGroup(String name) { + Group ein = new Group(new StringId(name), new Relevance(0)); + HitList hits = new HitList(UniqueGroupingSearcher.LABEL_HITS); + hits.add(new Hit(name)); + ein.add(hits); + return ein; + } + + private static Group makeSortingHitGroup(String name) { + Hit hit = new Hit(name); + + HitList hits = new HitList(UniqueGroupingSearcher.LABEL_HITS); + hits.add(hit); + + Group dedupGroup = new Group(new StringId(name), new Relevance(0)); + dedupGroup.add(hits); + + GroupList dedupedHits = new GroupList(UniqueGroupingSearcher.LABEL_GROUPS); + dedupedHits.add(dedupGroup); + + Group ein = new Group(new StringId(name), new Relevance(0)); + ein.add(dedupedHits); + return ein; + } + + private static Result search(String query, MockResultProvider result) { + return new Execution(new Chain<>(new UniqueGroupingSearcher(), result), + Execution.Context.createContextStub()).search(new Query(query)); + } + + private static class MockResultProvider extends Searcher { + + final RootGroup resultGroup; + final long totalHitCount; + final boolean addGroupingData; + + MockResultProvider(long totalHitCount, boolean addGroupingData) { + this.addGroupingData = addGroupingData; + this.resultGroup = new RootGroup(0, null); + this.totalHitCount = totalHitCount; + } + + MockResultProvider addGroupList(GroupList groupList) { + resultGroup.add(groupList); + return this; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = new Result(query); + if (addGroupingData) { + result.hits().add(resultGroup); + GroupingRequest.getRequests(query).get(0).setResultGroup(resultGroup); + result.setTotalHitCount(totalHitCount); + } + return result; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java new file mode 100644 index 00000000000..0ee23a3f37f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java @@ -0,0 +1,212 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.text.ChoiceFormat; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +@SuppressWarnings({ "rawtypes" }) +public class BucketResolverTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void testResolve() { + BucketResolver resolver = new BucketResolver(); + resolver.push(new StringValue("a"), true); + try { + resolver.resolve(new AttributeValue("foo")); + fail(); + } catch (IllegalStateException e) { + assertEquals("Missing to-limit of last bucket.", e.getMessage()); + } + + resolver.push(new StringValue("b"), false); + PredefinedFunction fnc = resolver.resolve(new AttributeValue("foo")); + assertNotNull(fnc); + assertEquals(1, fnc.getNumBuckets()); + BucketValue exp = fnc.getBucket(0); + assertNotNull(exp); + assertTrue(exp.getFrom() instanceof StringValue); + assertTrue(exp.getTo() instanceof StringValue); + BucketValue val = exp; + assertEquals("a", val.getFrom().getValue()); + assertEquals("b", val.getTo().getValue()); + + resolver.push(new StringValue("c"), true); + try { + resolver.resolve(new AttributeValue("foo")); + fail(); + } catch (IllegalStateException e) { + assertEquals("Missing to-limit of last bucket.", e.getMessage()); + } + + resolver.push(new StringValue("d"), false); + fnc = resolver.resolve(new AttributeValue("foo")); + assertNotNull(fnc); + assertEquals(2, fnc.getNumBuckets()); + assertNotNull(exp = fnc.getBucket(0)); + assertTrue(exp.getFrom() instanceof StringValue); + assertTrue(exp.getTo() instanceof StringValue); + val = exp; + assertEquals("a", val.getFrom().getValue()); + assertEquals("b", val.getTo().getValue()); + assertNotNull(exp = fnc.getBucket(1)); + assertTrue(exp.getFrom() instanceof StringValue); + assertTrue(exp.getTo() instanceof StringValue); + val = exp; + assertEquals("c", val.getFrom().getValue()); + assertEquals("d", val.getTo().getValue()); + } + + @Test + public void testBucketType() { + checkPushFail(Arrays.asList((ConstantValue)new StringValue("a"), new LongValue(1L)), + "Bucket type mismatch, expected 'StringValue' got 'LongValue'."); + checkPushFail(Arrays.asList((ConstantValue)new StringValue("a"), new DoubleValue(1.0)), + "Bucket type mismatch, expected 'StringValue' got 'DoubleValue'."); + checkPushFail(Arrays.asList((ConstantValue)new LongValue(1L), new StringValue("a")), + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + checkPushFail(Arrays.asList((ConstantValue)new LongValue(1L), new DoubleValue(1.0)), + "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); + checkPushFail(Arrays.asList((ConstantValue)new DoubleValue(1.0), new StringValue("a")), + "Bucket type mismatch, expected 'DoubleValue' got 'StringValue'."); + checkPushFail(Arrays.asList((ConstantValue)new DoubleValue(1.0), new LongValue(1L)), + "Bucket type mismatch, expected 'DoubleValue' got 'LongValue'."); + checkPushFail(Arrays.asList((ConstantValue)new InfiniteValue(new Infinite(true)), new InfiniteValue(new Infinite(false))), + "Bucket type mismatch, cannot both be infinity."); + + } + + @Test + public void testBucketOrder() { + checkPushFail(Arrays.asList((ConstantValue)new LongValue(2L), new LongValue(1L)), + "Bucket to-value can not be less than from-value."); + checkPushFail(Arrays.asList((ConstantValue)new DoubleValue(2.0), new DoubleValue(1.0)), + "Bucket to-value can not be less than from-value."); + checkPushFail(Arrays.asList((ConstantValue)new StringValue("b"), new StringValue("a")), + "Bucket to-value can not be less than from-value."); + } + + public void assertBucketRange(BucketValue expected, ConstantValue from, boolean inclusiveFrom, ConstantValue to, boolean inclusiveTo) { + BucketResolver resolver = new BucketResolver(); + resolver.push(from, inclusiveFrom); + resolver.push(to, inclusiveTo); + PredefinedFunction fnc = resolver.resolve(new AttributeValue("foo")); + assertNotNull(fnc); + BucketValue result = fnc.getBucket(0); + assertEquals(result.getFrom().getValue(), expected.getFrom().getValue()); + assertEquals(result.getTo().getValue(), expected.getTo().getValue()); + } + + public void assertBucketOrder(BucketResolver resolver) { + PredefinedFunction fnc = resolver.resolve(new AttributeValue("foo")); + BucketValue prev = null; + for (int i = 0; i < fnc.getNumBuckets(); i++) { + BucketValue b = fnc.getBucket(i); + if (prev != null) { + assertTrue(prev.compareTo(b) < 0); + } + prev = b; + } + } + + @Test + public void requireThatBucketRangesWork() { + BucketValue expected = new LongBucket(2, 5); + assertBucketRange(expected, new LongValue(1), false, new LongValue(4), true); + assertBucketRange(expected, new LongValue(1), false, new LongValue(5), false); + assertBucketRange(expected, new LongValue(2), true, new LongValue(4), true); + assertBucketRange(expected, new LongValue(2), true, new LongValue(5), false); + + + BucketResolver resolver = new BucketResolver(); + resolver.push(new LongValue(1), true).push(new LongValue(2), false); + resolver.push(new LongValue(2), true).push(new LongValue(4), true); + resolver.push(new LongValue(4), false).push(new LongValue(5), false); + resolver.push(new LongValue(5), false).push(new LongValue(8), true); + assertBucketOrder(resolver); + + + expected = new StringBucket("aba ", "bab "); + assertBucketRange(expected, new StringValue("aba"), false, new StringValue("bab"), true); + assertBucketRange(expected, new StringValue("aba"), false, new StringValue("bab "), false); + assertBucketRange(expected, new StringValue("aba "), true, new StringValue("bab"), true); + assertBucketRange(expected, new StringValue("aba "), true, new StringValue("bab "), false); + + resolver = new BucketResolver(); + resolver.push(new StringValue("aaa"), true).push(new StringValue("aab"), false); + resolver.push(new StringValue("aab"), true).push(new StringValue("aac"), true); + resolver.push(new StringValue("aac"), false).push(new StringValue("aad"), false); + resolver.push(new StringValue("aad"), false).push(new StringValue("aae"), true); + assertBucketOrder(resolver); + + RawBuffer r1 = new RawBuffer(new byte[]{0, 1, 3}); + RawBuffer r1next = new RawBuffer(new byte[]{0, 1, 3, 0}); + RawBuffer r2 = new RawBuffer(new byte[]{0, 2, 2}); + RawBuffer r2next = new RawBuffer(new byte[]{0, 2, 2, 0}); + RawBuffer r2nextnext = new RawBuffer(new byte[]{0, 2, 2, 0, 4}); + + expected = new RawBucket(r1next, r2next); + assertBucketRange(expected, new RawValue(r1), false, new RawValue(r2), true); + assertBucketRange(expected, new RawValue(r1), false, new RawValue(r2next), false); + assertBucketRange(expected, new RawValue(r1next), true, new RawValue(r2), true); + assertBucketRange(expected, new RawValue(r1next), true, new RawValue(r2next), false); + + resolver = new BucketResolver(); + resolver.push(new RawValue(r1), true).push(new RawValue(r1next), false); + resolver.push(new RawValue(r1next), true).push(new RawValue(r2), true); + resolver.push(new RawValue(r2), false).push(new RawValue(r2next), false); + resolver.push(new RawValue(r2next), false).push(new RawValue(r2nextnext), true); + assertBucketOrder(resolver); + + double d1next = ChoiceFormat.nextDouble(1.414); + double d2next = ChoiceFormat.nextDouble(3.14159); + double d1 = ChoiceFormat.nextDouble(d1next); + double d2 = ChoiceFormat.nextDouble(d2next); + expected = new DoubleBucket(d1, d2); + assertBucketRange(expected, new DoubleValue(d1next), false, new DoubleValue(d2next), true); + assertBucketRange(expected, new DoubleValue(d1next), false, new DoubleValue(d2), false); + assertBucketRange(expected, new DoubleValue(d1), true, new DoubleValue(d2next), true); + assertBucketRange(expected, new DoubleValue(d1), true, new DoubleValue(d2), false); + + resolver = new BucketResolver(); + resolver.push(new DoubleValue(d1next), true).push(new DoubleValue(d1), false); + resolver.push(new DoubleValue(d1), true).push(new DoubleValue(d2next), true); + resolver.push(new DoubleValue(d2next), false).push(new DoubleValue(d2), false); + resolver.push(new DoubleValue(d2), false).push(new DoubleValue(ChoiceFormat.nextDouble(d2)), true); + assertBucketOrder(resolver); + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static void checkPushFail(List args, String expectedException) { + BucketResolver resolver = new BucketResolver(); + try { + int i = 0; + for (ConstantValue exp : args) { + boolean inclusive = ((i % 2) == 0); + resolver.push(exp, inclusive); + i++; + } + fail(); + } catch (IllegalArgumentException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java new file mode 100644 index 00000000000..f5d30497671 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author Simon Thoresen + */ +public class ExpressionVisitorTestCase { + + @Test + public void requireThatExpressionsAreVisited() { + GroupingOperation op = new AllOperation(); + + final List lst = new LinkedList<>(); + GroupingExpression exp = new AttributeValue("groupBy"); + op.setGroupBy(exp); + lst.add(exp); + + op.addOrderBy(exp = new AttributeValue("orderBy1")); + lst.add(exp); + op.addOrderBy(exp = new AttributeValue("orderBy1")); + lst.add(exp); + + op.addOutput(exp = new AttributeValue("output1")); + lst.add(exp); + op.addOutput(exp = new AttributeValue("output2")); + lst.add(exp); + + op.visitExpressions(exp1 -> assertNotNull(lst.remove(exp1))); + assertTrue(lst.isEmpty()); + } + + @Test + public void requireThatChildOperationsAreVisited() { + GroupingOperation root, parentA, childA1, childA2, parentB, childB1; + root = new AllOperation() + .addChild(parentA = new AllOperation() + .addChild(childA1 = new AllOperation()) + .addChild(childA2 = new AllOperation())) + .addChild(parentB = new AllOperation() + .addChild(childB1 = new AllOperation())); + + final List lst = new LinkedList<>(); + GroupingExpression exp = new AttributeValue("parentA"); + parentA.setGroupBy(exp); + lst.add(exp); + + childA1.setGroupBy(exp = new AttributeValue("childA1")); + lst.add(exp); + + childA2.setGroupBy(exp = new AttributeValue("childA2")); + lst.add(exp); + + parentB.setGroupBy(exp = new AttributeValue("parentB")); + lst.add(exp); + + childB1.setGroupBy(exp = new AttributeValue("childB1")); + lst.add(exp); + + root.visitExpressions(exp1 -> assertNotNull(lst.remove(exp1))); + assertTrue(lst.isEmpty()); + } + + @Test + public void requireThatExpressionsArgumentsAreVisited() { + final List lst = new LinkedList<>(); + GroupingExpression arg1 = new AttributeValue("arg1"); + lst.add(arg1); + GroupingExpression arg2 = new AttributeValue("arg2"); + lst.add(arg2); + + new AndFunction(arg1, arg2).visit(exp -> assertNotNull(lst.remove(exp))); + assertTrue(lst.isEmpty()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java new file mode 100644 index 00000000000..614a126b54d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import com.yahoo.search.grouping.request.parser.ParseException; +import com.yahoo.search.grouping.request.parser.TokenMgrError; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class GroupingOperationTestCase { + + @Test + public void requireThatAccessorsWork() { + GroupingOperation op = new AllOperation(); + GroupingExpression exp = new AttributeValue("alias"); + op.putAlias("alias", exp); + assertSame(exp, op.getAlias("alias")); + + assertEquals(0, op.getHints().size()); + assertFalse(op.containsHint("foo")); + assertFalse(op.containsHint("bar")); + + op.addHint("foo"); + assertEquals(1, op.getHints().size()); + assertTrue(op.containsHint("foo")); + assertFalse(op.containsHint("bar")); + + op.addHint("bar"); + assertEquals(2, op.getHints().size()); + assertTrue(op.containsHint("foo")); + assertTrue(op.containsHint("bar")); + + op.setForceSinglePass(true); + assertTrue(op.getForceSinglePass()); + op.setForceSinglePass(false); + assertFalse(op.getForceSinglePass()); + + exp = new AttributeValue("orderBy"); + op.addOrderBy(exp); + assertEquals(1, op.getOrderBy().size()); + assertSame(exp, op.getOrderBy(0)); + + exp = new AttributeValue("output"); + op.addOutput(exp); + assertEquals(1, op.getOutputs().size()); + assertSame(exp, op.getOutput(0)); + + GroupingOperation child = new AllOperation(); + op.addChild(child); + assertEquals(1, op.getChildren().size()); + assertSame(child, op.getChild(0)); + + exp = new AttributeValue("groupBy"); + op.setGroupBy(exp); + assertSame(exp, op.getGroupBy()); + + op.setWhere("whereA"); + assertEquals("whereA", op.getWhere()); + op.setWhere("whereB"); + assertEquals("whereB", op.getWhere()); + + op.setAccuracy(0.6); + assertEquals(0.6, op.getAccuracy(), 1E-6); + op.setAccuracy(0.9); + assertEquals(0.9, op.getAccuracy(), 1E-6); + + op.setPrecision(6); + assertEquals(6, op.getPrecision()); + op.setPrecision(9); + assertEquals(9, op.getPrecision()); + + assertFalse(op.hasMax()); + op.setMax(6); + assertTrue(op.hasMax()); + assertEquals(6, op.getMax()); + op.setMax(9); + assertEquals(9, op.getMax()); + assertTrue(op.hasMax()); + op.setMax(0); + assertTrue(op.hasMax()); + op.setMax(-7); + assertFalse(op.hasMax()); + } + + @Test + public void requireThatFromStringAsListParsesAllOperations() { + List lst = GroupingOperation.fromStringAsList(""); + assertTrue(lst.isEmpty()); + + lst = GroupingOperation.fromStringAsList("all()"); + assertEquals(1, lst.size()); + assertTrue(lst.get(0) instanceof AllOperation); + + lst = GroupingOperation.fromStringAsList("each()"); + assertEquals(1, lst.size()); + assertTrue(lst.get(0) instanceof EachOperation); + + lst = GroupingOperation.fromStringAsList("all();each()"); + assertEquals(2, lst.size()); + assertTrue(lst.get(0) instanceof AllOperation); + assertTrue(lst.get(1) instanceof EachOperation); + } + + @Test + public void requireThatFromStringAcceptsOnlyOneOperation() { + try { + GroupingOperation.fromString(""); + fail(); + } catch (IllegalArgumentException e) { + + } + assertTrue(GroupingOperation.fromString("all()") instanceof AllOperation); + assertTrue(GroupingOperation.fromString("each()") instanceof EachOperation); + try { + GroupingOperation.fromString("all();each()"); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatParseExceptionsAreRethrown() { + try { + GroupingOperation.fromString("all(foo)"); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Encountered \"foo\" at line 1, column 5.\n")); + assertTrue(e.getCause() instanceof ParseException); + } + } + + @Test + public void requireThatTokenErrorsAreRethrown() { + try { + GroupingOperation.fromString("all(\\foo)"); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Lexical error at line 1, column 6.")); + assertTrue(e.getCause() instanceof TokenMgrError); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java new file mode 100644 index 00000000000..14274e98182 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; + +/** + * @author Einar M R Rosenvinge + * @since 5.1.9 + */ +public class MathFunctionsTestCase { + @Test + public void testMathFunctions() { + //if this fails, update the count AND add a test in each of the two blocks below + assertThat(MathFunctions.Function.values().length, is(21)); + + + assertThat(MathFunctions.Function.create(0), sameInstance(MathFunctions.Function.EXP)); + assertThat(MathFunctions.Function.create(1), sameInstance(MathFunctions.Function.POW)); + assertThat(MathFunctions.Function.create(2), sameInstance(MathFunctions.Function.LOG)); + assertThat(MathFunctions.Function.create(3), sameInstance(MathFunctions.Function.LOG1P)); + assertThat(MathFunctions.Function.create(4), sameInstance(MathFunctions.Function.LOG10)); + assertThat(MathFunctions.Function.create(5), sameInstance(MathFunctions.Function.SIN)); + assertThat(MathFunctions.Function.create(6), sameInstance(MathFunctions.Function.ASIN)); + assertThat(MathFunctions.Function.create(7), sameInstance(MathFunctions.Function.COS)); + assertThat(MathFunctions.Function.create(8), sameInstance(MathFunctions.Function.ACOS)); + assertThat(MathFunctions.Function.create(9), sameInstance(MathFunctions.Function.TAN)); + assertThat(MathFunctions.Function.create(10), sameInstance(MathFunctions.Function.ATAN)); + assertThat(MathFunctions.Function.create(11), sameInstance(MathFunctions.Function.SQRT)); + assertThat(MathFunctions.Function.create(12), sameInstance(MathFunctions.Function.SINH)); + assertThat(MathFunctions.Function.create(13), sameInstance(MathFunctions.Function.ASINH)); + assertThat(MathFunctions.Function.create(14), sameInstance(MathFunctions.Function.COSH)); + assertThat(MathFunctions.Function.create(15), sameInstance(MathFunctions.Function.ACOSH)); + assertThat(MathFunctions.Function.create(16), sameInstance(MathFunctions.Function.TANH)); + assertThat(MathFunctions.Function.create(17), sameInstance(MathFunctions.Function.ATANH)); + assertThat(MathFunctions.Function.create(18), sameInstance(MathFunctions.Function.CBRT)); + assertThat(MathFunctions.Function.create(19), sameInstance(MathFunctions.Function.HYPOT)); + assertThat(MathFunctions.Function.create(20), sameInstance(MathFunctions.Function.FLOOR)); + + + assertThat(MathFunctions.newInstance(MathFunctions.Function.EXP, null, null), instanceOf(MathExpFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.POW, null, null), instanceOf(MathPowFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.LOG, null, null), instanceOf(MathLogFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.LOG1P, null, null), instanceOf(MathLog1pFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.LOG10, null, null), instanceOf(MathLog10Function.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.SIN, null, null), instanceOf(MathSinFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ASIN, null, null), instanceOf(MathASinFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.COS, null, null), instanceOf(MathCosFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ACOS, null, null), instanceOf(MathACosFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.TAN, null, null), instanceOf(MathTanFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ATAN, null, null), instanceOf(MathATanFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.SQRT, null, null), instanceOf(MathSqrtFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.SINH, null, null), instanceOf(MathSinHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ASINH, null, null), instanceOf(MathASinHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.COSH, null, null), instanceOf(MathCosHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ACOSH, null, null), instanceOf(MathACosHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.TANH, null, null), instanceOf(MathTanHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ATANH, null, null), instanceOf(MathATanHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.CBRT, null, null), instanceOf(MathCbrtFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.HYPOT, null, null), instanceOf(MathHypotFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.FLOOR, null, null), instanceOf(MathFloorFunction.class)); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java new file mode 100644 index 00000000000..af007a6f85c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author Simon Thoresen + */ +public class MathResolverTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void testOperators() { + MathResolver resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.ADD, new LongValue(2)); + assertEquals("add(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.SUB, new LongValue(2)); + assertEquals("sub(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.DIV, new LongValue(2)); + assertEquals("div(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.MOD, new LongValue(2)); + assertEquals("mod(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.MUL, new LongValue(2)); + assertEquals("mul(1, 2)", + resolver.resolve().toString()); + } + + @Test + public void testOperatorPrecedence() { + assertResolve("add(add(1, 2), 3)", MathResolver.Type.ADD, MathResolver.Type.ADD); + assertResolve("add(1, sub(2, 3))", MathResolver.Type.ADD, MathResolver.Type.SUB); + assertResolve("add(1, div(2, 3))", MathResolver.Type.ADD, MathResolver.Type.DIV); + assertResolve("add(1, mod(2, 3))", MathResolver.Type.ADD, MathResolver.Type.MOD); + assertResolve("add(1, mul(2, 3))", MathResolver.Type.ADD, MathResolver.Type.MUL); + + assertResolve("add(sub(1, 2), 3)", MathResolver.Type.SUB, MathResolver.Type.ADD); + assertResolve("sub(sub(1, 2), 3)", MathResolver.Type.SUB, MathResolver.Type.SUB); + assertResolve("sub(1, div(2, 3))", MathResolver.Type.SUB, MathResolver.Type.DIV); + assertResolve("sub(1, mod(2, 3))", MathResolver.Type.SUB, MathResolver.Type.MOD); + assertResolve("sub(1, mul(2, 3))", MathResolver.Type.SUB, MathResolver.Type.MUL); + + assertResolve("add(div(1, 2), 3)", MathResolver.Type.DIV, MathResolver.Type.ADD); + assertResolve("sub(div(1, 2), 3)", MathResolver.Type.DIV, MathResolver.Type.SUB); + assertResolve("div(div(1, 2), 3)", MathResolver.Type.DIV, MathResolver.Type.DIV); + assertResolve("div(1, mod(2, 3))", MathResolver.Type.DIV, MathResolver.Type.MOD); + assertResolve("div(1, mul(2, 3))", MathResolver.Type.DIV, MathResolver.Type.MUL); + + assertResolve("add(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.ADD); + assertResolve("sub(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.SUB); + assertResolve("div(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.DIV); + assertResolve("mod(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.MOD); + assertResolve("mod(1, mul(2, 3))", MathResolver.Type.MOD, MathResolver.Type.MUL); + + assertResolve("add(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.ADD); + assertResolve("sub(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.SUB); + assertResolve("div(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.DIV); + assertResolve("mod(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.MOD); + assertResolve("mul(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.MUL); + + assertResolve("add(1, sub(div(2, mod(3, mul(4, 5))), 6))", + MathResolver.Type.ADD, MathResolver.Type.DIV, MathResolver.Type.MOD, + MathResolver.Type.MUL, MathResolver.Type.SUB); + assertResolve("add(sub(1, div(mod(mul(2, 3), 4), 5)), 6)", + MathResolver.Type.SUB, MathResolver.Type.MUL, MathResolver.Type.MOD, + MathResolver.Type.DIV, MathResolver.Type.ADD); + assertResolve("add(1, sub(2, div(3, mod(4, mul(5, 6)))))", + MathResolver.Type.ADD, MathResolver.Type.SUB, MathResolver.Type.DIV, + MathResolver.Type.MOD, MathResolver.Type.MUL); + assertResolve("add(sub(div(mod(mul(1, 2), 3), 4), 5), 6)", + MathResolver.Type.MUL, MathResolver.Type.MOD, MathResolver.Type.DIV, + MathResolver.Type.SUB, MathResolver.Type.ADD); + } + + @Test + public void testOperatorSupport() { + MathResolver resolver = new MathResolver(); + for (MathResolver.Type type : MathResolver.Type.values()) { + if (type == MathResolver.Type.ADD) { + continue; + } + try { + resolver.push(type, new AttributeValue("foo")); + } catch (IllegalArgumentException e) { + assertEquals("First item in an arithmetic operation must be an addition.", e.getMessage()); + } + } + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static void assertResolve(String expected, MathResolver.Type... types) { + MathResolver resolver = new MathResolver(); + + int val = 0; + resolver.push(MathResolver.Type.ADD, new LongValue(++val)); + for (MathResolver.Type type : types) { + resolver.push(type, new LongValue(++val)); + } + + GroupingExpression exp = resolver.resolve(); + assertNotNull(exp); + assertEquals(expected, exp.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java new file mode 100644 index 00000000000..eba5a458cfd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author Ulf Lilleengen + */ +public class RawBufferTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests. + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatCompareWorks() { + RawBuffer buffer = new RawBuffer(); + buffer.put((byte)'a'); + buffer.put((byte)'b'); + + RawBuffer buffer2 = new RawBuffer(); + buffer2.put((byte)'k'); + buffer2.put((byte)'a'); + + ArrayList backing = new ArrayList<>(); + backing.add((byte)'a'); + backing.add((byte)'b'); + RawBuffer buffer3 = new RawBuffer(backing); + + assertEquals(buffer.compareTo(buffer2), -1); + assertEquals(buffer2.compareTo(buffer), 1); + assertEquals(buffer.compareTo(buffer3), 0); + } + + @Test + public void requireThatToStringWorks() { + assertToString(Arrays.asList("a".getBytes()[0], "b".getBytes()[0]), "{97,98}"); + assertToString(Arrays.asList(new Byte((byte)2), new Byte((byte)6)), "{2,6}"); + } + + public void assertToString(List data, String expected) { + RawBuffer buffer = new RawBuffer(); + for (Byte b : data) { + buffer.put(b.byteValue()); + } + assertEquals(buffer.toString(), expected); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java new file mode 100644 index 00000000000..f2f8316f2db --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class RequestTestCase { + + @Test + public void requireThatApiWorks() { + GroupingOperation op = new AllOperation() + .setGroupBy(new AttributeValue("foo")) + .addOrderBy(new CountAggregator()) + .addChildren(Arrays.asList(new AllOperation(), new EachOperation())) + .addChild(new EachOperation() + .addOutput(new CountAggregator()) + .addOutput(new MinAggregator(new AttributeValue("bar"))) + .addChild(new EachOperation() + .addOutput(new AddFunction( + new LongValue(69), + new AttributeValue("baz"))) + .addOutput(new SummaryValue("cox")))); + assertEquals("all(group(foo) order(count()) all() each() " + + "each(output(count(), min(bar)) each(output(add(69, baz), summary(cox)))))", + op.toString()); + op.resolveLevel(1); + + GroupingExpression exp = op.getGroupBy(); + assertNotNull(exp); + assertTrue(exp instanceof AttributeValue); + assertEquals("foo", ((AttributeValue)exp).getAttributeName()); + assertEquals(1, op.getNumOrderBy()); + assertNotNull(exp = op.getOrderBy(0)); + assertTrue(exp instanceof CountAggregator); + + assertEquals(3, op.getNumChildren()); + assertTrue(op.getChild(0) instanceof AllOperation); + assertTrue(op.getChild(1) instanceof EachOperation); + assertNotNull(op = op.getChild(2)); + assertTrue(op instanceof EachOperation); + assertEquals(2, op.getNumOutputs()); + assertNotNull(exp = op.getOutput(0)); + assertTrue(exp instanceof CountAggregator); + assertNotNull(exp = op.getOutput(1)); + assertTrue(exp instanceof MinAggregator); + assertNotNull(exp = ((MinAggregator)exp).getExpression()); + assertTrue(exp instanceof AttributeValue); + assertEquals("bar", ((AttributeValue)exp).getAttributeName()); + + assertEquals(1, op.getNumChildren()); + assertNotNull(op = op.getChild(0)); + assertTrue(op instanceof EachOperation); + assertEquals(2, op.getNumOutputs()); + assertNotNull(exp = op.getOutput(0)); + assertTrue(exp instanceof AddFunction); + assertEquals(2, ((AddFunction)exp).getNumArgs()); + GroupingExpression arg = ((AddFunction)exp).getArg(0); + assertNotNull(arg); + assertTrue(arg instanceof LongValue); + assertEquals(69L, ((LongValue)arg).getValue().longValue()); + assertNotNull(arg = ((AddFunction)exp).getArg(1)); + assertTrue(arg instanceof AttributeValue); + assertEquals("baz", ((AttributeValue)arg).getAttributeName()); + assertNotNull(exp = op.getOutput(1)); + assertTrue(exp instanceof SummaryValue); + assertEquals("cox", ((SummaryValue)exp).getSummaryName()); + } + + @Test + public void requireThatPredefinedApiWorks() { + PredefinedFunction fnc = new LongPredefined(new AttributeValue("foo"), + new LongBucket(1, 2), + new LongBucket(3, 4)); + assertEquals(2, fnc.getNumBuckets()); + BucketValue bucket = fnc.getBucket(0); + assertNotNull(bucket); + assertTrue(bucket instanceof LongBucket); + assertEquals(1L, bucket.getFrom().getValue()); + assertEquals(2L, bucket.getTo().getValue()); + + assertNotNull(bucket = fnc.getBucket(1)); + assertTrue(bucket instanceof LongBucket); + assertEquals(3L, bucket.getFrom().getValue()); + assertEquals(4L, bucket.getTo().getValue()); + } + + @Test + public void requireThatBucketIntegrityIsChecked() { + try { + new LongBucket(2, 1); + } catch (IllegalArgumentException e) { + assertEquals("Bucket to-value can not be less than from-value.", e.getMessage()); + } + try { + new LongPredefined(new AttributeValue("foo"), + new LongBucket(3, 4), + new LongBucket(1, 2)); + } catch (IllegalArgumentException e) { + assertEquals("Buckets must be monotonically increasing, got bucket[3, 4> before bucket[1, 2>.", + e.getMessage()); + } + } + + @Test + public void requireThatAliasWorks() { + GroupingOperation all = new AllOperation(); + all.putAlias("myalias", new AttributeValue("foo")); + GroupingExpression exp = all.getAlias("myalias"); + assertNotNull(exp); + assertTrue(exp instanceof AttributeValue); + assertEquals("foo", ((AttributeValue)exp).getAttributeName()); + + GroupingOperation each = new EachOperation(); + all.addChild(each); + assertNotNull(exp = each.getAlias("myalias")); + assertTrue(exp instanceof AttributeValue); + assertEquals("foo", ((AttributeValue)exp).getAttributeName()); + + each.putAlias("myalias", new AttributeValue("bar")); + assertNotNull(exp = each.getAlias("myalias")); + assertTrue(exp instanceof AttributeValue); + assertEquals("bar", ((AttributeValue)exp).getAttributeName()); + } + + @Test + public void testOrderBy() { + GroupingOperation all = new AllOperation(); + all.addOrderBy(new AttributeValue("foo")); + try { + all.resolveLevel(0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Operation 'all(order(foo))' can not order single hit.", e.getMessage()); + } + all.resolveLevel(1); + assertEquals(0, all.getOrderBy(0).getLevel()); + } + + @Test + public void testMax() { + GroupingOperation all = new AllOperation(); + all.setMax(69); + try { + all.resolveLevel(0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Operation 'all(max(69))' can not apply max to single hit.", e.getMessage()); + } + all.resolveLevel(1); + } + + @Test + public void testAccuracy() { + GroupingOperation all = new AllOperation(); + all.setAccuracy(0.53); + assertEquals((long)(100.0 * all.getAccuracy()), 53); + try { + all.setAccuracy(1.2); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Illegal accuracy '1.2'. Must be between 0 and 1.", e.getMessage()); + } + try { + all.setAccuracy(-0.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Illegal accuracy '-0.5'. Must be between 0 and 1.", e.getMessage()); + } + } + + @Test + public void testLevelChange() { + GroupingOperation all = new AllOperation(); + all.resolveLevel(0); + assertEquals(0, all.getLevel()); + all.setGroupBy(new AttributeValue("foo")); + all.resolveLevel(1); + assertEquals(2, all.getLevel()); + + GroupingOperation each = new EachOperation(); + try { + each.resolveLevel(0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Operation '" + each + "' can not operate on single hit.", e.getMessage()); + } + each.resolveLevel(1); + assertEquals(0, each.getLevel()); + each.setGroupBy(new AttributeValue("foo")); + each.resolveLevel(2); + assertEquals(2, each.getLevel()); + } + + @Test + public void testLevelInheritance() { + GroupingOperation grandParent, parent, child, grandChild; + grandParent = new AllOperation() + .addChild(parent = new EachOperation() + .addChild(child = new AllOperation() + .addChild(grandChild = new EachOperation()))); + + grandParent.resolveLevel(69); + assertEquals(69, grandParent.getLevel()); + assertEquals(68, parent.getLevel()); + assertEquals(68, child.getLevel()); + assertEquals(67, grandChild.getLevel()); + } + + @Test + public void testLevelPropagation() { + GroupingOperation all = new AllOperation() + .setGroupBy(new AttributeValue("foo")) + .addOrderBy(new MaxAggregator(new AttributeValue("bar"))) + .addChild(new EachOperation() + .addOutput(new MaxAggregator(new AttributeValue("baz")))); + + all.resolveLevel(1); + assertEquals(0, all.getGroupBy().getLevel()); + assertEquals(1, all.getOrderBy(0).getLevel()); + assertEquals(1, all.getChild(0).getOutput(0).getLevel()); + assertEquals(0, ((AggregatorNode)all.getChild(0).getOutput(0)).getExpression().getLevel()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java new file mode 100644 index 00000000000..2abd4a01dd7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -0,0 +1,270 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request.parser; + +import com.yahoo.search.grouping.request.GroupingOperation; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author Simon Thoresen + */ +public class GroupingParserBenchmarkTest { + + private static final int NUM_RUNS = 10;//000; + private static final Map PREV_RESULTS = new LinkedHashMap<>(); + + static { + PREV_RESULTS.put("Original", 79276393L); + PREV_RESULTS.put("NoCache", 71728602L); + PREV_RESULTS.put("CharStream", 43852348L); + PREV_RESULTS.put("CharArray", 22936513L); + } + + @Test + public void requireThatGroupingParserIsFast() { + List inputs = getInputs(); + long ignore = 0; + long now = 0; + for (int i = 0; i < 2; ++i) { + now = System.nanoTime(); + for (int j = 0; j < NUM_RUNS; ++j) { + for (String str : inputs) { + ignore += parseRequest(str); + } + } + } + long micros = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - now); + System.out.format("%d \u03bcs (avg %.2f)\n", micros, (double)micros / (NUM_RUNS * inputs.size())); + for (Map.Entry entry : PREV_RESULTS.entrySet()) { + System.out.format("%-20s : %4.2f\n", entry.getKey(), (double)micros / entry.getValue()); + } + System.out.println("\nignore " + ignore); + } + + private static int parseRequest(String str) { + return GroupingOperation.fromStringAsList(str).size(); + } + + private static List getInputs() { + return Arrays.asList( + " all(group(foo)each(output(max(bar))))", + "all( group(foo)each(output(max(bar))))", + "all(group( foo)each(output(max(bar))))", + "all(group(foo )each(output(max(bar))))", + "all(group(foo) each(output(max(bar))))", + "all(group(foo)each( output(max(bar))))", + "all(group(foo)each(output( max(bar))))", + "all(group(foo)each(output(max(bar))))", + "all(group(foo)each(output(max( bar))))", + "all(group(foo)each(output(max(bar ))))", + "all(group(foo)each(output(max(bar) )))", + "all(group(foo)each(output(max(bar)) ))", + "all(group(foo)each(output(max(bar))) )", + "all(group(foo)each(output(max(bar)))) ", + "all()", + "each()", + "all(each())", + "each(all())", + "all(all() all())", + "all(all() each())", + "all(each() all())", + "all(each() each())", + "each(all() all())", + "all(group(foo))", + "all(max(1))", + "all(order(foo))", + "all(order(+foo))", + "all(order(-foo))", + "all(order(foo, bar))", + "all(order(foo, +bar))", + "all(order(foo, -bar))", + "all(order(+foo, bar))", + "all(order(+foo, +bar))", + "all(order(+foo, -bar))", + "all(order(-foo, bar))", + "all(order(-foo, +bar))", + "all(order(-foo, -bar))", + "all(output(min(a)))", + "all(output(min(a), min(b)))", + "all(precision(1))", + "all(where(foo))", + "all(where($foo))", + "all(group(fixedwidth(foo, 1)))", + "all(group(fixedwidth(foo, 1.2)))", + "all(group(md5(foo, 1)))", + "all(group(predefined(foo, bucket(1, 2))))", + "all(group(predefined(foo, bucket(-1, 2))))", + "all(group(predefined(foo, bucket(-2, -1))))", + "all(group(predefined(foo, bucket(1, 2), bucket(3, 4))))", + "all(group(predefined(foo, bucket(1, 2), bucket(3, 4), bucket(5, 6))))", + "all(group(predefined(foo, bucket(1, 2), bucket(2, 3), bucket(3, 4))))", + "all(group(predefined(foo, bucket(-100, 0), bucket(0), bucket<0, 100))))", + "all(group(predefined(foo, bucket[1, 2>)))", + "all(group(predefined(foo, bucket[-1, 2>)))", + "all(group(predefined(foo, bucket[-2, -1>)))", + "all(group(predefined(foo, bucket[1, 2>, bucket(3, 4>)))", + "all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>, bucket[5, 6>)))", + "all(group(predefined(foo, bucket[1, 2>, bucket[2, 3>, bucket[3, 4>)))", + "all(group(predefined(foo, bucket<1, 5>)))", + "all(group(predefined(foo, bucket[1, 5>)))", + "all(group(predefined(foo, bucket<1, 5])))", + "all(group(predefined(foo, bucket[1, 5])))", + "all(group(predefined(foo, bucket<1, inf>)))", + "all(group(predefined(foo, bucket<-inf, -1>)))", + "all(group(predefined(foo, bucket)))", + "all(group(predefined(foo, bucket<'a', inf>)))", + "all(group(predefined(foo, bucket<-inf, a>)))", + "all(group(predefined(foo, bucket[-inf, 'a'>)))", + "all(group(predefined(foo, bucket<-inf, -0.3>)))", + "all(group(predefined(foo, bucket<0.3, inf])))", + "all(group(predefined(foo, bucket<0.3, inf])))", + "all(group(predefined(foo, bucket)))", + "all(group(predefined(foo, bucket[1.0, 2.0>)))", + "all(group(predefined(foo, bucket<1.0, 2.0])))", + "all(group(predefined(foo, bucket[1.0, 2.0])))", + "all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>)))", + "all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>, bucket[5.0, 6.0>)))", + "all(group(predefined(foo, bucket[1.0, 2.0>, bucket[2.0], bucket<2.0, 6.0>)))", + "all(group(predefined(foo, bucket('a', 'b'))))", + "all(group(predefined(foo, bucket['a', 'b'>)))", + "all(group(predefined(foo, bucket<'a', 'c'>)))", + "all(group(predefined(foo, bucket<'a', 'b'])))", + "all(group(predefined(foo, bucket['a', 'b'])))", + "all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'))))", + "all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'), bucket('e', 'f'))))", + "all(group(predefined(foo, bucket(\"a\", \"b\"))))", + "all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"))))", + "all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"), bucket(\"e\", \"f\"))))", + "all(group(predefined(foo, bucket(a, b))))", + "all(group(predefined(foo, bucket(a, b), bucket(c, d))))", + "all(group(predefined(foo, bucket(a, b), bucket(c, d), bucket(e, f))))", + "all(group(predefined(foo, bucket(a, b), bucket(c), bucket(e, f))))", + "all(group(predefined(foo, bucket('a', \"b\"))))", + "all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'))))", + "all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'), bucket(\"e\", f))))", + "all(group(predefined(foo, bucket('a(', \"b)\"), bucket(c, 'd()'))))", + "all(group(predefined(foo, bucket({2}, {6}), bucket({7}, {12}))))", + "all(group(predefined(foo, bucket({0, 2}, {0, 6}), bucket({0, 7}, {0, 12}))))", + "all(group(predefined(foo, bucket({'b', 'a'}, {'k', 'a'}), bucket({'k', 'a'}, {'u', 'b'}))))", + "all(group(xorbit(foo, 1)))", + "all(group(1))", + "all(group(1+2))", + "all(group(1-2))", + "all(group(1*2))", + "all(group(1/2))", + "all(group(1%2))", + "all(group(1+2+3))", + "all(group(1+2-3))", + "all(group(1+2*3))", + "all(group(1+2/3))", + "all(group(1+2%3))", + "all(group((1+2)+3))", + "all(group((1+2)-3))", + "all(group((1+2)*3))", + "all(group((1+2)/3))", + "all(group((1+2)%3))", + "each() as(foo)", + "all(each() as(foo) each() as(bar))", + "all(group(a) each(each() as(foo) each() as(bar)) each() as(baz))", + "all(output(min(a) as(foo)))", + "all(output(min(a) as(foo), max(b) as(bar)))", + "all(where(bar) all(group(foo)))", + "all(group(foo)) where(bar)", + "all(group(attribute(foo)))", + "all(group(attribute(foo)) order(sum(attribute(a))))", + "all(accuracy(0.5))", + "all(group(foo) accuracy(1.0))", + "all(group(my.little{key}))", "all(group(my.little{\"key\"}))", + "all(group(my.little{key }))", "all(group(my.little{\"key\"}))", + "all(group(my.little{\"key\"}))", "all(group(my.little{\"key\"}))", + "all(group(my.little{\"key{}%\"}))", "all(group(my.little{\"key{}%\"}))", + "all(group(artist) max(7))", + "all(max(76) all(group(artist) max(7)))", + "all(group(artist) max(7) where(true))", + "all(group(artist) order(sum(a)) output(count()))", + "all(group(artist) order(+sum(a)) output(count()))", + "all(group(artist) order(-sum(a)) output(count()))", + "all(group(artist) order(-sum(a), +xor(b)) output(count()))", + "all(group(artist) max(1) output(count()))", + "all(group(-(m)) max(1) output(count()))", + "all(group(min) max(1) output(count()))", + "all(group(artist) max(2) each(each(output(summary()))))", + "all(group(artist) max(2) each(each(output(summary(simple)))))", + "all(group(artist) max(5) each(output(count()) each(output(summary()))))", + "all(group(ymum()))", + "all(group(strlen(attr)))", + "all(group(normalizesubject(attr)))", + "all(group(strcat(attr, attr2)))", + "all(group(tostring(attr)))", + "all(group(toraw(attr)))", + "all(group(zcurve.x(attr)))", + "all(group(zcurve.y(attr)))", + "all(group(uca(attr, \"foo\")))", + "all(group(uca(attr, \"foo\", \"PRIMARY\")))", + "all(group(uca(attr, \"foo\", \"SECONDARY\")))", + "all(group(uca(attr, \"foo\", \"TERTIARY\")))", + "all(group(uca(attr, \"foo\", \"QUATERNARY\")))", + "all(group(uca(attr, \"foo\", \"IDENTICAL\")))", + "all(group(tolong(attr)))", + "all(group(sort(attr)))", + "all(group(reverse(attr)))", + "all(group(docidnsspecific()))", + "all(group(relevance()))", + "all(group(artist) each(each(output(summary()))))", + "all(group(artist) max(13) " + + " each(group(fixedwidth(year, 21.34)) max(55) output(count()) " + + " each(each(output(summary())))))", + "all(group(artist) max(13) " + + " each(group(predefined(year, bucket(7, 19), bucket(90, 300))) max(55) output(count()) " + + " each(each(output(summary())))))", + "all(group(artist) max(13) " + + " each(group(predefined(year, bucket(7.1, 19.0), bucket(90.7, 300.0))) max(55) output(count()) " + + " each(each(output(summary())))))", + "all(group(artist) max(13) " + + " each(group(predefined(year, bucket('a', 'b'), bucket('peder', 'pedersen'))) " + + " max(55) output(count()) " + + " each(each(output(summary())))))", + "all(output(count()))", + "all(group(album) output(count()))", + "all(group(album) each(output(count())))", + "all(group(artist) each(group(album) output(count()))" + + " each(group(song) output(count())))", + "all(group(artist) output(count())" + + " each(group(album) output(count())" + + " each(group(song) output(count())" + + " each(each(output(summary()))))))", + "all(group(album) order(-$total=sum(length)) each(output($total)))", + "all(group(album) max(1) each(output(sum(length))))", + "all(group(artist) each(max(2) each(output(summary()))))", + "all(group(artist) max(3)" + + " each(group(album as(albumsongs)) each(each(output(summary()))))" + + " each(group(album as(albumlength)) output(sum(sum(length)))))", + "all(group(artist) max(15)" + + " each(group(album) " + + " each(group(song)" + + " each(max(2) each(output(summary()))))))", + "all(group(artist) max(15)" + + " each(group(album)" + + " each(group(song)" + + " each(max(2) each(output(summary())))))" + + " each(group(song) max(5) order(sum(popularity))" + + " each(output(sum(sold)) each(output(summary())))))", + "all(group(artist) order(max(relevance) * count()) each(output(count())))", + "all(group(artist) each(output(sum(popularity) / count())))", + "all(group(artist) accuracy(0.1) each(output(sum(popularity) / count())))", + "all(group(debugwait(artist, 3.3, true)))", + "all(group(debugwait(artist, 3.3, false)))"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java new file mode 100644 index 00000000000..c9fbcad28f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -0,0 +1,619 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request.parser; + +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.yql.VespaGroupingStep; +import com.yahoo.search.yql.YqlParser; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class GroupingParserTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests. + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatMathAllowsWhitespace() { + for (String op : Arrays.asList("+", " +", " + ", "+ ", + "-", " -", " - ", "- ", + "*", " *", " * ", "* ", + "/", " /", " / ", "/ ", + "%", " %", " % ", "% ")) { + assertParse("all(group(foo " + op + " 69) each(output(count())))"); + assertParse("all(group(foo " + op + " 6 " + op + " 9) each(output(count())))"); + assertParse("all(group(69 " + op + " foo) each(output(count())))"); + assertParse("all(group(6 " + op + " 9 " + op + " foo) each(output(count())))"); + } + } + + @Test + public void testRequestList() { + List lst = GroupingOperation.fromStringAsList("all();each();all() where(true);each()"); + assertNotNull(lst); + assertEquals(4, lst.size()); + assertTrue(lst.get(0) instanceof AllOperation); + assertTrue(lst.get(1) instanceof EachOperation); + assertTrue(lst.get(2) instanceof AllOperation); + assertTrue(lst.get(3) instanceof EachOperation); + } + + @Test + public void testAttributeFunctions() { + assertParse("all(group(foo) each(output(sum(attribute(bar)))))", + "all(group(foo) each(output(sum(attribute(bar)))))"); + assertParse("all(group(foo) each(output(sum(interpolatedlookup(bar, 0.25)))))", + "all(group(foo) each(output(sum(interpolatedlookup(bar, 0.25)))))"); + assertParse("all(group(foo) each(output(sum(array.at(bar, 42.0)))))", + "all(group(foo) each(output(sum(array.at(bar, 42.0)))))"); + } + + @Test + public void requireThatTokenImagesAreNotReservedWords() { + List images = Arrays.asList("acos", + "acosh", + "accuracy", + "add", + "alias", + "all", + "and", + "array", + "as", + "at", + "asin", + "asinh", + "atan", + "atanh", + "attribute", + "avg", + "bucket", + "cat", + "cbrt", + "cos", + "cosh", + "count", + "debugwait", + "div", + "docidnsspecific", + "each", + "exp", + "fixedwidth", + "floor", + "group", + "hint", + "hypot", + "log", + "log1p", + "log10", + "math", + "max", + "md5", + "min", + "mod", + "mul", + "neg", + "normalizesubject", + "now", + "or", + "order", + "output", + "pow", + "precision", + "predefined", + "relevance", + "reverse", + "sin", + "sinh", + "size", + "sort", + "interpolatedlookup", + "sqrt", + "strcat", + "strlen", + "sub", + "sum", + "summary", + "tan", + "tanh", + "time", + "date", + "dayofmonth", + "dayofweek", + "dayofyear", + "hourofday", + "minuteofhour", + "monthofyear", + "secondofminute", + "year", + "todouble", + "tolong", + "toraw", + "tostring", + "true", + "false", + "uca", + "where", + "x", + "xor", + "xorbit", + "y", + "ymum", + "zcurve"); + for (String image : images) { + assertParse("all(group(" + image + "))", "all(group(" + image + "))"); + } + } + + @Test + public void testTokenizedWhitespace() { + String expected = "all(group(foo) each(output(max(bar))))"; + + assertParse(" all(group(foo)each(output(max(bar))))", expected); + assertIllegalArgument("all (group(foo)each(output(max(bar))))", "Encountered \" \" at line 1, column 4."); + assertParse("all( group(foo)each(output(max(bar))))", expected); + assertIllegalArgument("all(group (foo)each(output(max(bar))))", "Encountered \" \" at line 1, column 10."); + assertParse("all(group( foo)each(output(max(bar))))", expected); + assertParse("all(group(foo )each(output(max(bar))))", expected); + assertParse("all(group(foo) each(output(max(bar))))", expected); + assertIllegalArgument("all(group(foo)each (output(max(bar))))", "Encountered \" \" at line 1, column 19."); + assertParse("all(group(foo)each( output(max(bar))))", expected); + assertIllegalArgument("all(group(foo)each(output (max(bar))))", "Encountered \" \" at line 1, column 26."); + assertParse("all(group(foo)each(output( max(bar))))", expected); + assertParse("all(group(foo)each(output(max(bar))))", expected); + assertParse("all(group(foo)each(output(max( bar))))", expected); + assertParse("all(group(foo)each(output(max(bar ))))", expected); + assertParse("all(group(foo)each(output(max(bar) )))", expected); + assertParse("all(group(foo)each(output(max(bar)) ))", expected); + assertParse("all(group(foo)each(output(max(bar))) )", expected); + assertParse("all(group(foo)each(output(max(bar)))) ", expected); + } + + @Test + public void testOperationTypes() { + assertParse("all()"); + assertParse("each()"); + assertParse("all(each())"); + assertParse("each(all())"); + assertParse("all(all() all())"); + assertParse("all(all() each())"); + assertParse("all(each() all())"); + assertParse("all(each() each())"); + assertParse("each(all() all())"); + assertIllegalArgument("each(all() each())", + "Operation 'each()' can not operate on single hit."); + assertIllegalArgument("each(group(foo) all() each())", + "Operation 'each(group(foo) all() each())' can not group single hit."); + assertIllegalArgument("each(each() all())", + "Operation 'each()' can not operate on single hit."); + assertIllegalArgument("each(group(foo) each() all())", + "Operation 'each(group(foo) each() all())' can not group single hit."); + assertIllegalArgument("each(each() each())", + "Operation 'each()' can not operate on single hit."); + assertIllegalArgument("each(group(foo) each() each())", + "Operation 'each(group(foo) each() each())' can not group single hit."); + } + + @Test + public void testOperationParts() { + assertParse("all(group(foo))"); + assertParse("all(hint(foo))"); + assertParse("all(hint(foo) hint(bar))"); + assertParse("all(max(1))"); + assertParse("all(order(foo))"); + assertParse("all(order(+foo))"); + assertParse("all(order(-foo))"); + assertParse("all(order(foo, bar))"); + assertParse("all(order(foo, +bar))"); + assertParse("all(order(foo, -bar))"); + assertParse("all(order(+foo, bar))"); + assertParse("all(order(+foo, +bar))"); + assertParse("all(order(+foo, -bar))"); + assertParse("all(order(-foo, bar))"); + assertParse("all(order(-foo, +bar))"); + assertParse("all(order(-foo, -bar))"); + assertParse("all(output(min(a)))"); + assertParse("all(output(min(a), min(b)))"); + assertParse("all(precision(1))"); + assertParse("all(where(foo))"); + assertParse("all(where($foo))"); + } + + @Test + public void testComplexExpressionTypes() { + // fixedwidth + assertParse("all(group(fixedwidth(foo, 1)))"); + assertParse("all(group(fixedwidth(foo, 1.2)))"); + + // md5 + assertParse("all(group(md5(foo, 1)))"); + + // predefined + assertParse("all(group(predefined(foo, bucket(1, 2))))"); + assertParse("all(group(predefined(foo, bucket(-1, 2))))"); + assertParse("all(group(predefined(foo, bucket(-2, -1))))"); + assertParse("all(group(predefined(foo, bucket(1, 2), bucket(3, 4))))"); + assertParse("all(group(predefined(foo, bucket(1, 2), bucket(3, 4), bucket(5, 6))))"); + assertParse("all(group(predefined(foo, bucket(1, 2), bucket(2, 3), bucket(3, 4))))"); + assertParse("all(group(predefined(foo, bucket(-100, 0), bucket(0), bucket<0, 100))))"); + + assertParse("all(group(predefined(foo, bucket[1, 2>)))"); + assertParse("all(group(predefined(foo, bucket[-1, 2>)))"); + assertParse("all(group(predefined(foo, bucket[-2, -1>)))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket(3, 4>)))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>, bucket[5, 6>)))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket[2, 3>, bucket[3, 4>)))"); + + assertParse("all(group(predefined(foo, bucket<1, 5>)))"); + assertParse("all(group(predefined(foo, bucket[1, 5>)))"); + assertParse("all(group(predefined(foo, bucket<1, 5])))"); + assertParse("all(group(predefined(foo, bucket[1, 5])))"); + + assertParse("all(group(predefined(foo, bucket<1, inf>)))"); + assertParse("all(group(predefined(foo, bucket<-inf, -1>)))"); + assertParse("all(group(predefined(foo, bucket)))"); + assertParse("all(group(predefined(foo, bucket<'a', inf>)))"); + assertParse("all(group(predefined(foo, bucket<-inf, a>)))"); + assertParse("all(group(predefined(foo, bucket[-inf, 'a'>)))"); + assertParse("all(group(predefined(foo, bucket<-inf, -0.3>)))"); + assertParse("all(group(predefined(foo, bucket<0.3, inf])))"); + assertParse("all(group(predefined(foo, bucket<0.3, inf])))"); + assertParse("all(group(predefined(foo, bucket)))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>)))"); + assertParse("all(group(predefined(foo, bucket<1.0, 2.0])))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0])))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>)))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>, bucket[5.0, 6.0>)))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>, bucket[2.0], bucket<2.0, 6.0>)))"); + + assertParse("all(group(predefined(foo, bucket('a', 'b'))))"); + assertParse("all(group(predefined(foo, bucket['a', 'b'>)))"); + assertParse("all(group(predefined(foo, bucket<'a', 'c'>)))"); + assertParse("all(group(predefined(foo, bucket<'a', 'b'])))"); + assertParse("all(group(predefined(foo, bucket['a', 'b'])))"); + assertParse("all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'))))"); + assertParse("all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'), bucket('e', 'f'))))"); + + assertParse("all(group(predefined(foo, bucket(\"a\", \"b\"))))"); + assertParse("all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"))))"); + assertParse("all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"), bucket(\"e\", \"f\"))))"); + + assertParse("all(group(predefined(foo, bucket(a, b))))"); + assertParse("all(group(predefined(foo, bucket(a, b), bucket(c, d))))"); + assertParse("all(group(predefined(foo, bucket(a, b), bucket(c, d), bucket(e, f))))"); + assertParse("all(group(predefined(foo, bucket(a, b), bucket(c), bucket(e, f))))"); + + assertParse("all(group(predefined(foo, bucket('a', \"b\"))))"); + assertParse("all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'))))"); + assertParse("all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'), bucket(\"e\", f))))"); + + assertParse("all(group(predefined(foo, bucket('a(', \"b)\"), bucket(c, 'd()'))))"); + assertParse("all(group(predefined(foo, bucket({2}, {6}), bucket({7}, {12}))))"); + assertParse("all(group(predefined(foo, bucket({0, 2}, {0, 6}), bucket({0, 7}, {0, 12}))))"); + assertParse("all(group(predefined(foo, bucket({'b', 'a'}, {'k', 'a'}), bucket({'k', 'a'}, {'u', 'b'}))))"); + + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2.0))))", + "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, '2'))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket(3.0, 4.0))))", + "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket('3', '4'))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket(\"3\", \"4\"))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket(three, four))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket<-inf, inf>)))", + "Bucket type mismatch, cannot both be infinity"); + assertIllegalArgument("all(group(predefined(foo, bucket)))", + "Encountered \"inf\" at line 1, column 34."); + + assertIllegalArgument("all(group(predefined(foo, bucket(2, 1))))", + "Bucket to-value can not be less than from-value."); + assertIllegalArgument("all(group(predefined(foo, bucket(3, 4), bucket(1, 2))))", + "Buckets must be monotonically increasing, got bucket[3, 4> before bucket[1, 2>."); + assertIllegalArgument("all(group(predefined(foo, bucket(b, a))))", + "Bucket to-value can not be less than from-value."); + assertIllegalArgument("all(group(predefined(foo, bucket(b, -inf))))", + "Encountered \"-inf\" at line 1, column 37."); + assertIllegalArgument("all(group(predefined(foo, bucket(c, d), bucket(a, b))))", + "Buckets must be monotonically increasing, got bucket[\"c\", \"d\"> before bucket[\"a\", \"b\">."); + assertIllegalArgument("all(group(predefined(foo, bucket(c, d), bucket(-inf, e))))", + "Buckets must be monotonically increasing, got bucket[\"c\", \"d\"> before bucket[-inf, \"e\">."); + assertIllegalArgument("all(group(predefined(foo, bucket(u, inf), bucket(e, i))))", + "Buckets must be monotonically increasing, got bucket[\"u\", inf> before bucket[\"e\", \"i\">."); + + // xorbit + assertParse("all(group(xorbit(foo, 1)))"); + } + + @Test + public void testInfixArithmetic() { + assertParse("all(group(1))", "all(group(1))"); + assertParse("all(group(1+2))", "all(group(add(1, 2)))"); + assertParse("all(group(1-2))", "all(group(sub(1, 2)))"); + assertParse("all(group(1*2))", "all(group(mul(1, 2)))"); + assertParse("all(group(1/2))", "all(group(div(1, 2)))"); + assertParse("all(group(1%2))", "all(group(mod(1, 2)))"); + assertParse("all(group(1+2+3))", "all(group(add(add(1, 2), 3)))"); + assertParse("all(group(1+2-3))", "all(group(add(1, sub(2, 3))))"); + assertParse("all(group(1+2*3))", "all(group(add(1, mul(2, 3))))"); + assertParse("all(group(1+2/3))", "all(group(add(1, div(2, 3))))"); + assertParse("all(group(1+2%3))", "all(group(add(1, mod(2, 3))))"); + assertParse("all(group((1+2)+3))", "all(group(add(add(1, 2), 3)))"); + assertParse("all(group((1+2)-3))", "all(group(sub(add(1, 2), 3)))"); + assertParse("all(group((1+2)*3))", "all(group(mul(add(1, 2), 3)))"); + assertParse("all(group((1+2)/3))", "all(group(div(add(1, 2), 3)))"); + assertParse("all(group((1+2)%3))", "all(group(mod(add(1, 2), 3)))"); + } + + @Test + public void testOperationLabel() { + assertParse("each() as(foo)", + "each() as(foo)"); + assertParse("all(each() as(foo)" + + " each() as(bar))", + "all(each() as(foo) each() as(bar))"); + assertParse("all(group(a) each(each() as(foo)" + + " each() as(bar))" + + " each() as(baz))", + "all(group(a) each(each() as(foo) each() as(bar)) each() as(baz))"); + + assertIllegalArgument("all() as(foo)", "Encountered \"as\" at line 1, column 7."); + assertIllegalArgument("all(all() as(foo))", "Encountered \"as\" at line 1, column 11."); + assertIllegalArgument("each(all() as(foo))", "Encountered \"as\" at line 1, column 12."); + } + + @Test + public void testAttributeName() { + assertParse("all(group(foo))"); + assertIllegalArgument("all(group(foo.))", + "Encountered \")\" at line 1, column 15."); + assertParse("all(group(foo.bar))"); + assertIllegalArgument("all(group(foo.bar.))", + "Encountered \")\" at line 1, column 19."); + assertParse("all(group(foo.bar.baz))"); + } + + @Test + public void testOutputLabel() { + assertParse("all(output(min(a) as(foo)))", + "all(output(min(a) as(foo)))"); + assertParse("all(output(min(a) as(foo), max(b) as(bar)))", + "all(output(min(a) as(foo), max(b) as(bar)))"); + + assertIllegalArgument("all(output(min(a)) as(foo))", + "Encountered \"as\" at line 1, column 20."); + } + + @Test + public void testRootWhere() { + String expected = "all(where(bar) all(group(foo)))"; + assertParse("all(where(bar) all(group(foo)))", expected); + assertParse("all(group(foo)) where(bar)", expected); + } + + @Test + public void testParseBadRequest() { + assertIllegalArgument("output(count())", + "Encountered \"output\" at line 1, column 1."); + assertIllegalArgument("each(output(count()))", + "Expression 'count()' not applicable for single hit."); + assertIllegalArgument("all(output(count())))", + "Encountered \")\" at line 1, column 21."); + } + + @Test + public void testAttributeFunction() { + assertParse("all(group(attribute(foo)))"); + assertParse("all(group(attribute(foo)) order(sum(attribute(a))))"); + } + + @Test + public void testAccuracy() { + assertParse("all(accuracy(0.5))"); + assertParse("all(group(foo) accuracy(1.0))"); + } + + @Test + public void testMapSyntax() { + assertParse("all(group(my.little{key}))", "all(group(my.little{\"key\"}))"); + assertParse("all(group(my.little{key }))", "all(group(my.little{\"key\"}))"); + assertParse("all(group(my.little{\"key\"}))", "all(group(my.little{\"key\"}))"); + assertParse("all(group(my.little{\"key{}%\"}))", "all(group(my.little{\"key{}%\"}))"); + } + + @Test + public void testMisc() { + for (String fnc : Arrays.asList("time.date", + "time.dayofmonth", + "time.dayofweek", + "time.dayofyear", + "time.hourofday", + "time.minuteofhour", + "time.monthofyear", + "time.secondofminute", + "time.year")) { + assertParse("each(output(" + fnc + "(foo)))"); + } + + assertParse("all(group(artist) max(7))"); + assertParse("all(max(76) all(group(artist) max(7)))"); + assertParse("all(group(artist) max(7) where(true))"); + assertParse("all(group(artist) order(sum(a)) output(count()))"); + assertParse("all(group(artist) order(+sum(a)) output(count()))"); + assertParse("all(group(artist) order(-sum(a)) output(count()))"); + assertParse("all(group(artist) order(-sum(a), +xor(b)) output(count()))"); + assertParse("all(group(artist) max(1) output(count()))"); + assertParse("all(group(-(m)) max(1) output(count()))"); + assertParse("all(group(min) max(1) output(count()))"); + assertParse("all(group(artist) max(2) each(each(output(summary()))))"); + assertParse("all(group(artist) max(2) each(each(output(summary(simple)))))"); + assertParse("all(group(artist) max(5) each(output(count()) each(output(summary()))))"); + assertParse("all(group(ymum()))"); + assertParse("all(group(strlen(attr)))"); + assertParse("all(group(normalizesubject(attr)))"); + assertParse("all(group(strcat(attr, attr2)))"); + assertParse("all(group(tostring(attr)))"); + assertParse("all(group(toraw(attr)))"); + assertParse("all(group(zcurve.x(attr)))"); + assertParse("all(group(zcurve.y(attr)))"); + assertParse("all(group(uca(attr, \"foo\")))"); + assertParse("all(group(uca(attr, \"foo\", \"PRIMARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"SECONDARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"TERTIARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"QUATERNARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"IDENTICAL\")))"); + assertIllegalArgument("all(group(uca(attr, \"foo\", \"foo\")))", "Not a valid UCA strength: foo"); + assertParse("all(group(tolong(attr)))"); + assertParse("all(group(sort(attr)))"); + assertParse("all(group(reverse(attr)))"); + assertParse("all(group(docidnsspecific()))"); + assertParse("all(group(relevance()))"); + // TODO: assertParseRequest("all(group(a) each(output(xor(md5(b)) xor(md5(b, 0, 64)))))"); + // TODO: assertParseRequest("all(group(a) each(output(xor(xorbit(b)) xor(xorbit(b, 64)))))"); + assertParse("all(group(artist) each(each(output(summary()))))"); + assertParse("all(group(artist) max(13) each(group(fixedwidth(year, 21.34)) max(55) output(count()) " + + "each(each(output(summary())))))"); + assertParse("all(group(artist) max(13) each(group(predefined(year, bucket(7, 19), bucket(90, 300))) " + + "max(55) output(count()) each(each(output(summary())))))"); + assertParse("all(group(artist) max(13) each(group(predefined(year, bucket(7.1, 19.0), bucket(90.7, 300.0))) " + + "max(55) output(count()) each(each(output(summary())))))"); + assertParse("all(group(artist) max(13) each(group(predefined(year, bucket('a', 'b'), bucket('cd', 'cde'))) " + + "max(55) output(count()) each(each(output(summary())))))"); + + assertParse("all(output(count()))"); + assertParse("all(group(album) output(count()))"); + assertParse("all(group(album) each(output(count())))"); + assertParse("all(group(artist) each(group(album) output(count()))" + + " each(group(song) output(count())))"); + assertParse("all(group(artist) output(count())" + + " each(group(album) output(count())" + + " each(group(song) output(count())" + + " each(each(output(summary()))))))"); + assertParse("all(group(album) order(-$total=sum(length)) each(output($total)))"); + assertParse("all(group(album) max(1) each(output(sum(length))))"); + assertParse("all(group(artist) each(max(2) each(output(summary()))))"); + assertParse("all(group(artist) max(3)" + + " each(group(album as(albumsongs)) each(each(output(summary()))))" + + " each(group(album as(albumlength)) output(sum(sum(length)))))"); + assertParse("all(group(artist) max(15)" + + " each(group(album) " + + " each(group(song)" + + " each(max(2) each(output(summary()))))))"); + assertParse("all(group(artist) max(15)" + + " each(group(album)" + + " each(group(song)" + + " each(max(2) each(output(summary())))))" + + " each(group(song) max(5) order(sum(popularity))" + + " each(output(sum(sold)) each(output(summary())))))"); + + assertParse("all(group(artist) order(max(relevance) * count()) each(output(count())))"); + assertParse("all(group(artist) each(output(sum(popularity) / count())))"); + assertParse("all(group(artist) accuracy(0.1) each(output(sum(popularity) / count())))"); + assertParse("all(group(debugwait(artist, 3.3, true)))"); + assertParse("all(group(debugwait(artist, 3.3, false)))"); + assertIllegalArgument("all(group(debugwait(artist, -3.3, true)))", + "Encountered \"-\" at line 1, column 29"); + assertIllegalArgument("all(group(debugwait(artist, 3.3, lol)))", + "Encountered \"lol\" at line 1, column 34"); + } + + @Test + public void requireThatParseExceptionMessagesContainErrorMarker() { + assertIllegalArgument("foo", + "Encountered \"foo\" at line 1, column 1.\n" + + "Was expecting one of:\n" + + " ...\n" + + " \"all\" ...\n" + + " \"each\" ...\n" + + " \n" + + "At position:\n" + + "foo\n" + + "^"); + assertIllegalArgument("\n foo", + "Encountered \"foo\" at line 2, column 2.\n" + + "Was expecting one of:\n" + + " ...\n" + + " \"all\" ...\n" + + " \"each\" ...\n" + + " \n" + + "At position:\n" + + " foo\n" + + " ^"); + } + + // -------------------------------------------------------------------------------- + // + // Utilities. + // + // -------------------------------------------------------------------------------- + + private static void assertParse(String request, String... expectedOperations) { + List operations = GroupingOperation.fromStringAsList(request); + List actual = new ArrayList<>(operations.size()); + for (GroupingOperation operation : operations) { + operation.resolveLevel(1); + actual.add(operation.toString()); + } + if (expectedOperations.length > 0) { + assertEquals(Arrays.asList(expectedOperations), actual); + } + + // make sure that operation does not mutate through toString() -> fromString() + for (GroupingOperation operation : operations) { + assertEquals(operation.toString(), GroupingOperation.fromString(operation.toString()).toString()); + } + + // make sure that yql+ is capable of handling request + assertYqlParsable(request, expectedOperations); + } + + private static void assertYqlParsable(String request, String... expectedOperations) { + YqlParser parser = new YqlParser(new ParserEnvironment()); + parser.parse(new Parsable().setQuery("select foo from bar where baz contains 'baz' | " + request + ";")); + List steps = parser.getGroupingSteps(); + List actual = new ArrayList<>(steps.size()); + for (VespaGroupingStep step : steps) { + actual.add(step.getOperation().toString()); + } + if (expectedOperations.length > 0) { + assertEquals(Arrays.asList(expectedOperations), actual); + } + } + + private static void assertIllegalArgument(String request, String expectedException) { + try { + GroupingOperation.fromString(request).resolveLevel(1); + fail("Expected: " + expectedException); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().startsWith(expectedException)); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java new file mode 100644 index 00000000000..e3bccde2767 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class GroupIdTestCase { + + @Test + public void requireThatAccessorsWork() { + ValueGroupId valueId = new DoubleId(6.9); + assertEquals(6.9, valueId.getValue()); + BucketGroupId rangeId = new DoubleBucketId(6.0, 9.0); + assertEquals(6.0, rangeId.getFrom()); + assertEquals(9.0, rangeId.getTo()); + + valueId = new LongId(69L); + assertEquals(69L, valueId.getValue()); + rangeId = new LongBucketId(6L, 9L); + assertEquals(6L, rangeId.getFrom()); + assertEquals(9L, rangeId.getTo()); + + valueId = new RawId(new byte[] { 6, 9 }); + assertArrayEquals(new byte[] { 6, 9 }, (byte[])valueId.getValue()); + rangeId = new RawBucketId(new byte[] { 6, 9 }, new byte[] { 9, 6 }); + assertArrayEquals(new byte[] { 6, 9 }, (byte[])rangeId.getFrom()); + assertArrayEquals(new byte[] { 9, 6 }, (byte[])rangeId.getTo()); + + valueId = new StringId("69"); + assertEquals("69", valueId.getValue()); + rangeId = new StringBucketId("6", "9"); + assertEquals("6", rangeId.getFrom()); + assertEquals("9", rangeId.getTo()); + } + + @Test + public void requireThatToStringCorrespondsToType() { + assertEquals("group:double:6.9", new DoubleId(6.9).toString()); + assertEquals("group:double_bucket:6.0:9.0", new DoubleBucketId(6.0, 9.0).toString()); + assertEquals("group:long:69", new LongId(69L).toString()); + assertEquals("group:long_bucket:6:9", new LongBucketId(6L, 9L).toString()); + assertEquals("group:null", new NullId().toString()); + assertEquals("group:raw:[6, 9]", new RawId(new byte[] { 6, 9 }).toString()); + assertEquals("group:raw_bucket:[6, 9]:[9, 6]", new RawBucketId(new byte[] { 6, 9 }, new byte[] { 9, 6 }).toString()); + assertTrue(new RootId(0).toString().startsWith("group:root:")); + assertEquals("group:string:69", new StringId("69").toString()); + assertEquals("group:string_bucket:6:9", new StringBucketId("6", "9").toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java new file mode 100644 index 00000000000..c9aa0848a8b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author Simon Thoresen + */ +public class GroupListTestCase { + + @Test + public void requireThatAccessorsWork() { + GroupList lst = new GroupList("foo"); + assertEquals("foo", lst.getLabel()); + assertEquals(0, lst.continuations().size()); + + MyContinuation foo = new MyContinuation(); + lst.continuations().put("foo", foo); + assertEquals(1, lst.continuations().size()); + assertSame(foo, lst.continuations().get("foo")); + + MyContinuation bar = new MyContinuation(); + lst.continuations().put("bar", bar); + assertEquals(2, lst.continuations().size()); + assertSame(bar, lst.continuations().get("bar")); + } + + private static class MyContinuation extends Continuation { + + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java new file mode 100644 index 00000000000..9acab986ac2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * @author Simon Thoresen + */ +public class GroupTestCase { + + @Test + public void requireThatListsAreAccessibleByLabel() { + Group grp = new Group(new LongId(69L), new Relevance(1)); + grp.add(new Hit("hit")); + grp.add(new HitList("hitList")); + grp.add(new GroupList("groupList")); + + assertNotNull(grp.getGroupList("groupList")); + assertNull(grp.getGroupList("unknownGroupList")); + assertNull(grp.getGroupList("hitList")); + + assertNotNull(grp.getHitList("hitList")); + assertNull(grp.getHitList("unknownHitList")); + assertNull(grp.getHitList("groupList")); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java new file mode 100644 index 00000000000..f9f7047abc0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author Simon Thoresen + */ +public class HitListTestCase { + + @Test + public void requireThatAccessorsWork() { + HitList lst = new HitList("foo"); + assertEquals("foo", lst.getLabel()); + assertEquals(0, lst.continuations().size()); + + MyContinuation foo = new MyContinuation(); + lst.continuations().put("foo", foo); + assertEquals(1, lst.continuations().size()); + assertSame(foo, lst.continuations().get("foo")); + + MyContinuation bar = new MyContinuation(); + lst.continuations().put("bar", bar); + assertEquals(2, lst.continuations().size()); + assertSame(bar, lst.continuations().get("bar")); + } + + private static class MyContinuation extends Continuation { + + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java new file mode 100644 index 00000000000..97a2e81d9ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java @@ -0,0 +1,174 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.text.Utf8; +import com.yahoo.text.XMLWriter; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class HitRendererTestCase { + + @Test + public void requireThatGroupListsRenderAsExpected() { + assertRender(new GroupList("foo"), "\n"); + assertRender(new GroupList("b\u00e6z"), "\n"); + + GroupList lst = new GroupList("foo"); + lst.continuations().put("bar.key", new MyContinuation("bar.val")); + lst.continuations().put("baz.key", new MyContinuation("baz.val")); + assertRender(lst, "\n" + + "bar.val\n" + + "baz.val\n" + + "\n"); + } + + @Test + public void requireThatGroupIdsRenderAsExpected() { + assertRender(newGroup(new DoubleId(6.9)), + "\n" + + "6.9\n" + + "\n"); + assertRender(newGroup(new LongId(69L)), + "\n" + + "69\n" + + "\n"); + assertRender(newGroup(new NullId()), + "\n" + + "\n" + + "\n"); + assertRender(newGroup(new RawId(Utf8.toBytes("foo"))), + "\n" + + "[102, 111, 111]\n" + + "\n"); + assertRender(newGroup(new StringId("foo")), + "\n" + + "foo\n" + + "\n"); + assertRender(newGroup(new StringId("b\u00e6z")), + "\n" + + "b\u00e6z\n" + + "\n"); + assertRender(newGroup(new DoubleBucketId(6.9, 9.6)), + "\n" + + "\n6.9\n9.6\n\n" + + "\n"); + assertRender(newGroup(new LongBucketId(6L, 9L)), + "\n" + + "\n6\n9\n\n" + + "\n"); + assertRender(newGroup(new StringBucketId("bar", "baz")), + "\n" + + "\nbar\nbaz\n\n" + + "\n"); + assertRender(newGroup(new StringBucketId("b\u00e6r", "b\u00e6z")), + "\n" + + "\nb\u00e6r\nb\u00e6z\n\n" + + "\n"); + assertRender(newGroup(new RawBucketId(Utf8.toBytes("bar"), Utf8.toBytes("baz"))), + "\n" + + "\n[98, 97, 114]\n[98, 97, 122]\n\n" + + "\n"); + } + + @Test + public void requireThatGroupsRenderAsExpected() { + Group group = newGroup(new StringId("foo")); + group.setField("foo", "bar"); + group.setField("baz", "cox"); + assertRender(group, "\n" + + "foo\n" + + "bar\n" + + "cox\n" + + "\n"); + + group = newGroup(new StringId("foo")); + group.setField("foo", "b\u00e6r"); + group.setField("b\u00e5z", "cox"); + assertRender(group, "\n" + + "foo\n" + + "b\u00e6r\n" + + "cox\n" + + "\n"); + } + + @Test + public void requireThatRootGroupsRenderAsExpected() { + RootGroup group = new RootGroup(0, new MyContinuation("69")); + group.setField("foo", "bar"); + group.setField("baz", "cox"); + assertRender(group, "\n" + + "\n" + + "69\n" + + "bar\n" + + "cox\n" + + "\n"); + + group = new RootGroup(0, new MyContinuation("96")); + group.setField("foo", "b\u00e6r"); + group.setField("b\u00e5z", "cox"); + assertRender(group, "\n" + + "\n" + + "96\n" + + "b\u00e6r\n" + + "cox\n" + + "\n"); + } + + @Test + public void requireThatHitListsRenderAsExpected() { + assertRender(new HitList("foo"), "\n"); + assertRender(new HitList("b\u00e6z"), "\n"); + + HitList lst = new HitList("foo"); + lst.continuations().put("bar.key", new MyContinuation("bar.val")); + lst.continuations().put("baz.key", new MyContinuation("baz.val")); + assertRender(lst, "\n" + + "bar.val\n" + + "baz.val\n" + + "\n"); +} + + private static Group newGroup(GroupId id) { + return new Group(id, new Relevance(1)); + } + + @SuppressWarnings("deprecation") + private static void assertRender(HitGroup hit, String expectedXml) { + StringWriter str = new StringWriter(); + XMLWriter out = new XMLWriter(str, 0, -1); + try { + HitRenderer.renderHeader(hit, out); + while (out.openTags().size() > 0) { + out.closeTag(); + } + } catch (IOException e) { + fail(); + } + assertEquals(expectedXml, str.toString()); + } + + private static class MyContinuation extends Continuation { + + final String str; + + MyContinuation(String str) { + this.str = str; + } + + @Override + public String toString() { + return str; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java new file mode 100644 index 00000000000..5d2d584af9b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class CompositeContinuationTestCase { + + @Test + public void requireThatAccessorsWork() { + CompositeContinuation cnt = new CompositeContinuation(); + Iterator it = cnt.iterator(); + assertFalse(it.hasNext()); + + EncodableContinuation foo = new MyContinuation(); + cnt.add(foo); + it = cnt.iterator(); + assertTrue(it.hasNext()); + assertSame(foo, it.next()); + assertFalse(it.hasNext()); + + EncodableContinuation bar = new MyContinuation(); + cnt.add(bar); + it = cnt.iterator(); + assertTrue(it.hasNext()); + assertSame(foo, it.next()); + assertTrue(it.hasNext()); + assertSame(bar, it.next()); + assertFalse(it.hasNext()); + } + + @Test + public void requireThatCompositeContinuationsAreFlattened() { + assertEncode("BCBCBCBEBGBCBKCBACBKCCK", + newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21))); + assertEncode("BCBBBBBDBFBCBJBPCBJCCJ", + newComposite(newComposite(newOffset(-1, -1, -2, -3)), newComposite(newOffset(-5, -8, -13, -21)))); + } + + @Test + public void requireThatEmptyStringCanBeDecoded() { + assertDecode("", new CompositeContinuation()); + } + + @Test + public void requireThatCompositeContinuationsCanBeDecoded() { + assertDecode("BCBCBCBEBGBCBKCBACBKCCK", + newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21))); + assertDecode("BCBBBBBDBFBCBJBPCBJCCJ", + newComposite(newOffset(-1, -1, -2, -3), newOffset(-5, -8, -13, -21))); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(newComposite().hashCode(), newComposite().hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + CompositeContinuation cnt = newComposite(); + assertFalse(cnt.equals(new Object())); + assertEquals(cnt, newComposite()); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3)))); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21)))); + assertFalse(cnt.equals(newComposite(newOffset(5, 8, 13, 21)))); + + cnt = newComposite(newOffset(1, 1, 2, 3)); + assertFalse(cnt.equals(new Object())); + assertEquals(cnt, newComposite(newOffset(1, 1, 2, 3))); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21)))); + assertFalse(cnt.equals(newComposite(newOffset(5, 8, 13, 21)))); + + cnt = newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21)); + assertFalse(cnt.equals(new Object())); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3)))); + assertEquals(cnt, newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21))); + assertFalse(cnt.equals(newComposite(newOffset(5, 8, 13, 21)))); + } + + private static CompositeContinuation newComposite(EncodableContinuation... children) { + CompositeContinuation ret = new CompositeContinuation(); + for (EncodableContinuation child : children) { + ret.add(child); + } + return ret; + } + + private static OffsetContinuation newOffset(int resultId, int tag, int offset, int flags) { + return new OffsetContinuation(ResultId.valueOf(resultId), tag, offset, flags); + } + + private static void assertEncode(String expected, EncodableContinuation toEncode) { + IntegerEncoder actual = new IntegerEncoder(); + toEncode.encode(actual); + assertEquals(expected, actual.toString()); + } + + private static void assertDecode(String toDecode, Continuation expected) { + assertEquals(expected, ContinuationDecoder.decode(toDecode)); + } + + private static class MyContinuation extends EncodableContinuation { + + @Override + public void encode(IntegerEncoder out) { + + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java new file mode 100644 index 00000000000..386e8346cae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -0,0 +1,765 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.container.protect.Error; +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.HitList; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.searchlib.aggregation.CountAggregationResult; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.aggregation.HitsAggregationResult; +import com.yahoo.searchlib.aggregation.MaxAggregationResult; +import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.expression.AggregationRefNode; +import com.yahoo.searchlib.expression.ConstantNode; +import com.yahoo.searchlib.expression.IntegerResultNode; +import com.yahoo.searchlib.expression.StringResultNode; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class GroupingExecutorTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatNullRequestsPass() { + Result res = newExecution(new GroupingExecutor()).search(newQuery()); + assertNotNull(res); + assertEquals(0, res.hits().size()); + } + + @Test + public void requireThatEmptyRequestsPass() { + Query query = newQuery(); + GroupingRequest.newInstance(query).setRootOperation(new AllOperation()); + Result res = newExecution(new GroupingExecutor()).search(query); + assertNotNull(res); + assertEquals(0, res.hits().size()); + } + + @Test + public void requireThatRequestsAreTransformed() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + try { + newExecution(new GroupingExecutor(), new GroupingListThrower()).search(query); + fail(); + } catch (GroupingListException e) { + assertNotNull(e.lst); + assertEquals(1, e.lst.size()); + Grouping grp = e.lst.get(0); + assertNotNull(grp); + } + } + + @Test + public void requireThatEachBelowAllDoesNotBlowUp() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(each(output(summary(bar))))")); + Result res = newExecution(new GroupingExecutor()).search(query); + assertNotNull(res); + assertEquals(1, res.hits().size()); + } + + @Test + public void requireThatSearchIsMultiPass() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + PassCounter cnt = new PassCounter(); + newExecution(new GroupingExecutor(), cnt).search(query); + assertEquals(2, cnt.numPasses); + } + + @Test + public void requireThatPassRequestsSingleLevel() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + GroupingCollector clt = new GroupingCollector(); + newExecution(new GroupingExecutor(), clt).search(query); + assertEquals(2, clt.lst.size()); + Grouping grp = clt.lst.get(0); + assertEquals(0, grp.getFirstLevel()); + assertEquals(0, grp.getLastLevel()); + grp = clt.lst.get(1); + assertEquals(1, grp.getFirstLevel()); + assertEquals(1, grp.getLastLevel()); + } + + @Test + public void requireThatAggregationPerHitWithoutGroupingDoesNotWorkYet() { + try { + execute("each(output(strlen(customer)))"); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatAggregationWithoutGroupingWorks() { + List groupings=execute("all(output(count()))"); + assertEquals(1,groupings.size()); + assertEquals(0, groupings.get(0).getLevels().size()); + assertEquals(ConstantNode.class, groupings.get(0).getRoot().getAggregationResults().get(0).getExpression().getClass()); + } + + @Test + public void requireThatGroupingIsParallel() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))) as(max)" + + " each(output(min(bar))) as(min))")); + GroupingCounter cnt = new GroupingCounter(); + newExecution(new GroupingExecutor(), cnt).search(query); + assertEquals(2, cnt.passList.size()); + assertEquals(2, cnt.passList.get(0).intValue()); + assertEquals(2, cnt.passList.get(1).intValue()); + } + + @Test + public void requireThatParallelGroupingIsNotRedundant() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))) as(shallow)" + + " each(group(baz) each(output(max(cox)))) as(deep))")); + GroupingCounter cnt = new GroupingCounter(); + newExecution(new GroupingExecutor(), cnt).search(query); + assertEquals(3, cnt.passList.size()); + assertEquals(2, cnt.passList.get(0).intValue()); + assertEquals(2, cnt.passList.get(1).intValue()); + assertEquals(1, cnt.passList.get(2).intValue()); + } + + @Test + public void requireThatPassResultsAreMerged() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(min(bar), max(bar))))")); + + Grouping grpA = new Grouping(0); + grpA.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("uniqueA")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(6)).setTag(4))) + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("common")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(9)).setTag(4))) + ); + Grouping grpB = new Grouping(0); + grpB.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("uniqueB")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(9)).setTag(4))) + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("common")).addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3))) + ); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grpA), null), + new GroupingListHit(Arrays.asList(grpB), null)))); + Group grp = req.getResultGroup(exec.search(query)); + assertEquals(1, grp.size()); + Hit hit = grp.get(0); + assertTrue(hit instanceof GroupList); + GroupList lst = (GroupList)hit; + assertEquals(3, lst.size()); + assertNotNull(hit = lst.get("group:string:uniqueA")); + assertEquals(6L, hit.getField("max(bar)")); + assertNotNull(hit = lst.get("group:string:uniqueB")); + assertEquals(9L, hit.getField("max(bar)")); + assertNotNull(hit = lst.get("group:string:common")); + assertEquals(6L, hit.getField("min(bar)")); + assertEquals(9L, hit.getField("max(bar)")); + } + + @Test + public void requireThatUnexpectedGroupingResultsAreIgnored() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + + Grouping grpExpected = new Grouping(0); + grpExpected.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("expected")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(69)).setTag(3))) + ); + Grouping grpUnexpected = new Grouping(1); + grpUnexpected.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("unexpected")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(96)).setTag(3))) + ); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grpExpected), null), + new GroupingListHit(Arrays.asList(grpUnexpected), null)))); + Group grp = req.getResultGroup(exec.search(query)); + assertEquals(1, grp.size()); + Hit hit = grp.get(0); + assertTrue(hit instanceof GroupList); + GroupList lst = (GroupList)hit; + assertEquals(1, lst.size()); + assertNotNull(hit = lst.get("group:string:expected")); + assertEquals(69L, hit.getField("max(bar)")); + assertNull(lst.get("group:string:unexpected")); + } + + @Test + public void requireThatHitsAreFilled() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp0 = new Grouping(0); + grp0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + Grouping grp1 = new Grouping(0); + grp1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit())) + )); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null))), + new FillRequestThrower()); + Result res = exec.search(query); + try { + exec.fill(res); + fail(); + } catch (FillRequestException e) { + assertEquals("bar", e.summaryClass); + } + } + + @Test + public void requireThatUnfilledHitsRenderError() throws IOException { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp0 = new Grouping(0); + grp0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")))); + Grouping grp1 = new Grouping(0); + grp1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult( + new HitsAggregationResult(1, "bar") + .addHit(new com.yahoo.searchlib.aggregation.FS4Hit())))); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null))), + new FillErrorProvider()); + Result res = exec.search(query); + exec.fill(res); + assertNotNull(res.hits().getError()); + } + + @Test + public void requireThatGroupRelevanceCanBeSynthesized() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) order(count()) each(output(count())))")); + + Grouping grp = new Grouping(0); + grp.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("foo")) + .addAggregationResult(new CountAggregationResult(1)) + .addOrderBy(new AggregationRefNode(0), true)) + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("bar")) + .addAggregationResult(new CountAggregationResult(2)) + .addOrderBy(new AggregationRefNode(0), true))); + Result res = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp), null), + new GroupingListHit(Arrays.asList(grp), null)))).search(query); + + GroupList groupList = (GroupList)req.getResultGroup(res).get(0); + assertEquals(1.0, groupList.get(0).getRelevance().getScore(), 1E-6); + assertEquals(0.5, groupList.get(1).getRelevance().getScore(), 1E-6); + } + + @Test + public void requireThatErrorsAreHandled() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp0 = new Grouping(0); + grp0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + Grouping grp1 = new Grouping(0); + grp1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit())) + )); + + ErrorProvider err = new ErrorProvider(1); + Execution exec = newExecution(new GroupingExecutor(), + err, + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null)))); + Result res = exec.search(query); + assertTrue(res.hits().getError() != null); + assertEquals(Error.TIMEOUT.code, res.hits().getError().getCode()); + assertFalse(err.continuedOnFail); + + err = new ErrorProvider(0); + exec = newExecution(new GroupingExecutor(), + err, + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null)))); + res = exec.search(query); + assertTrue(res.hits().getError() != null); + assertEquals(Error.TIMEOUT.code, res.hits().getError().getCode()); + assertFalse(err.continuedOnFail); + } + + @Test + public void requireThatHitsAreFilledWithCorrectSummary() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar))) as(bar) " + + " each(output(summary(baz))) as(baz)))")); + Grouping pass0A = new Grouping(0); + pass0A.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + Grouping pass0B = new Grouping(1); + pass0B.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "baz")) + )); + GlobalId gid1 = new GlobalId((new DocumentId("doc:test:1")).getGlobalId()); + GlobalId gid2 = new GlobalId((new DocumentId("doc:test:2")).getGlobalId()); + Grouping pass1A = new Grouping(0); + pass1A.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit(1, gid1, 3))) + )); + Grouping pass1B = new Grouping(1); + pass1B.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "baz").addHit(new com.yahoo.searchlib.aggregation.FS4Hit(4, gid2, 6))) + )); + SummaryMapper sm = new SummaryMapper(); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(pass0A, pass0B), null), + new GroupingListHit(Arrays.asList(pass1A, pass1B), null))), + sm); + exec.fill(exec.search(query), "default"); + assertEquals(2, sm.hitsBySummary.size()); + + List lst = sm.hitsBySummary.get("bar"); + assertNotNull(lst); + assertEquals(1, lst.size()); + Hit hit = lst.get(0); + assertTrue(hit instanceof FastHit); + assertEquals(1, ((FastHit)hit).getPartId()); + assertEquals(gid1, ((FastHit)hit).getGlobalId()); + + assertNotNull(lst = sm.hitsBySummary.get("baz")); + assertNotNull(lst); + assertEquals(1, lst.size()); + hit = lst.get(0); + assertTrue(hit instanceof FastHit); + assertEquals(4, ((FastHit)hit).getPartId()); + assertEquals(gid2, ((FastHit)hit).getGlobalId()); + } + + @Test + public void requireThatDefaultSummaryNameFillsHitsWithNull() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary()))) as(foo))")); + + Grouping pass0 = new Grouping(0); + pass0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("foo")) + .addAggregationResult( + new HitsAggregationResult(1, ExpressionConverter.DEFAULT_SUMMARY_NAME)))); + Grouping pass1 = new Grouping(0); + pass1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("foo")) + .addAggregationResult( + new HitsAggregationResult(1, ExpressionConverter.DEFAULT_SUMMARY_NAME) + .addHit(new com.yahoo.searchlib.aggregation.FS4Hit())))); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(pass0), null), + new GroupingListHit(Arrays.asList(pass1), null)))); + Result res = exec.search(query); + exec.fill(res); + + Hit hit = ((HitList)((Group)((GroupList)req.getResultGroup(res).get(0)).get(0)).get(0)).get(0); + assertTrue(hit instanceof FastHit); + assertTrue(hit.isFilled(null)); + } + + @Test + public void requireThatHitsAreAttachedToCorrectQuery() { + Query queryA = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(queryA); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp = new Grouping(0); + grp.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + GroupingListHit pass0 = new GroupingListHit(Arrays.asList(grp), null); + + GlobalId gid = new GlobalId((new DocumentId("doc:test:1")).getGlobalId()); + grp = new Grouping(0); + grp.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit(4, gid, 6))) + )); + GroupingListHit pass1 = new GroupingListHit(Arrays.asList(grp), null); + Query queryB = newQuery(); /** required by {@link GroupingListHit#getSearchQuery()} */ + pass1.setQuery(queryB); + + QueryMapper qm = new QueryMapper(); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList(pass0, pass1)), + qm); + exec.fill(exec.search(queryA)); + assertEquals(1, qm.hitsByQuery.size()); + assertTrue(qm.hitsByQuery.containsKey(queryB)); + } + + /** + * Tests the internal rewriting of rank properties which happens in the query.prepare() call + * (triggered by the exc.search call in the below). + */ + @Test + public void testRankProperties() { + Execution exc = newExecution(new GroupingExecutor()); + { + Query query = new Query("?query=foo"); + exc.search(query); + } + { + Query query = new Query("?query=foo&rankfeature.fieldMatch(foo)=2"); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + exc.search(query); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + } + { + Query query = new Query("?query=foo&rankfeature.query(now)=4"); + assertEquals("4", query.getRanking().getFeatures().get("query(now)")); + exc.search(query); + assertEquals("4", query.getRanking().getProperties().get("now").get(0)); + } + { + Query query = new Query("?query=foo&rankfeature.$bar=8"); + assertEquals("8", query.getRanking().getFeatures().get("$bar")); + exc.search(query); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + } + { + Query query = new Query("?query=foo&rankproperty.bar=8"); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + exc.search(query); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + } + { + Query query = new Query("?query=foo&rankfeature.fieldMatch(foo)=2&rankfeature.query(now)=4&rankproperty.bar=8"); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals("4", query.getRanking().getFeatures().get("query(now)")); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + exc.search(query); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals("4", query.getRanking().getProperties().get("now").get(0)); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + } + } + + @Test + public void testIllegalQuery() { + Execution exc = newExecution(new GroupingExecutor()); + + Query query = new Query(); + NotItem notItem = new NotItem(); + + notItem.addNegativeItem(new WordItem("negative")); + query.getModel().getQueryTree().setRoot(notItem); + + Result result = exc.search(query); + com.yahoo.search.result.ErrorMessage message = result.hits().getError(); + + assertNotNull("Got error", message); + assertEquals("Illegal query", message.getMessage()); + assertEquals("Can not search for only negative items", + message.getDetailedMessage()); + assertEquals(3, message.getCode()); + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static Query newQuery() { + return new Query("?query=dummy"); + } + + private static Execution newExecution(Searcher... searchers) { + return new Execution(new SearchChain(new ComponentId("foo"), Arrays.asList(searchers)), + Execution.Context.createContextStub()); + } + + private List execute(String groupingExpression) { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString(groupingExpression)); + GroupingCollector collector = new GroupingCollector(); + newExecution(new GroupingExecutor(), collector).search(query); + return collector.lst; + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class FillRequestThrower extends Searcher { + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + throw new FillRequestException(summaryClass); + } + } + + @SuppressWarnings("serial") + private static class FillRequestException extends RuntimeException { + + final String summaryClass; + + FillRequestException(String summaryClass) { + this.summaryClass = summaryClass; + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class GroupingListThrower extends Searcher { + + @Override + public Result search(Query query, Execution exec) { + throw new GroupingListException(GroupingExecutor.getGroupingList(query)); + } + } + + @SuppressWarnings("serial") + private static class GroupingListException extends RuntimeException { + + final List lst; + + GroupingListException(List lst) { + this.lst = lst; + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class GroupingCollector extends Searcher { + + List lst = new ArrayList<>(); + + @Override + public Result search(Query query, Execution exec) { + for (Grouping grp : GroupingExecutor.getGroupingList(query)) { + lst.add(grp.clone()); + } + return exec.search(query); + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class ErrorProvider extends Searcher { + private final int failOnPassN; + private int passnum; + public boolean continuedOnFail; + + public ErrorProvider(int failOnPassN) { + this.failOnPassN = failOnPassN; + this.passnum = 0; + this.continuedOnFail = false; + } + @Override + public Result search(Query query, Execution exec) { + Result ret = exec.search(query); + if (passnum > failOnPassN) { + continuedOnFail = true; + return ret; + } + if (passnum == failOnPassN) { + ret.hits().setError(ErrorMessage.createTimeout("timeout")); + } + passnum++; + return ret; + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class PassCounter extends Searcher { + + int numPasses = 0; + + @Override + public Result search(Query query, Execution exec) { + ++numPasses; + return exec.search(query); + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class GroupingCounter extends Searcher { + + List passList = new ArrayList<>(); + + @Override + public Result search(Query query, Execution exec) { + passList.add(GroupingExecutor.getGroupingList(query).size()); + return exec.search(query); + } + } + + private static class QueryMapper extends Searcher { + + final Map> hitsByQuery = new HashMap<>(); + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + for (Iterator it = result.hits().deepIterator(); it.hasNext();) { + Hit hit = it.next(); + Query query = hit.getQuery(); + List lst = hitsByQuery.get(query); + if (lst == null) { + lst = new LinkedList<>(); + hitsByQuery.put(query, lst); + } + lst.add(hit); + } + } + } + + + @After (GroupingExecutor.COMPONENT_NAME) + private static class SummaryMapper extends Searcher { + + final Map> hitsBySummary = new HashMap<>(); + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + for (Iterator it = result.hits().deepIterator(); it.hasNext();) { + Hit hit = it.next(); + List lst = hitsBySummary.get(summaryClass); + if (lst == null) { + lst = new LinkedList<>(); + hitsBySummary.put(summaryClass, lst); + } + lst.add(hit); + } + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class ResultProvider extends Searcher { + + final Queue hits = new LinkedList<>(); + int pass = 0; + + ResultProvider(List hits) { + this.hits.addAll(hits); + } + + @Override + public Result search(Query query, Execution exec) { + GroupingListHit hit = hits.poll(); + for (Grouping grp : hit.getGroupingList()) { + grp.setFirstLevel(pass); + grp.setLastLevel(pass); + } + ++pass; + Result res = exec.search(query); + res.hits().add(hit); + return res; + } + } + + private static class FillErrorProvider extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + result.hits().addError(ErrorMessage.createInternalServerError("foo")); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java new file mode 100644 index 00000000000..898a73a3320 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java @@ -0,0 +1,227 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class GroupingTransformTestCase { + + private static final int REQUEST_ID = 0; + + @Test + public void requireThatLabelCanBeSet() { + GroupingTransform transform = newTransform(); + transform.putLabel(0, 1, "foo", "my_type"); + assertEquals("foo", transform.getLabel(1)); + } + + @Test + public void requireThatLabelCanNotBeReplaced() { + GroupingTransform transform = newTransform(); + transform.putLabel(0, 1, "foo", "my_type"); + try { + transform.putLabel(0, 1, "bar", "my_type"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Can not set label of my_type 1 to 'bar' because it is already set to 'foo'.", + e.getMessage()); + } + } + + @Test + public void requireThatLabelIsUniqueAmongSiblings() { + GroupingTransform transform = newTransform(); + transform.putLabel(0, 1, "foo", "my_type"); + try { + transform.putLabel(0, 2, "foo", "my_type"); + fail(); + } catch (UnsupportedOperationException e) { + assertEquals("Can not use my_type label 'foo' for multiple siblings.", + e.getMessage()); + } + } + + @Test + public void requireThatMaxDefaultsToZero() { + GroupingTransform transform = newTransform(); + assertEquals(0, transform.getMax(6)); + assertEquals(0, transform.getMax(9)); + } + + @Test + public void requireThatMaxCanBeSet() { + GroupingTransform transform = newTransform(); + transform.putMax(0, 69, "my_type"); + assertEquals(69, transform.getMax(0)); + } + + @Test + public void requireThatMaxCanNotBeReplaced() { + GroupingTransform transform = newTransform(); + transform.putMax(0, 6, "my_type"); + try { + transform.putMax(0, 9, "my_type"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Can not set max of my_type 0 to 9 because it is already set to 6.", + e.getMessage()); + } + assertEquals(6, transform.getMax(0)); + } + + @Test + public void requireThatOffsetDefaultsToZero() { + GroupingTransform transform = newTransform(); + assertEquals(0, transform.getOffset(6)); + assertEquals(0, transform.getOffset(9)); + } + + @Test + public void requireThatOffsetContinuationsCanBeAdded() { + GroupingTransform transform = newTransform(); + transform.addContinuation(newStableOffset(newResultId(), 6, 9)); + assertEquals(9, transform.getOffset(6)); + } + + @Test + public void requireThatOffsetByIdCanBeReplaced() { + GroupingTransform transform = newTransform(); + ResultId id = newResultId(6, 9); + transform.addContinuation(newStableOffset(id, 0, 6)); + assertEquals(6, transform.getOffset(id)); + transform.addContinuation(newStableOffset(id, 0, 69)); + assertEquals(69, transform.getOffset(id)); + transform.addContinuation(newStableOffset(id, 0, 9)); + assertEquals(9, transform.getOffset(id)); + transform.addContinuation(newStableOffset(id, 0, 96)); + assertEquals(96, transform.getOffset(id)); + } + + @Test + public void requireThatOffsetByTagEqualsHighestSibling() { + GroupingTransform transform = newTransform(); + transform.addContinuation(newStableOffset(newResultId(1), 69, 6)); + assertEquals(6, transform.getOffset(69)); + transform.addContinuation(newStableOffset(newResultId(2), 69, 69)); + assertEquals(69, transform.getOffset(69)); + transform.addContinuation(newStableOffset(newResultId(3), 69, 9)); + assertEquals(69, transform.getOffset(69)); + transform.addContinuation(newStableOffset(newResultId(4), 69, 96)); + assertEquals(96, transform.getOffset(69)); + } + + @Test + public void requireThatOffsetContinuationsCanBeReplaced() { + GroupingTransform transform = newTransform(); + ResultId id = newResultId(6, 9); + transform.addContinuation(newStableOffset(id, 1, 1)); + assertEquals(1, transform.getOffset(1)); + assertEquals(1, transform.getOffset(id)); + assertTrue(transform.isStable(id)); + + transform.addContinuation(newUnstableOffset(id, 1, 2)); + assertEquals(2, transform.getOffset(1)); + assertEquals(2, transform.getOffset(id)); + assertFalse(transform.isStable(id)); + + transform.addContinuation(newStableOffset(id, 1, 3)); + assertEquals(3, transform.getOffset(1)); + assertEquals(3, transform.getOffset(id)); + assertTrue(transform.isStable(id)); + } + + @Test + public void requireThatUnstableOffsetsAreTracked() { + GroupingTransform transform = newTransform(); + ResultId stableId = newResultId(6); + transform.addContinuation(newStableOffset(stableId, 1, 1)); + assertTrue(transform.isStable(stableId)); + ResultId unstableId = newResultId(9); + transform.addContinuation(newUnstableOffset(unstableId, 2, 3)); + assertTrue(transform.isStable(stableId)); + assertFalse(transform.isStable(unstableId)); + } + + @Test + public void requireThatCompositeContinuationsAreDecomposed() { + GroupingTransform transform = newTransform(); + transform.addContinuation(new CompositeContinuation() + .add(newStableOffset(newResultId(), 6, 9)) + .add(newStableOffset(newResultId(), 9, 6))); + assertEquals(9, transform.getOffset(6)); + assertEquals(6, transform.getOffset(9)); + } + + @Test + public void requireThatUnsupportedContinuationsCanNotBeAdded() { + GroupingTransform transform = newTransform(); + try { + transform.addContinuation(new Continuation() { + + }); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatUnrelatedContinuationsAreIgnored() { + GroupingTransform transform = new GroupingTransform(REQUEST_ID); + ResultId id = ResultId.valueOf(REQUEST_ID + 1, 1); + transform.addContinuation(new OffsetContinuation(id, 2, 3, OffsetContinuation.FLAG_UNSTABLE)); + assertEquals(0, transform.getOffset(2)); + assertEquals(0, transform.getOffset(id)); + assertTrue(transform.isStable(id)); + } + + @Test + public void requireThatToStringIsVerbose() { + GroupingTransform transform = new GroupingTransform(REQUEST_ID); + transform.putLabel(1, 1, "label1", "type1"); + transform.putLabel(2, 2, "label2", "type2"); + transform.addContinuation(newStableOffset(ResultId.valueOf(REQUEST_ID), 3, 3)); + transform.addContinuation(newStableOffset(ResultId.valueOf(REQUEST_ID), 4, 4)); + transform.putMax(5, 5, "type5"); + transform.putMax(6, 6, "type6"); + assertEquals("groupingTransform {\n" + + "\tlabels {\n" + + "\t\t1 : label1\n" + + "\t\t2 : label2\n" + + "\t}\n" + + "\toffsets {\n" + + "\t\t3 : 3\n" + + "\t\t4 : 4\n" + + "\t}\n" + + "\tmaxes {\n" + + "\t\t5 : 5\n" + + "\t\t6 : 6\n" + + "\t}\n" + + "}", transform.toString()); + } + + private static GroupingTransform newTransform() { + return new GroupingTransform(REQUEST_ID); + } + + private static ResultId newResultId(int... indexes) { + ResultId id = ResultId.valueOf(REQUEST_ID); + for (int i : indexes) { + id = id.newChildId(i); + } + return id; + } + + private static OffsetContinuation newStableOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, 0); + } + + private static OffsetContinuation newUnstableOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, OffsetContinuation.FLAG_UNSTABLE); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java new file mode 100644 index 00000000000..ebd663d80b0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.fs4.QueryPacketData; +import com.yahoo.net.URI; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.searchlib.aggregation.FS4Hit; +import com.yahoo.searchlib.aggregation.VdsHit; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class HitConverterTestCase { + + private GlobalId createGlobalId(int docId) { + return new GlobalId((new DocumentId("doc:test:" + docId)).getGlobalId()); + } + + @Test + public void requireThatHitsAreConverted() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(new Hit("hit:ctx"))); + assertNotNull(hit); + assertEquals(new URI("index:0/1/0/" + FastHit.asHexString(createGlobalId(2))), hit.getId()); + + hit = converter.toSearchHit("default", new FS4Hit(4, createGlobalId(5), 6).setContext(new Hit("hit:ctx"))); + assertNotNull(hit); + assertEquals(new URI("index:0/4/0/" + FastHit.asHexString(createGlobalId(5))), hit.getId()); + } + + @Test + public void requireThatContextDataIsCopied() { + Hit ctxHit = new Hit("hit:ctx"); + ctxHit.setSource("69"); + ctxHit.setSourceNumber(69); + Query ctxQuery = new Query(); + ctxHit.setQuery(ctxQuery); + + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(ctxHit)); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertEquals(1, ((FastHit)hit).getPartId()); + assertEquals(createGlobalId(2), ((FastHit)hit).getGlobalId()); + assertSame(ctxQuery, hit.getQuery()); + assertEquals(ctxHit.getSource(), hit.getSource()); + assertEquals(ctxHit.getSourceNumber(), hit.getSourceNumber()); + } + + @Test + public void requireThatHitTagIsCopiedFromGroupingListContext() { + QueryPacketData ctxTag = new QueryPacketData(); + GroupingListHit ctxHit = new GroupingListHit(null, null); + ctxHit.setQueryPacketData(ctxTag); + + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(ctxHit)); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertSame(ctxTag, ((FastHit)hit).getQueryPacketData()); + } + + @Test + public void requireThatSummaryClassIsSet() { + Searcher searcher = new MySearcher(); + HitConverter converter = new HitConverter(searcher, new Query()); + Hit hit = converter.toSearchHit("69", new FS4Hit(1, createGlobalId(2), 3).setContext(new Hit("hit:ctx"))); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertEquals("69", hit.getSearcherSpecificMetaData(searcher)); + } + + @Test + public void requireThatHitHasContext() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + try { + converter.toSearchHit("69", new FS4Hit(1, createGlobalId(2), 3)); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatUnsupportedHitClassThrows() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + try { + converter.toSearchHit("69", new com.yahoo.searchlib.aggregation.Hit() { + + }); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + private static DocumentdbInfoConfig.Documentdb sixtynine() { + DocumentdbInfoConfig.Documentdb.Builder summaryConfig = new DocumentdbInfoConfig.Documentdb.Builder(); + summaryConfig.name("none"); + summaryConfig.summaryclass(new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder().id(0).name("69")); + return new DocumentdbInfoConfig.Documentdb(summaryConfig); + } + + @Test + public void requireThatVdsHitCanBeConverted() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + GroupingListHit context = new GroupingListHit(null, new DocsumDefinitionSet(sixtynine())); + VdsHit lowHit = new VdsHit("doc:scheme:", new byte[] { 0, 0, 0, 0 }, 1); + lowHit.setContext(context); + Hit hit = converter.toSearchHit("69", lowHit); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertEquals(new Relevance(1), hit.getRelevance()); + assertTrue(hit.isFilled("69")); + } + + private static class MySearcher extends Searcher { + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java new file mode 100644 index 00000000000..9389482010e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class IntegerDecoderTestCase { + + @Test + public void requireThatIntDecoderWorksAsExpected() { + assertDecode("A", 0); + assertDecode("BC", 1); + assertDecode("CBI", 12); + assertDecode("CPG", 123); + assertDecode("DJKE", 1234); + assertDecode("EGAHC", 12345); + assertDecode("FDMEIA", 123456); + assertDecode("GCFKNAO", 1234567); + assertDecode("HBHIMCJM", 12345678); + assertDecode("HOLHJKCK", 123456789); + assertDecode("IJDCMAFKE", 1234567890); + assertDecode("IIKKEBPOF", -1163005939); + assertDecode("IECKEIKID", -559039810); + } + + @Test + public void requireThatDecoderThrowsExceptionOnBadInput() { + try { + new IntegerDecoder("B").next(); + fail(); + } catch (IndexOutOfBoundsException e) { + + } + try { + new IntegerDecoder("11X1Y").next(); + fail(); + } catch (NumberFormatException e) { + + } + } + + private static void assertDecode(String toDecode, int expected) { + IntegerDecoder decoder = new IntegerDecoder(toDecode); + assertTrue(decoder.hasNext()); + assertEquals(expected, decoder.next()); + assertFalse(decoder.hasNext()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java new file mode 100644 index 00000000000..4780f23ca9d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Simon Thoresen + */ +public class IntegerEncoderTestCase { + + @Test + public void requireThatIntEncoderWorksAsExpected() { + assertEncode("A", 0); + assertEncode("BC", 1); + assertEncode("CBI", 12); + assertEncode("CPG", 123); + assertEncode("DJKE", 1234); + assertEncode("EGAHC", 12345); + assertEncode("FDMEIA", 123456); + assertEncode("GCFKNAO", 1234567); + assertEncode("HBHIMCJM", 12345678); + assertEncode("HOLHJKCK", 123456789); + assertEncode("IJDCMAFKE", 1234567890); + assertEncode("IIKKEBPOF", -1163005939); + assertEncode("IECKEIKID", -559039810); + } + + private static void assertEncode(String expected, int toEncode) { + IntegerEncoder actual = new IntegerEncoder(); + actual.append(toEncode); + assertEquals(expected, actual.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java new file mode 100644 index 00000000000..8184a52c0ee --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author Simon Thoresen + */ +public class OffsetContinuationTestCase { + + @Test + public void requireThatNullResultIdThrowsException() { + try { + new OffsetContinuation(null, 0, 0, 0); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatAccessorsWork() { + OffsetContinuation cnt = new OffsetContinuation(ResultId.valueOf(1), 2, 3, 4); + assertEquals(ResultId.valueOf(1), cnt.getResultId()); + assertEquals(2, cnt.getTag()); + assertEquals(3, cnt.getOffset()); + assertEquals(4, cnt.getFlags()); + + cnt = new OffsetContinuation(ResultId.valueOf(5), 6, 7, 8); + assertEquals(ResultId.valueOf(5), cnt.getResultId()); + assertEquals(6, cnt.getTag()); + assertEquals(7, cnt.getOffset()); + assertEquals(8, cnt.getFlags()); + + for (int i = 0; i < 30; ++i) { + cnt = new OffsetContinuation(ResultId.valueOf(1), 2, 3, (1 << i) + (1 << i + 1)); + assertTrue(cnt.testFlag(1 << i)); + assertTrue(cnt.testFlag(1 << i + 1)); + assertFalse(cnt.testFlag(1 << i + 2)); + } + } + + @Test + public void requireThatOffsetContinuationsCanBeEncoded() { + assertEncode("BCBCBCBEBG", newOffset(1, 1, 2, 3)); + assertEncode("BCBKCBACBKCCK", newOffset(5, 8, 13, 21)); + assertEncode("BCBBBBBDBF", newOffset(-1, -1, -2, -3)); + assertEncode("BCBJBPCBJCCJ", newOffset(-5, -8, -13, -21)); + } + + @Test + public void requireThatOffsetContinuationsCanBeDecoded() { + assertDecode("BCBCBCBEBG", newOffset(1, 1, 2, 3)); + assertDecode("BCBKCBACBKCCK", newOffset(5, 8, 13, 21)); + assertDecode("BCBBBBBDBF", newOffset(-1, -1, -2, -3)); + assertDecode("BCBJBPCBJCCJ", newOffset(-5, -8, -13, -21)); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(newOffset(1, 1, 2, 3).hashCode(), newOffset(1, 1, 2, 3).hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + Continuation cnt = newOffset(1, 1, 2, 3); + assertFalse(cnt.equals(new Object())); + assertFalse(cnt.equals(newOffset(0, 1, 2, 3))); + assertFalse(cnt.equals(newOffset(1, 0, 2, 3))); + assertFalse(cnt.equals(newOffset(1, 1, 0, 3))); + assertFalse(cnt.equals(newOffset(1, 1, 2, 0))); + assertEquals(cnt, newOffset(1, 1, 2, 3)); + } + + + private static OffsetContinuation newOffset(int resultId, int tag, int offset, int flags) { + return new OffsetContinuation(ResultId.valueOf(resultId), tag, offset, flags); + } + + private static void assertEncode(String expected, EncodableContinuation toEncode) { + IntegerEncoder actual = new IntegerEncoder(); + toEncode.encode(actual); + assertEquals(expected, actual.toString()); + } + + private static void assertDecode(String toDecode, Continuation expected) { + assertEquals(expected, OffsetContinuation.decode(new IntegerDecoder(toDecode))); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java new file mode 100644 index 00000000000..9fd32577737 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java @@ -0,0 +1,885 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.request.*; +import com.yahoo.searchlib.aggregation.*; +import com.yahoo.searchlib.expression.*; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class RequestBuilderTestCase { + + @Test + public void requireThatAllAggregationResulsAreSupported() { + assertLayout("all(group(a) each(output(avg(b))))", "[[{ Attribute, result = [Average] }]]"); + assertLayout("all(group(a) each(output(count())))", "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(a) each(output(max(b))))", "[[{ Attribute, result = [Max] }]]"); + assertLayout("all(group(a) each(output(min(b))))", "[[{ Attribute, result = [Min] }]]"); + assertLayout("all(group(a) each(output(sum(b))))", "[[{ Attribute, result = [Sum] }]]"); + assertLayout("all(group(a) each(each(output(summary()))))", "[[{ Attribute, result = [Hits] }]]"); + assertLayout("all(group(a) each(output(xor(b))))", "[[{ Attribute, result = [Xor] }]]"); + } + + @Test + public void requireThatExpressionCountAggregationResultIsSupported() { + RequestBuilder builder = new RequestBuilder(0); + builder.setRootOperation(GroupingOperation.fromString("all(group(foo) output(count()))")); + builder.build(); + AggregationResult aggr = builder.getRequestList().get(0).getRoot().getAggregationResults().get(0); + assertTrue(aggr instanceof ExpressionCountAggregationResult); + assertEquals(new AttributeNode("foo"), aggr.getExpression()); + } + + @Test + public void requireThatAllExpressionNodesAreSupported() { + assertLayout("all(group(add(a,b)) each(output(count())))", "[[{ Add, result = [Count] }]]"); + assertLayout("all(group(and(a,b)) each(output(count())))", "[[{ And, result = [Count] }]]"); + assertLayout("all(group(a) each(output(count())))", "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(cat(a,b)) each(output(count())))", "[[{ Cat, result = [Count] }]]"); + assertLayout("all(group(debugwait(a, 69, true)) each(output(count())))", "[[{ DebugWait, result = [Count] }]]"); + assertLayout("all(group(docidnsspecific()) each(output(count())))", "[[{ GetDocIdNamespaceSpecific, result = [Count] }]]"); + assertLayout("all(group(1.0) each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(div(a,b)) each(output(count())))", "[[{ Divide, result = [Count] }]]"); + assertLayout("all(group(fixedwidth(a,1)) each(output(count())))", "[[{ FixedWidthBucket, result = [Count] }]]"); + assertLayout("all(group(fixedwidth(a,1.0)) each(output(count())))", "[[{ FixedWidthBucket, result = [Count] }]]"); + assertLayout("all(group(1) each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(max(a,b)) each(output(count())))", "[[{ Max, result = [Count] }]]"); + assertLayout("all(group(md5(a,1)) each(output(count())))", "[[{ MD5Bit, result = [Count] }]]"); + assertLayout("all(group(uca(a,b)) each(output(count())))", "[[{ Uca, result = [Count] }]]"); + assertLayout("all(group(uca(a,b,PRIMARY)) each(output(count())))", "[[{ Uca, result = [Count] }]]"); + assertLayout("all(group(min(a,b)) each(output(count())))", "[[{ Min, result = [Count] }]]"); + assertLayout("all(group(mod(a,b)) each(output(count())))", "[[{ Modulo, result = [Count] }]]"); + assertLayout("all(group(mul(a,b)) each(output(count())))", "[[{ Multiply, result = [Count] }]]"); + assertLayout("all(group(neg(a)) each(output(count())))", "[[{ Negate, result = [Count] }]]"); + assertLayout("all(group(normalizesubject(a)) each(output(count())))", "[[{ NormalizeSubject, result = [Count] }]]"); + assertLayout("all(group(now()) each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(or(a,b)) each(output(count())))", "[[{ Or, result = [Count] }]]"); + assertLayout("all(group(predefined(a,bucket(1,2))) each(output(count())))", "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(relevance()) each(output(count())))", "[[{ Relevance, result = [Count] }]]"); + assertLayout("all(group(reverse(a)) each(output(count())))", "[[{ Reverse, result = [Count] }]]"); + assertLayout("all(group(size(a)) each(output(count())))", "[[{ NumElem, result = [Count] }]]"); + assertLayout("all(group(sort(a)) each(output(count())))", "[[{ Sort, result = [Count] }]]"); + assertLayout("all(group(strcat(a,b)) each(output(count())))", "[[{ StrCat, result = [Count] }]]"); + assertLayout("all(group('a') each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(strlen(a)) each(output(count())))", "[[{ StrLen, result = [Count] }]]"); + assertLayout("all(group(sub(a,b)) each(output(count())))", "[[{ Add, result = [Count] }]]"); + assertLayout("all(group(todouble(a)) each(output(count())))", "[[{ ToFloat, result = [Count] }]]"); + assertLayout("all(group(tolong(a)) each(output(count())))", "[[{ ToInt, result = [Count] }]]"); + assertLayout("all(group(toraw(a)) each(output(count())))", "[[{ ToRaw, result = [Count] }]]"); + assertLayout("all(group(tostring(a)) each(output(count())))", "[[{ ToString, result = [Count] }]]"); + assertLayout("all(group(time.date(a)) each(output(count())))", "[[{ StrCat, result = [Count] }]]"); + assertLayout("all(group(math.sqrt(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.cbrt(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.log(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.log1p(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.log10(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.exp(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.pow(a,b)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.hypot(a,b)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.sin(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.asin(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.cos(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.acos(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.tan(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.atan(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.sinh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.asinh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.cosh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.acosh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.tanh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(zcurve.x(a)) each(output(count())))", "[[{ ZCurve, result = [Count] }]]"); + assertLayout("all(group(zcurve.y(a)) each(output(count())))", "[[{ ZCurve, result = [Count] }]]"); + assertLayout("all(group(time.dayofmonth(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.dayofweek(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.dayofyear(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.hourofday(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.minuteofhour(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.monthofyear(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.secondofminute(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.year(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(xor(a,b)) each(output(count())))", "[[{ Xor, result = [Count] }]]"); + assertLayout("all(group(xorbit(a,1)) each(output(count())))", "[[{ XorBit, result = [Count] }]]"); + assertLayout("all(group(ymum()) each(output(count())))", "[[{ GetYMUMChecksum, result = [Count] }]]"); + } + + @Test + public void requireThatForceSinglePassIsSupported() { + assertForceSinglePass("all(group(foo) each(output(count())))", "[false]"); + assertForceSinglePass("all(group(foo) hint(singlepass) each(output(count())))", "[true]"); + assertForceSinglePass("all(hint(singlepass) " + + " all(group(foo) each(output(count())))" + + " all(group(bar) each(output(count()))))", + "[true, true]"); + + // it would be really nice if this test returned [true, true], but that is not how the AST is built + assertForceSinglePass("all(all(group(foo) hint(singlepass) each(output(count())))" + + " all(group(bar) hint(singlepass) each(output(count()))))", + "[false, false]"); + } + + @Test + public void requireThatThereCanBeOnlyOneBuildCall() { + RequestBuilder builder = new RequestBuilder(0); + builder.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(count())))")); + builder.build(); + try { + builder.build(); + fail(); + } catch (IllegalStateException e) { + + } + } + + @Test + public void requireThatNullSummaryClassProvidesDefault() { + RequestBuilder reqBuilder = new RequestBuilder(0); + reqBuilder.setRootOperation(new AllOperation() + .setGroupBy(new AttributeValue("foo")) + .addChild(new EachOperation() + .addChild(new EachOperation() + .addOutput(new SummaryValue())))); + reqBuilder.setDefaultSummaryName(null); + reqBuilder.build(); + + HitsAggregationResult hits = (HitsAggregationResult)reqBuilder.getRequestList().get(0) + .getLevels().get(0) + .getGroupPrototype() + .getAggregationResults().get(0); + assertEquals(ExpressionConverter.DEFAULT_SUMMARY_NAME, hits.getSummaryClass()); + } + + @Test + public void requireThatGroupOfGroupsAreNotSupported() { + // "Can not group list of groups." + assertBuildFail("all(group(a) all(group(avg(b)) each(each(each(output(summary()))))))", + "Can not operate on list of list of groups."); + } + + @Test + public void requireThatAnonymousListsAreNotSupported() { + assertBuildFail("all(group(a) all(each(each(output(summary())))))", + "Can not create anonymous list of groups."); + } + + @Test + public void requireThatOffsetContinuationCanModifyGroupingLevel() { + assertOffset("all(group(a) max(5) each(output(count())))", + newOffset(2, 5), + "[[{ tag = 2, max = [5, 11], hits = [] }]]"); + assertOffset("all(group(a) max(5) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + newOffset(2, 5), + "[[{ tag = 2, max = [5, 11], hits = [] }]," + + " [{ tag = 4, max = [5, 6], hits = [] }]]"); + assertOffset("all(group(a) max(5) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + newComposite(newOffset(2, 5), newOffset(4, 10)), + "[[{ tag = 2, max = [5, 11], hits = [] }]," + + " [{ tag = 4, max = [5, 16], hits = [] }]]"); + } + + @Test + public void requireThatOffsetContinuationCanModifyHitAggregator() { + assertOffset("all(group(a) each(max(5) each(output(summary()))))", + newOffset(3, 5), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [5, 11] }] }]]"); + assertOffset("all(group(a) each(max(5) each(output(summary()))) as(foo)" + + " each(max(5) each(output(summary()))) as(bar))", + newOffset(3, 5), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 4, max = [0, -1], hits = [{ tag = 5, max = [5, 6] }] }]]"); + assertOffset("all(group(a) each(max(5) each(output(summary()))) as(foo)" + + " each(max(5) each(output(summary()))) as(bar))", + newComposite(newOffset(3, 5), newOffset(5, 10)), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 4, max = [0, -1], hits = [{ tag = 5, max = [5, 16] }] }]]"); + } + + @Test + public void requireThatOffsetContinuationIsNotAppliedToGroupingLevelWithoutMax() { + assertOffset("all(group(a) each(output(count())))", + newOffset(2, 5), + "[[{ tag = 2, max = [0, -1], hits = [] }]]"); + } + + @Test + public void requireThatOffsetContinuationIsNotAppliedToHitAggregatorWithoutMax() { + assertOffset("all(group(a) each(each(output(summary()))))", + newOffset(3, 5), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [0, -1] }] }]]"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectRequestedGroupLists() { + String request = "all(group(a) max(5) each(group(b) max(5) each(output(count())) as(a1_b1)" + + " each(output(count())) as(a1_b2)) as(a1)" + + " each(group(b) max(5) each(output(count())) as(a2_b1)" + + " each(output(count())) as(a2_b2)) as(a2))"; + CompositeContinuation session = newComposite(newOffset(2, 5), newOffset(3, 5), newOffset(5, 5), + newOffset(7, 5), newOffset(8, 5), newOffset(10, 5)); + assertOffset(request, newComposite(session), + "[[{ tag = 2, max = [5, 11], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 11], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10)), + "[[{ tag = 2, max = [5, 16], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 16], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(7, 10)), + "[[{ tag = 2, max = [5, 11], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 11], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10), newUnstableOffset(7, 10)), + "[[{ tag = 2, max = [5, 16], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 16], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectRequestedHitLists() { + String request = "all(group(a) max(5) each(max(5) each(output(summary())) as(a1_h1)" + + " each(output(summary())) as(a1_h2)) as(a1)" + + " each(max(5) each(output(summary())) as(a2_h1)" + + " each(output(summary())) as(a2_h2)) as(a2))"; + CompositeContinuation session = newComposite(newOffset(2, 5), newOffset(3, 5), newOffset(4, 5), + newOffset(5, 5), newOffset(6, 5), newOffset(7, 5)); + assertOffset(request, newComposite(session), + "[[{ tag = 2, max = [5, 11], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 11], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 7, max = [5, 11] }] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10)), + "[[{ tag = 2, max = [5, 16], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 16], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 7, max = [5, 11] }] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(5, 10)), + "[[{ tag = 2, max = [5, 11], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 11], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 7, max = [5, 11] }] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10), newUnstableOffset(5, 10)), + "[[{ tag = 2, max = [5, 16], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 16], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 7, max = [5, 11] }] }]]"); + } + + @Test + public void requireThatExpressionsCanBeAliased() { + OutputWriter writer = (groupingList, transform) -> groupingList.get(0).getLevels().get(0).getGroupPrototype().getAggregationResults().get(0) + .toString(); + + RequestTest test = new RequestTest(); + test.expectedOutput = new SumAggregationResult().setTag(3).setExpression(new AttributeNode("price")).toString(); + test.request = "all(group(artist) alias(foo,sum(price)) each(output($foo)))"; + test.outputWriter = writer; + assertOutput(test); + + test = new RequestTest(); + test.expectedOutput = new SumAggregationResult().setTag(3).setExpression(new AttributeNode("price")).toString(); + test.request = "all(group(artist) order($foo=sum(price)) each(output($foo)))"; + test.outputWriter = writer; + assertOutput(test); + } + + @Test + public void requireThatGroupingLayoutIsCorrect() { + assertLayout("all(group(artist) each(max(69) output(count()) each(output(summary()))))", + "[[{ Attribute, result = [Count, Hits] }]]"); + assertLayout("all(group(artist) each(output(count()) all(group(album) each(output(count()) all(group(song) each(max(69) output(count()) each(output(summary()))))))))", + "[[{ Attribute, result = [Count] }, { Attribute, result = [Count] }, { Attribute, result = [Count, Hits] }]]"); + assertLayout("all(group(artist) each(output(count())))", + "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(artist) order(sum(price)) each(output(count())))", + "[[{ Attribute, result = [Count, Sum], order = [[1], [AggregationRef]] }]]"); + assertLayout("all(group(artist) each(max(69) output(count()) each(output(summary(foo)))))", + "[[{ Attribute, result = [Count, Hits] }]]"); + assertLayout("all(group(artist) each(output(count()) all(group(album) each(output(count())))))", + "[[{ Attribute, result = [Count] }, { Attribute, result = [Count] }]]"); + assertLayout("all(group(artist) max(5) each(output(count()) all(group(album) max(3) each(output(count())))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [4, 4], result = [Count] }]]"); + assertLayout("all(group(artist) max(5) each(output(count()) all(group(album) max(3) each(output(count())))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [4, 4], result = [Count] }]]"); + assertLayout("all(group(foo) max(10) each(output(count()) all(group(bar) max(10) each(output(count())))))", + "[[{ Attribute, max = [11, 11], result = [Count] }, { Attribute, max = [11, 11], result = [Count] }]]"); + assertLayout("all(group(a) max(5) each(max(69) output(count()) each(output(summary()))))", + "[[{ Attribute, max = [6, 6], result = [Count, Hits] }]]"); + assertLayout("all(group(a) max(5) each(output(count()) all(group(b) max(5) each(max(69) output(count()) each(output(summary()))))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [6, 6], result = [Count, Hits] }]]"); + assertLayout("all(group(a) max(5) each(output(count()) all(group(b) max(5) each(output(count()) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [6, 6], result = [Count, Hits] }]]"); + assertLayout("all(group(fixedwidth(n,3)) max(5) each(output(count()) all(group(a) max(2) each(output(count())))))", + "[[{ FixedWidthBucket, max = [6, 6], result = [Count] }, { Attribute, max = [3, 3], result = [Count] }]]"); + assertLayout("all(group(fixedwidth(n,3)) max(5) each(output(count()) all(group(a) max(2) each(output(count())))))", + "[[{ FixedWidthBucket, max = [6, 6], result = [Count] }, { Attribute, max = [3, 3], result = [Count] }]]"); + assertLayout("all(group(fixedwidth(n,3)) max(5) each(output(count()) all(group(a) max(2) each(max(1) output(count()) each(output(summary()))))))", + "[[{ FixedWidthBucket, max = [6, 6], result = [Count] }, { Attribute, max = [3, 3], result = [Count, Hits] }]]"); + assertLayout("all(group(predefined(n,bucket(1,3),bucket(6,9))) each(output(count())))", + "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(predefined(f,bucket(1.0,3.0),bucket(6.0,9.0))) each(output(count())))", + "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(predefined(s,bucket(\"ab\",\"cd\"),bucket(\"ef\",\"gh\"))) each(output(count())))", + "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(a) max(5) each(output(count())))", + "[[{ Attribute, max = [6, 6], result = [Count] }]]"); + assertLayout("all(group(a) max(5) each(output(count())))", + "[[{ Attribute, max = [6, 6], result = [Count] }]]"); + assertLayout("all(max(9) all(group(a) each(output(count()))))", + "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(where(true) all(group(a) each(output(count()))))", + "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(a) order(sum(n)) each(output(count())))", + "[[{ Attribute, result = [Count, Sum], order = [[1], [AggregationRef]] }]]"); + assertLayout("all(group(a) max(2) each(output(count())))", + "[[{ Attribute, max = [3, 3], result = [Count] }]]"); + assertLayout("all(group(a) max(2) precision(10) each(output(count())))", + "[[{ Attribute, max = [3, 10], result = [Count] }]]"); + assertLayout("all(group(fixedwidth(a,1)) each(output(count())))", + "[[{ FixedWidthBucket, result = [Count] }]]"); + } + + @Test + public void requireThatAggregatorCanBeUsedAsArgumentToOrderByFunction() { + assertLayout("all(group(a) order(sum(price) * count()) each(output(count())))", + "[[{ Attribute, result = [Count, Sum], order = [[1], [Multiply]] }]]"); + assertLayout("all(group(a) order(sum(price) + 4) each(output(sum(price))))", + "[[{ Attribute, result = [Sum], order = [[1], [Add]] }]]"); + assertLayout("all(group(a) order(sum(price) + 4, count()) each(output(sum(price))))", + "[[{ Attribute, result = [Sum, Count], order = [[1, 2], [Add, AggregationRef]] }]]"); + assertLayout("all(group(a) order(sum(price) + 4, -count()) each(output(sum(price))))", + "[[{ Attribute, result = [Sum, Count], order = [[1, -2], [Add, AggregationRef]] }]]"); + } + + @Test + public void requireThatSameAggregatorCanBeUsedMultipleTimes() { + assertLayout("all(group(a) each(output(count() as(b),count() as(c))))", + "[[{ Attribute, result = [Count, Count] }]]"); + } + + @Test + public void requireThatSiblingAggregatorsCanNotShareSameLabel() { + assertBuildFail("all(group(a) each(output(count(),count())))", + "Can not use output label 'count()' for multiple siblings."); + assertBuildFail("all(group(a) each(output(count() as(b),count() as(b))))", + "Can not use output label 'b' for multiple siblings."); + } + + @Test + public void requireThatOrderByReusesOutputResults() { + assertLayout("all(group(a) order(count()) each(output(count())))", + "[[{ Attribute, result = [Count], order = [[1], [AggregationRef]] }]]"); + assertLayout("all(group(a) order(count()) each(output(count() as(b))))", + "[[{ Attribute, result = [Count], order = [[1], [AggregationRef]] }]]"); + } + + @Test + public void requireThatNoopBranchesArePruned() { + assertLayout("all()", "[]"); + assertLayout("all(group(a))", "[]"); + assertLayout("all(group(a) each())", "[]"); + + String expectedA = "[{ Attribute, result = [Count] }]"; + assertLayout("all(group(a) each(output(count())))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all()))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b))))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each())", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each(group(b)))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each(group(b) each()))", + Arrays.asList(expectedA).toString()); + + String expectedB = "[{ Attribute }, { Attribute, result = [Count] }]"; + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each(group(b) each(output(count()))))", + Arrays.asList(expectedB, expectedA).toString()); + } + + @Test + public void requireThatAggregationLevelIsValidatedFails() { + assertBuildFail("all(group(artist) output(sum(length)))", + "Expression 'length' not applicable for single group."); + assertBuild("all(group(artist) each(output(count())))"); + assertBuildFail("all(group(artist) each(group(album) output(sum(length))))", + "Expression 'length' not applicable for single group."); + assertBuild("all(group(artist) each(group(album) each(output(count()))))"); + } + + @Test + public void requireThatCountOnListOfGroupsIsValidated() { + assertBuild("all(group(artist) output(count()))"); + assertBuild("all(group(artist) each(group(album) output(count())))"); + } + + @Test + public void requireThatGroupByIsValidated() { + assertBuild("all(group(artist) each(output(count())))"); + assertBuildFail("all(group(sum(artist)) each(output(count())))", + "Expression 'sum(artist)' not applicable for single hit."); + assertBuild("all(group(artist) each(group(album) each(output(count()))))"); + assertBuildFail("all(group(artist) each(group(sum(album)) each(output(count()))))", + "Expression 'sum(album)' not applicable for single hit."); + } + + @Test + public void requireThatGroupingLevelIsValidated() { + assertBuild("all(group(artist))"); + assertBuild("all(group(artist) each(group(album)))"); + assertBuildFail("all(group(artist) all(group(sum(price))))", + "Can not operate on list of list of groups."); + assertBuild("all(group(artist) each(group(album) each(group(song))))"); + assertBuildFail("all(group(artist) each(group(album) all(group(sum(price)))))", + "Can not operate on list of list of groups."); + } + + @Test + public void requireThatOrderByIsValidated() { + assertBuildFail("all(order(length))", + "Can not order single group content."); + assertBuild("all(group(artist) order(sum(length)))"); + assertBuildFail("all(group(artist) each(order(length)))", + "Can not order single group content."); + assertBuild("all(group(artist) each(group(album) order(sum(length))))"); + assertBuildFail("all(group(artist) each(group(album) each(order(length))))", + "Can not order single group content."); + } + + @Test + public void requireThatOrderByHasCorrectReference() { + assertOrderBy("all(group(a) order(count()) each(output(count())))", "[[[1]]]"); + assertOrderBy("all(group(a) order(-count()) each(output(count())))", "[[[-1]]]"); + assertOrderBy("all(group(a) order(count()) each(output(count(),sum(b))))", "[[[1]]]"); + assertOrderBy("all(group(a) order(-count()) each(output(count(),sum(b))))", "[[[-1]]]"); + assertOrderBy("all(group(a) order(count()) each(output(sum(b), count())))", "[[[1]]]"); + assertOrderBy("all(group(a) order(-count()) each(output(sum(b), count())))", "[[[-1]]]"); + + assertOrderBy("all(group(a) order(count(),sum(b)) each(output(count(),sum(b))))", "[[[1, 2]]]"); + assertOrderBy("all(group(a) order(count(),-sum(b)) each(output(count(),sum(b))))", "[[[1, -2]]]"); + assertOrderBy("all(group(a) order(-count(),sum(b)) each(output(count(),sum(b))))", "[[[-1, 2]]]"); + assertOrderBy("all(group(a) order(-count(),-sum(b)) each(output(count(),sum(b))))", "[[[-1, -2]]]"); + + // because order() is resolved before output(), index follows order() statement + assertOrderBy("all(group(a) order(count(),sum(b)) each(output(sum(b), count())))", "[[[1, 2]]]"); + assertOrderBy("all(group(a) order(count(),-sum(b)) each(output(sum(b), count())))", "[[[1, -2]]]"); + assertOrderBy("all(group(a) order(-count(),sum(b)) each(output(sum(b), count())))", "[[[-1, 2]]]"); + assertOrderBy("all(group(a) order(-count(),-sum(b)) each(output(sum(b), count())))", "[[[-1, -2]]]"); + + assertOrderBy("all(group(a) order(count()) each(output(count())) as(foo)" + + " each(output(sum(b))) as(bar))", + "[[[1]], [[1]]]"); + } + + + @Test + public void requireThatWhereIsValidated() { + assertBuild("all(where(true))"); + assertBuild("all(where($query))"); + assertBuildFail("all(where(foo))", + "Operation 'where' does not support 'foo'."); + assertBuildFail("all(group(artist) where(true))", + "Can not apply 'where' to non-root group."); + } + + @Test + public void requireThatRootAggregationCanBeTransformed() { + RequestTest test = new RequestTest(); + test.expectedOutput = CountAggregationResult.class.getName(); + test.request = "all(output(count()))"; + test.outputWriter = (groupingList, transform) -> groupingList.get(0).getRoot().getAggregationResults().get(0).getClass().getName(); + assertOutput(test); + } + + @Test + public void requireThatExpressionsCanBeLabeled() { + assertLabel("all(group(a) each(output(count())))", + "[[{ label = 'a', results = [count()] }]]"); + assertLabel("all(group(a) each(output(count())) as(b))", + "[[{ label = 'b', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(output(count()))))", + "[[{ label = 'a', results = [] }, { label = 'b', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(group(c) each(output(count())))))", + "[[{ label = 'a', results = [] }, { label = 'b', results = [] }, { label = 'c', results = [count()] }]]"); + assertBuildFail("all(group(a) each(output(count())) each(output(count())))", + "Can not use group list label 'a' for multiple siblings."); + assertBuildFail("all(all(group(a) each(output(count())))" + + " all(group(a) each(output(count()))))", + "Can not use group list label 'a' for multiple siblings."); + assertLabel("all(group(a) each(output(count())) as(a1)" + + " each(output(count())) as(a2))", + "[[{ label = 'a1', results = [count()] }], [{ label = 'a2', results = [count()] }]]"); + assertLabel("all(group(a) each(all(group(b) each(output(count())))" + + " all(group(c) each(output(count())))))", + "[[{ label = 'a', results = [] }, { label = 'b', results = [count()] }], [{ label = 'a', results = [] }, { label = 'c', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(output(count()))) as(a1)" + + " each(group(b) each(output(count()))) as(a2))", + "[[{ label = 'a1', results = [] }, { label = 'b', results = [count()] }], [{ label = 'a2', results = [] }, { label = 'b', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(group(c) each(output(count())))) as(a1)" + + " each(group(b) each(group(e) each(output(count())))) as(a2))", + "[[{ label = 'a1', results = [] }, { label = 'b', results = [] }, { label = 'c', results = [count()] }]," + + " [{ label = 'a2', results = [] }, { label = 'b', results = [] }, { label = 'e', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(output(count())) as(b1)" + + " each(output(count())) as(b2)))", + "[[{ label = 'a', results = [] }, { label = 'b1', results = [count()] }]," + + " [{ label = 'a', results = [] }, { label = 'b2', results = [count()] }]]"); + + assertBuildFail("all(group(a) each(each(output(summary() as(foo)))))", + "Can not label expression 'summary()'."); + assertLabel("all(group(foo) each(each(output(summary()))))", + "[[{ label = 'foo', results = [hits] }]]"); + assertLabel("all(group(foo) each(each(output(summary())) as(bar)))", + "[[{ label = 'foo', results = [bar] }]]"); + assertLabel("all(group(foo) each(each(output(summary())) as(bar)) as(baz))", + "[[{ label = 'baz', results = [bar] }]]"); + assertLabel("all(group(foo) each(each(output(summary())) as(bar)" + + " each(output(summary())) as(baz)))", + "[[{ label = 'foo', results = [bar] }]," + + " [{ label = 'foo', results = [baz] }]]"); + assertLabel("all(group(foo) each(each(output(summary())))" + + " each(each(output(summary()))) as(bar))", + "[[{ label = 'bar', results = [hits] }]," + + " [{ label = 'foo', results = [hits] }]]"); + } + + @Test + public void requireThatOrderByResultsAreNotLabeled() { + assertLabel("all(group(a) each(output(min(b), max(b), avg(b))))", + "[[{ label = 'a', results = [min(b), max(b), avg(b)] }]]"); + assertLabel("all(group(a) order(min(b)) each(output(max(b), avg(b))))", + "[[{ label = 'a', results = [max(b), avg(b), null] }]]"); + assertLabel("all(group(a) order(min(b), max(b)) each(output(avg(b))))", + "[[{ label = 'a', results = [avg(b), null, null] }]]"); + } + + @Test + public void requireThatTimeZoneIsAppliedToTimeFunctions() { + for (String timePart : Arrays.asList("dayofmonth", "dayofweek", "dayofyear", "hourofday", + "minuteofhour", "monthofyear", "secondofminute", "year")) + { + String request = "all(output(avg(time." + timePart + "(foo))))"; + assertTimeZone(request, "GMT-2", -7200L); + assertTimeZone(request, "GMT-1", -3600L); + assertTimeZone(request, "GMT", null); + assertTimeZone(request, "GMT+1", 3600L); + assertTimeZone(request, "GMT+2", 7200L); + } + } + + @Test + public void requireThatTimeDateIsExpanded() { + RequestTest test = new RequestTest(); + test.expectedOutput = new StrCatFunctionNode() + .addArg(new ToStringFunctionNode(new TimeStampFunctionNode(new AttributeNode("foo"), + TimeStampFunctionNode.TimePart.Year, true))) + .addArg(new ConstantNode(new StringResultNode("-"))) + .addArg(new ToStringFunctionNode(new TimeStampFunctionNode(new AttributeNode("foo"), + TimeStampFunctionNode.TimePart.Month, true))) + .addArg(new ConstantNode(new StringResultNode("-"))) + .addArg(new ToStringFunctionNode(new TimeStampFunctionNode(new AttributeNode("foo"), + TimeStampFunctionNode.TimePart.MonthDay, true))) + .toString(); + test.request = "all(output(avg(time.date(foo))))"; + test.outputWriter = (groupingList, transform) -> groupingList.get(0).getRoot().getAggregationResults().get(0).getExpression().toString(); + assertOutput(test); + } + + @Test + public void requireThatNowIsResolvedToCurrentTime() { + RequestTest test = new RequestTest(); + test.expectedOutput = Boolean.toString(true); + test.request = "all(output(avg(now() - foo)))"; + test.outputWriter = new OutputWriter() { + long before = System.currentTimeMillis(); + + @Override + public String write(List groupingList, GroupingTransform transform) { + AddFunctionNode add = + (AddFunctionNode)groupingList.get(0).getRoot().getAggregationResults().get(0).getExpression(); + long nowValue = ((ConstantNode)add.getArg(0)).getValue().getInteger(); + boolean preCond = nowValue >= (before / 1000); + long after = System.currentTimeMillis(); + boolean postCond = nowValue <= (after / 1000); + boolean allOk = preCond && postCond; + return Boolean.toString(allOk); + } + }; + assertOutput(test); + } + + private static CompositeContinuation newComposite(EncodableContinuation... conts) { + CompositeContinuation ret = new CompositeContinuation(); + for (EncodableContinuation cont : conts) { + ret.add(cont); + } + return ret; + } + + private static OffsetContinuation newOffset(int tag, int offset) { + return new OffsetContinuation(ResultId.valueOf(0), tag, offset, 0); + } + + private static OffsetContinuation newUnstableOffset(int tag, int offset) { + return new OffsetContinuation(ResultId.valueOf(0), tag, offset, OffsetContinuation.FLAG_UNSTABLE); + } + + private static void assertBuild(String request) { + RequestTest test = new RequestTest(); + test.request = request; + assertOutput(test); + } + + private static void assertBuildFail(String request, String expectedException) { + RequestTest test = new RequestTest(); + test.request = request; + test.expectedException = expectedException; + assertOutput(test); + } + + private static void assertTimeZone(String request, String timeZone, Long expectedOutput) { + RequestTest test = new RequestTest(); + test.request = request; + test.timeZone = timeZone; + test.outputWriter = (groupingList, transform) -> { + Long timeOffset = null; + ExpressionNode node = + ((TimeStampFunctionNode)groupingList.get(0).getRoot().getAggregationResults().get(0) + .getExpression()).getArg(0); + if (node instanceof AddFunctionNode) { + timeOffset = (((ConstantNode)((AddFunctionNode)node).getArg(1)).getValue()).getInteger(); + } + return String.valueOf(timeOffset); + }; + test.expectedOutput = String.valueOf(expectedOutput); + assertOutput(test); + } + + private static void assertLabel(String request, String expectedOutput) { + assertOutput(request, new LabelWriter(), expectedOutput); + } + + private static void assertLayout(String request, String expectedOutput) { + assertOutput(request, new LayoutWriter(), expectedOutput); + } + + private static void assertOrderBy(String request, String expectedOutput) { + assertOutput(request, new OrderByWriter(), expectedOutput); + } + + private static void assertOffset(String request, Continuation continuation, String expectedOutput) { + RequestTest ret = new RequestTest(); + ret.request = request; + ret.continuation = continuation; + ret.outputWriter = new OffsetWriter(); + ret.expectedOutput = expectedOutput; + assertOutput(ret); + } + + private static void assertForceSinglePass(String request, String expectedOutput) { + assertOutput(request, new ForceSinglePassWriter(), expectedOutput); + } + + private static void assertOutput(String request, OutputWriter writer, String expectedOutput) { + RequestTest ret = new RequestTest(); + ret.request = request; + ret.outputWriter = writer; + ret.expectedOutput = expectedOutput; + assertOutput(ret); + } + + private static void assertOutput(RequestTest test) { + RequestBuilder builder = new RequestBuilder(0); + builder.setRootOperation(GroupingOperation.fromString(test.request)); + builder.setTimeZone(TimeZone.getTimeZone(test.timeZone)); + builder.addContinuations(Arrays.asList(test.continuation)); + try { + builder.build(); + if (test.expectedException != null) { + fail("Expected exception '" + test.expectedException + "'."); + } + } catch (RuntimeException e) { + if (test.expectedException == null) { + throw e; + } + assertEquals(test.expectedException, e.getMessage()); + return; + } + if (test.outputWriter != null) { + String output = test.outputWriter.write(builder.getRequestList(), builder.getTransform()); + assertEquals(test.expectedOutput, output); + } + } + + private static class RequestTest { + + String request; + String timeZone = "utc"; + String expectedException; + String expectedOutput; + OutputWriter outputWriter; + Continuation continuation; + } + + private static interface OutputWriter { + + String write(List groupingList, GroupingTransform transform); + } + + private static class OffsetWriter implements OutputWriter { + + @Override + public String write(List groupingList, GroupingTransform transform) { + List foo = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List bar = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + List baz = new LinkedList<>(); + for (AggregationResult result : level.getGroupPrototype().getAggregationResults()) { + if (result instanceof HitsAggregationResult) { + int tag = result.getTag(); + baz.add("{ tag = " + tag + ", max = [" + transform.getMax(tag) + ", " + + ((HitsAggregationResult)result).getMaxHits() + "] }"); + } + } + int tag = level.getGroupPrototype().getTag(); + bar.add("{ tag = " + tag + ", max = [" + transform.getMax(tag) + ", " + level.getMaxGroups() + + "], hits = " + baz.toString() + " }"); + } + foo.add(bar.toString()); + } + Collections.sort(foo); + return foo.toString(); + } + } + + private static class LabelWriter implements OutputWriter { + + @Override + public String write(List groupingList, GroupingTransform transform) { + List foo = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List bar = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + List baz = new LinkedList<>(); + for (AggregationResult result : level.getGroupPrototype().getAggregationResults()) { + baz.add(transform.getLabel(result.getTag())); + } + bar.add("{ label = '" + transform.getLabel(level.getGroupPrototype().getTag()) + + "', results = " + baz.toString() + " }"); + } + foo.add(bar.toString()); + } + Collections.sort(foo); + return foo.toString(); + } + } + + private static class LayoutWriter implements OutputWriter { + + @Override + public String write(List groupingList, GroupingTransform transform) { + List foo = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List bar = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + StringBuilder str = new StringBuilder("{ "); + str.append(toSimpleName(level.getExpression())).append(", "); + if (level.getMaxGroups() >= 0 || level.getPrecision() >= 0) { + str.append("max = [").append(level.getMaxGroups()).append(", ") + .append(level.getPrecision()).append("], "); + } + Group group = level.getGroupPrototype(); + if (!group.getAggregationResults().isEmpty()) { + List baz = new LinkedList<>(); + for (AggregationResult exp : level.getGroupPrototype().getAggregationResults()) { + baz.add(toSimpleName(exp)); + } + str.append("result = ").append(baz).append(", "); + } + if (!group.getOrderByIndexes().isEmpty() || !group.getOrderByExpressions().isEmpty()) { + List baz = new LinkedList<>(); + for (Integer idx : level.getGroupPrototype().getOrderByIndexes()) { + baz.add(idx.toString()); + } + str.append("order = [").append(baz).append(", "); + baz = new LinkedList<>(); + for (ExpressionNode exp : level.getGroupPrototype().getOrderByExpressions()) { + baz.add(toSimpleName(exp)); + } + str.append(baz).append("], "); + } + str.setLength(str.length() - 2); + str.append(" }"); + bar.add(str.toString()); + } + foo.add(bar.toString()); + } + Collections.sort(foo); + return foo.toString(); + } + + private static String toSimpleName(ExpressionNode exp) { + String ret = exp.getClass().getSimpleName(); + if (ret.endsWith("AggregationResult")) { + return ret.substring(0, ret.length() - 17); + } + if (ret.endsWith("FunctionNode")) { + return ret.substring(0, ret.length() - 12); + } + if (ret.endsWith("Node")) { + return ret.substring(0, ret.length() - 4); + } + return ret; + } + } + + private static class OrderByWriter implements OutputWriter { + + @Override + public String write(List groupingList, GroupingTransform transform) { + List> ret = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List lst = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + lst.add(level.getGroupPrototype().getOrderByIndexes().toString()); + } + ret.add(lst); + } + return ret.toString(); + } + } + + private static class ForceSinglePassWriter implements OutputWriter { + + @Override + public String write(List groupingList, GroupingTransform transform) { + List ret = new LinkedList<>(); + for (Grouping grouping : groupingList) { + ret.add(String.valueOf(grouping.getForceSinglePass())); + } + return ret.toString(); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java new file mode 100644 index 00000000000..d8438ddc042 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java @@ -0,0 +1,1108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdString; +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.grouping.result.AbstractList; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.HitList; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.searchlib.aggregation.*; +import com.yahoo.searchlib.aggregation.hll.SparseSketch; +import com.yahoo.searchlib.expression.*; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen + */ +public class ResultBuilderTestCase { + + private static final int REQUEST_ID = 0; + private static final int ROOT_IDX = 0; + + @Test + public void requireThatAllGroupIdsCanBeConverted() { + assertGroupId("group:6.9", new FloatResultNode(6.9)); + assertGroupId("group:69", new IntegerResultNode(69)); + assertGroupId("group:null", new NullResultNode()); + assertGroupId("group:[6, 9]", new RawResultNode(new byte[] { 6, 9 })); + assertGroupId("group:a", new StringResultNode("a")); + assertGroupId("group:6.9:9.6", new FloatBucketResultNode(6.9, 9.6)); + assertGroupId("group:6:9", new IntegerBucketResultNode(6, 9)); + assertGroupId("group:a:b", new StringBucketResultNode("a", "b")); + assertGroupId("group:[6, 9]:[9, 6]", new RawBucketResultNode(new RawResultNode(new byte[] { 6, 9 }), + new RawResultNode(new byte[] { 9, 6 }))); + } + + @Test + public void requireThatUnknownGroupIdThrows() { + assertBuildFail("all(group(a) each(output(count())))", + Arrays.asList(newGrouping(new Group().setTag(2).setId(new MyResultNode()))), + "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyResultNode"); + } + + @Test + public void requireThatAllExpressionNodesCanBeConverted() { + assertResult("0", new AverageAggregationResult(new IntegerResultNode(6), 9)); + assertResult("69", new CountAggregationResult(69)); + assertResult("69", new MaxAggregationResult(new IntegerResultNode(69))); + assertResult("69", new MinAggregationResult(new IntegerResultNode(69))); + assertResult("69", new SumAggregationResult(new IntegerResultNode(69))); + assertResult("69", new XorAggregationResult(69)); + assertResult("69", new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 69)); + } + + @Test + public void requireThatUnknownExpressionNodeThrows() { + assertBuildFail("all(group(a) each(output(count())))", + Arrays.asList(newGrouping(newGroup(2, 2, new MyAggregationResult().setTag(3)))), + "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyAggregationResult"); + } + + @Test + public void requireThatRootResultsAreIncluded() { + assertLayout("all(output(count()))", + new Grouping().setRoot(newGroup(1, new CountAggregationResult(69).setTag(2))), + "RootGroup{id=group:root, count()=69}[]"); + } + + @Test + public void requireThatRootResultsAreIncludedUsingExpressionCountAggregationResult() { + assertLayout("all(group(a) output(count()))", + new Grouping().setRoot(newGroup(1, new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 69).setTag(2))), + "RootGroup{id=group:root, count()=69}[]"); + } + + @Test + public void requireThatNestedGroupingResultsCanBeTransformed() { + Grouping grouping = new Grouping() + .setRoot(new Group() + .setTag(1) + .addChild(new Group() + .setTag(2) + .setId(new StringResultNode("foo")) + .addAggregationResult(new CountAggregationResult(10).setTag(3)) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("foo_a")) + .addAggregationResult(new CountAggregationResult(15) + .setTag(5))) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("foo_b")) + .addAggregationResult(new CountAggregationResult(16) + .setTag(5)))) + .addChild(new Group() + .setTag(2) + .setId(new StringResultNode("bar")) + .addAggregationResult(new CountAggregationResult(20).setTag(3)) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("bar_a")) + .addAggregationResult( + new CountAggregationResult(25) + .setTag(5))) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("bar_b")) + .addAggregationResult( + new CountAggregationResult(26) + .setTag(5))))); + assertLayout("all(group(artist) max(5) each(output(count() as(baz)) all(group(album) " + + "max(5) each(output(count() as(cox))) as(group_album))) as(group_artist))", + grouping, + "RootGroup{id=group:root}[GroupList{label=group_artist}[" + + "Group{id=group:foo, baz=10}[GroupList{label=group_album}[Group{id=group:foo_a, cox=15}[], Group{id=group:foo_b, cox=16}[]]], " + + "Group{id=group:bar, baz=20}[GroupList{label=group_album}[Group{id=group:bar_a, cox=25}[], Group{id=group:bar_b, cox=26}[]]]]]"); + } + + @Test + public void requireThatParallelResultsAreTransformed() { + assertBuild("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", + Arrays.asList(new Grouping().setRoot(newGroup(1, 0)), + new Grouping().setRoot(newGroup(1, 0)))); + assertBuildFail("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", + Arrays.asList(new Grouping().setRoot(newGroup(2)), + new Grouping().setRoot(newGroup(3))), + "Expected 1 group, got 2."); + } + + @Test + public void requireThatTagsAreHandledCorrectly() { + assertBuild("all(group(a) each(output(count())))", + Arrays.asList(newGrouping( + newGroup(7, new CountAggregationResult(0))))); + } + + @Test + public void requireThatEmptyBranchesArePruned() { + assertBuildFail("all()", Collections.emptyList(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a))", Collections.emptyList(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a) each())", Collections.emptyList(), "Expected 1 group, got 0."); + + Grouping grouping = newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))); + String expectedOutput = "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"; + assertLayout("all(group(a) each(output(count())))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all()))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b))))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each()))" + + " each() as(foo))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each()))" + + " each(group(b)) as(foo))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each()))" + + " each(group(b) each()) as(foo))", grouping, expectedOutput); + } + + @Test + public void requireThatGroupListsAreLabeled() { + assertLayout("all(group(a) each(output(count())))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"); + assertLayout("all(group(a) each(output(count())) as(bar))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=bar}[Group{id=group:2, count()=69}[]]]"); + } + + @Test + public void requireThatHitListsAreLabeled() { + assertLayout("all(group(foo) each(each(output(summary()))))", + newGrouping(newGroup(2, newHitList(3, 2))), + "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + + "HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())) as(bar)))", + newGrouping(newGroup(2, newHitList(3, 2))), + "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())) as(bar)) as(baz))", + newGrouping(newGroup(2, newHitList(3, 2))), + "RootGroup{id=group:root}[GroupList{label=baz}[Group{id=group:2}[" + + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())) as(bar)" + + " each(output(summary())) as(baz)))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + newGrouping(newGroup(2, newHitList(4, 2)))), + "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}], " + + "HitList{label=baz}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())))" + + " each(each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + newGrouping(newGroup(4, newHitList(5, 2)))), + "RootGroup{id=group:root}[" + + "GroupList{label=foo}[Group{id=group:2}[HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]], " + + "GroupList{label=bar}[Group{id=group:4}[HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + } + + @Test + public void requireThatOutputsAreLabeled() { + assertLayout("all(output(count()))", + new Grouping().setRoot(newGroup(1, new CountAggregationResult(69).setTag(2))), + "RootGroup{id=group:root, count()=69}[]"); + assertLayout("all(output(count() as(foo)))", + new Grouping().setRoot(newGroup(1, new CountAggregationResult(69).setTag(2))), + "RootGroup{id=group:root, foo=69}[]"); + assertLayout("all(group(a) each(output(count())))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"); + assertLayout("all(group(a) each(output(count() as(foo))))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, foo=69}[]]]"); + } + + @Test + public void requireThatExpressionCountCanUseExactGroupCount() { + Group root1 = newGroup(1, new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 42).setTag(2)); + Grouping grouping1 = new Grouping().setRoot(root1); + + // Should return estimate when no groups are returned (since each() clause is absent). + assertLayout("all(group(artist) output(count()))", + grouping1, + "RootGroup{id=group:root, count()=42}[]"); + + Group root2 = newGroup(1, new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 42).setTag(2)); + Grouping grouping2 = new Grouping().setRoot(root2); + for (int i = 0; i < 3; ++i) { + root2.addChild(new Group() + .setTag(2) + .setId(new StringResultNode("foo" + i))) + .addAggregationResult(new CountAggregationResult(i).setTag(3)); + } + + // Should return the number of groups when max is not present. + assertLayout("all(group(artist) output(count()) each(output(count())))", + grouping2, + "RootGroup{id=group:root, count()=3, artist=2}" + + "[GroupList{label=count()}[Group{id=group:foo0}[], Group{id=group:foo1}[], Group{id=group:foo2}[]]]"); + + // Should return the number of groups when max is higher than group count. + assertLayout("all(group(artist) max(5) output(count()) each(output(count())))", + grouping2, + "RootGroup{id=group:root, count()=3, artist=2}" + + "[GroupList{label=count()}[Group{id=group:foo0}[], Group{id=group:foo1}[], Group{id=group:foo2}[]]]"); + + // Should return the estimate when number of groups is equal to max. + assertLayout("all(group(artist) max(3) output(count()) each(output(count())))", + grouping2, + "RootGroup{id=group:root, count()=42, artist=2}" + + "[GroupList{label=count()}[Group{id=group:foo0}[], Group{id=group:foo1}[], Group{id=group:foo2}[]]]"); + + } + + + @Test + public void requireThatResultContinuationContainsCurrentPages() { + String request = "all(group(a) max(2) each(output(count())))"; + Grouping result = newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))); + assertResultCont(request, result, newOffset(newResultId(0), 2, 0), "[]"); + assertResultCont(request, result, newOffset(newResultId(0), 2, 1), "[0=1]"); + assertResultCont(request, result, newOffset(newResultId(0), 2, 2), "[0=2]"); + assertResultCont(request, result, newOffset(newResultId(0), 2, 3), "[0=3]"); + + assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), + "[]"); + assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), + newOffset(newResultId(0), 2, 1), + "[0=1]"); + assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), + newComposite(newOffset(newResultId(0), 2, 2), + newOffset(newResultId(1), 4, 1)), + "[0=2, 1=1]"); + + request = "all(group(a) each(max(2) each(output(summary()))))"; + result = newGrouping(newGroup(2, newHitList(3, 4))); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 0), "[]"); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 1), "[0.0.0=1]"); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 2), "[0.0.0=2]"); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 3), "[0.0.0=3]"); + + assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))), + "[]"); + assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))), + newOffset(newResultId(0, 0, 0), 3, 1), + "[0.0.0=1]"); + assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))), + newComposite(newOffset(newResultId(0, 0, 0), 3, 2), + newOffset(newResultId(1, 0, 0), 5, 1)), + "[0.0.0=2, 1.0.0=1]"); + } + + @Test + public void requireThatGroupListContinuationsAreNotCreatedWhenUnlessMaxIsSet() { + assertContinuation("all(group(a) each(output(count())))", + newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:1', {}, [] }, { 'group:2', {}, [] }, { 'group:3', {}, [] }, { 'group:4', {}, [] }] }] }"); + } + + @Test + public void requireThatGroupListContinuationsCanBeSet() { + String request = "all(group(a) max(2) each(output(count())))"; + Grouping result = newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))); + assertContinuation(request, result, newOffset(newResultId(0), 2, 0), + "{ 'group:root', {}, [{ 'grouplist:a', {next=2}, [" + + "{ 'group:1', {}, [] }, { 'group:2', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 1), + "{ 'group:root', {}, [{ 'grouplist:a', {next=3, prev=0}, [" + + "{ 'group:2', {}, [] }, { 'group:3', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 2), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=0}, [" + + "{ 'group:3', {}, [] }, { 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 3), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=1}, [" + + "{ 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 4), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=2}, [" + + "] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 5), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=2}, [" + + "] }] }"); + } + + @Test + public void requireThatGroupListContinuationsCanBeSetInSiblingGroups() { + String request = "all(group(a) each(group(b) max(2) each(output(count()))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, new CountAggregationResult(1)), + newGroup(3, 302, new CountAggregationResult(2)), + newGroup(3, 303, new CountAggregationResult(3)), + newGroup(3, 304, new CountAggregationResult(4))), + newGroup(2, 202, + newGroup(3, 305, new CountAggregationResult(5)), + newGroup(3, 306, new CountAggregationResult(6)), + newGroup(3, 307, new CountAggregationResult(7)), + newGroup(3, 308, new CountAggregationResult(8)))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 0), + newOffset(newResultId(0, 1, 0), 2, 5)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [{ 'group:301', {}, [] }, { 'group:302', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 1), + newOffset(newResultId(0, 1, 0), 2, 4)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=3, prev=0}, [{ 'group:302', {}, [] }, { 'group:303', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 2), + newOffset(newResultId(0, 1, 0), 2, 3)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [{ 'group:303', {}, [] }, { 'group:304', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=1}, [{ 'group:308', {}, [] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 3), + newOffset(newResultId(0, 1, 0), 2, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=1}, [{ 'group:304', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=0}, [{ 'group:307', {}, [] }, { 'group:308', {}, [] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 4), + newOffset(newResultId(0, 1, 0), 2, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=3, prev=0}, [{ 'group:306', {}, [] }, { 'group:307', {}, [] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 5), + newOffset(newResultId(0, 1, 0), 2, 0)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [{ 'group:305', {}, [] }, { 'group:306', {}, [] }] }] }] }] }"); + } + + @Test + public void requireThatGroupListContinuationsCanBeSetInSiblingGroupLists() { + String request = "all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))"; + List result = Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))), + newGrouping(newGroup(4, 1, new CountAggregationResult(1)), + newGroup(4, 2, new CountAggregationResult(2)), + newGroup(4, 3, new CountAggregationResult(3)), + newGroup(4, 4, new CountAggregationResult(4)))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 0), + newOffset(newResultId(1), 4, 5)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {next=2}, [{ 'group:1', {}, [] }, { 'group:2', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=2}, [] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 1), + newOffset(newResultId(1), 4, 4)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {next=3, prev=0}, [{ 'group:2', {}, [] }, { 'group:3', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=2}, [] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 2), + newOffset(newResultId(1), 4, 3)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=0}, [{ 'group:3', {}, [] }, { 'group:4', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=1}, [{ 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 3), + newOffset(newResultId(1), 4, 2)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=1}, [{ 'group:4', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=0}, [{ 'group:3', {}, [] }, { 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 4), + newOffset(newResultId(1), 4, 1)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=2}, [] }, " + + "{ 'grouplist:bar', {next=3, prev=0}, [{ 'group:2', {}, [] }, { 'group:3', {}, [] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 5), + newOffset(newResultId(1), 4, 0)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=2}, [] }, " + + "{ 'grouplist:bar', {next=2}, [{ 'group:1', {}, [] }, { 'group:2', {}, [] }] }] }"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectSiblingGroupLists() { + String request = "all(group(a) each(group(b) max(2) each(group(c) max(2) each(output(count())))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, + newGroup(4, 401, new CountAggregationResult(1)), + newGroup(4, 402, new CountAggregationResult(1)), + newGroup(4, 403, new CountAggregationResult(1)), + newGroup(4, 404, new CountAggregationResult(1))), + newGroup(3, 302, + newGroup(4, 405, new CountAggregationResult(1)), + newGroup(4, 406, new CountAggregationResult(1)), + newGroup(4, 407, new CountAggregationResult(1)), + newGroup(4, 408, new CountAggregationResult(1))), + newGroup(3, 303, + newGroup(4, 409, new CountAggregationResult(1)), + newGroup(4, 410, new CountAggregationResult(1)), + newGroup(4, 411, new CountAggregationResult(1)), + newGroup(4, 412, new CountAggregationResult(1))), + newGroup(3, 304, + newGroup(4, 413, new CountAggregationResult(1)), + newGroup(4, 414, new CountAggregationResult(1)), + newGroup(4, 415, new CountAggregationResult(1)), + newGroup(4, 416, new CountAggregationResult(1)))), + newGroup(2, 202, + newGroup(3, 305, + newGroup(4, 417, new CountAggregationResult(1)), + newGroup(4, 418, new CountAggregationResult(1)), + newGroup(4, 419, new CountAggregationResult(1)), + newGroup(4, 420, new CountAggregationResult(1))), + newGroup(3, 306, + newGroup(4, 421, new CountAggregationResult(1)), + newGroup(4, 422, new CountAggregationResult(1)), + newGroup(4, 423, new CountAggregationResult(1)), + newGroup(4, 424, new CountAggregationResult(1))), + newGroup(3, 307, + newGroup(4, 425, new CountAggregationResult(1)), + newGroup(4, 426, new CountAggregationResult(1)), + newGroup(4, 427, new CountAggregationResult(1)), + newGroup(4, 428, new CountAggregationResult(1))), + newGroup(3, 308, + newGroup(4, 429, new CountAggregationResult(1)), + newGroup(4, 430, new CountAggregationResult(1)), + newGroup(4, 431, new CountAggregationResult(1)), + newGroup(4, 432, new CountAggregationResult(1))))); + assertContinuation(request, result, newComposite(), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=2}, [{ 'group:401', {}, [] }, { 'group:402', {}, [] }] }] }, " + + "{ 'group:302', {}, [{ 'grouplist:c', {next=2}, [{ 'group:405', {}, [] }, { 'group:406', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'grouplist:c', {next=2}, [{ 'group:417', {}, [] }, { 'group:418', {}, [] }] }] }, " + + "{ 'group:306', {}, [{ 'grouplist:c', {next=2}, [{ 'group:421', {}, [] }, { 'group:422', {}, [] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=2}, [{ 'group:401', {}, [] }, { 'group:402', {}, [] }] }] }, " + + "{ 'group:302', {}, [{ 'grouplist:c', {next=2}, [{ 'group:405', {}, [] }, { 'group:406', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'grouplist:c', {next=2}, [{ 'group:417', {}, [] }, { 'group:418', {}, [] }] }] }, " + + "{ 'group:306', {}, [{ 'grouplist:c', {prev=0}, [{ 'group:423', {}, [] }, { 'group:424', {}, [] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'grouplist:c', {next=2}, [{ 'group:409', {}, [] }, { 'group:410', {}, [] }] }] }, " + + "{ 'group:304', {}, [{ 'grouplist:c', {next=2}, [{ 'group:413', {}, [] }, { 'group:414', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'grouplist:c', {next=2}, [{ 'group:417', {}, [] }, { 'group:418', {}, [] }] }] }, " + + "{ 'group:306', {}, [{ 'grouplist:c', {prev=0}, [{ 'group:423', {}, [] }, { 'group:424', {}, [] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2), + newUnstableOffset(newResultId(0, 1, 0), 2, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'grouplist:c', {next=2}, [{ 'group:409', {}, [] }, { 'group:410', {}, [] }] }] }, " + + "{ 'group:304', {}, [{ 'grouplist:c', {next=2}, [{ 'group:413', {}, [] }, { 'group:414', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=3, prev=0}, [" + + "{ 'group:306', {}, [{ 'grouplist:c', {next=2}, [{ 'group:421', {}, [] }, { 'group:422', {}, [] }] }] }, " + + "{ 'group:307', {}, [{ 'grouplist:c', {next=2}, [{ 'group:425', {}, [] }, { 'group:426', {}, [] }] }] }] }] }] }] }"); + } + + @Test + public void requireThatUnstableContinuationsAffectAllDecendants() { + String request = "all(group(a) each(group(b) max(1) each(group(c) max(1) each(group(d) max(1) each(output(count()))))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, + newGroup(4, 401, + newGroup(5, 501, new CountAggregationResult(1)), + newGroup(5, 502, new CountAggregationResult(1))), + newGroup(4, 402, + newGroup(5, 503, new CountAggregationResult(1)), + newGroup(5, 504, new CountAggregationResult(1)))), + newGroup(3, 302, + newGroup(4, 403, + newGroup(5, 505, new CountAggregationResult(1)), + newGroup(5, 506, new CountAggregationResult(1))), + newGroup(4, 404, + newGroup(5, 507, new CountAggregationResult(1)), + newGroup(5, 508, new CountAggregationResult(1)))))); + assertContinuation(request, result, newComposite(), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=1}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=1}, [" + + "{ 'group:401', {}, [{ 'grouplist:d', {next=1}, [" + + "{ 'group:501', {}, [] }] }] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0, 0, 0, 0, 0), 5, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=1}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=1}, [" + + "{ 'group:401', {}, [{ 'grouplist:d', {prev=0}, [" + + "{ 'group:502', {}, [] }] }] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0, 0, 0, 0, 0), 5, 1), + newUnstableOffset(newResultId(0, 0, 0, 0, 0), 4, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=1}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {prev=0}, [" + + "{ 'group:402', {}, [{ 'grouplist:d', {next=1}, [" + + "{ 'group:503', {}, [] }] }] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0, 0, 0, 0, 0), 5, 1), + newUnstableOffset(newResultId(0, 0, 0, 0, 0), 4, 1), + newUnstableOffset(newResultId(0, 0, 0), 3, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:302', {}, [{ 'grouplist:c', {next=1}, [" + + "{ 'group:403', {}, [{ 'grouplist:d', {next=1}, [" + + "{ 'group:505', {}, [] }] }] }] }] }] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsAreNotCreatedUnlessMaxIsSet() { + assertContinuation("all(group(a) each(each(output(summary()))))", + newGrouping(newGroup(2, newHitList(3, 4))), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {}, [hit:1, hit:2, hit:3, hit:4] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsCanBeSet() { + String request = "all(group(a) each(max(2) each(output(summary()))))"; + Grouping result = newGrouping(newGroup(2, newHitList(3, 4))); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 0), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 1), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 2), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 3), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 4), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 5), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsCanBeSetInSiblingGroups() { + String request = "all(group(a) each(max(2) each(output(summary()))))"; + Grouping result = newGrouping(newGroup(2, 201, newHitList(3, 4)), + newGroup(2, 202, newHitList(3, 4))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 0), + newOffset(newResultId(0, 1, 0), 3, 5)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 1), + newOffset(newResultId(0, 1, 0), 3, 4)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 2), + newOffset(newResultId(0, 1, 0), 3, 3)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 3), + newOffset(newResultId(0, 1, 0), 3, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 4), + newOffset(newResultId(0, 1, 0), 3, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 5), + newOffset(newResultId(0, 1, 0), 3, 0)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsCanBeSetInSiblingHitLists() { + String request = "all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))"; + List result = Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 0), + newOffset(newResultId(1, 0, 0), 5, 5)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 1), + newOffset(newResultId(1, 0, 0), 5, 4)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 2), + newOffset(newResultId(1, 0, 0), 5, 3)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 3), + newOffset(newResultId(1, 0, 0), 5, 2)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 4), + newOffset(newResultId(1, 0, 0), 5, 1)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 5), + newOffset(newResultId(1, 0, 0), 5, 0)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectSiblingHitLists() { + String request = "all(group(a) each(group(b) max(2) each(max(2) each(output(summary())))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, newHitList(4, 4)), + newGroup(3, 302, newHitList(4, 4)), + newGroup(3, 303, newHitList(4, 4)), + newGroup(3, 304, newHitList(4, 4))), + newGroup(2, 202, + newGroup(3, 305, newHitList(4, 4)), + newGroup(3, 306, newHitList(4, 4)), + newGroup(3, 307, newHitList(4, 4)), + newGroup(3, 308, newHitList(4, 4)))); + assertContinuation(request, result, newComposite(), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:302', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:306', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:302', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:306', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:304', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:306', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2), + newUnstableOffset(newResultId(0, 1, 0), 2, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:304', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=3, prev=0}, [" + + "{ 'group:306', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:307', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }] }] }"); + } + + // -------------------------------------------------------------------------------- + // + // Utilities. + // + // -------------------------------------------------------------------------------- + + private static CompositeContinuation newComposite(EncodableContinuation... conts) { + CompositeContinuation ret = new CompositeContinuation(); + for (EncodableContinuation cont : conts) { + ret.add(cont); + } + return ret; + } + + private static ResultId newResultId(int... indexes) { + ResultId id = ResultId.valueOf(REQUEST_ID).newChildId(ROOT_IDX); + for (int i : indexes) { + id = id.newChildId(i); + } + return id; + } + + private static OffsetContinuation newOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, 0); + } + + private static OffsetContinuation newUnstableOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, OffsetContinuation.FLAG_UNSTABLE); + } + + private static Grouping newGrouping(Group... children) { + Group root = new Group(); + root.setTag(1); + for (Group child : children) { + root.addChild(child); + } + Grouping grouping = new Grouping(); + grouping.setRoot(root); + return grouping; + } + + private static Group newGroup(int tag, AggregationResult... results) { + return newGroup(tag, tag > 1 ? tag : 0, results); + } + + private static Group newGroup(int tag, int id, AggregationResult... results) { + Group group = new Group(); + group.setTag(tag); + if (id > 0) { + group.setId(new IntegerResultNode(id)); + } + for (AggregationResult result : results) { + group.addAggregationResult(result); + } + return group; + } + + private static Group newGroup(int tag, int id, Group child0, Group... childN) { + Group group = new Group(); + group.setTag(tag); + if (id > 0) { + group.setId(new IntegerResultNode(id)); + } + group.addChild(child0); + for (Group child : childN) { + group.addChild(child); + } + return group; + } + + private static HitsAggregationResult newHitList(int hitsTag, int numHits) { + HitsAggregationResult res = new HitsAggregationResult(); + res.setTag(hitsTag); + res.setSummaryClass("default"); + for (int i = 0; i < numHits; ++i) { + res.addHit(new FS4Hit(i + 1, new GlobalId(IdString.createIdString("doc:scheme:")), 1)); + } + return res; + } + + private static void assertGroupId(String expected, ResultNode actual) { + assertLayout("all(group(a) each(output(count())))", + newGrouping(new Group().setTag(2).setId(actual)), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=" + expected + "}[]]]"); + } + + private static void assertResult(String expected, AggregationResult actual) { + actual.setTag(3); + assertLayout("all(group(a) each(output(count())))", + newGrouping(newGroup(2, 2, actual)), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=" + expected + "}[]]]"); + } + + private static void assertBuild(String request, List result) { + ResultTest test = new ResultTest(); + test.result.addAll(result); + test.request = request; + assertOutput(test); + } + + private static void assertBuildFail(String request, List result, String expected) { + ResultTest test = new ResultTest(); + test.result.addAll(result); + test.request = request; + test.expectedException = expected; + assertOutput(test); + } + + private static void assertResultCont(String request, Grouping result, Continuation cont, String expected) { + assertOutput(request, Arrays.asList(result), cont, new ResultContWriter(), expected); + } + + private static void assertResultCont(String request, List result, String expected) { + assertOutput(request, result, null, new ResultContWriter(), expected); + } + + private static void assertResultCont(String request, List result, Continuation cont, String expected) { + assertOutput(request, result, cont, new ResultContWriter(), expected); + } + + private static void assertContinuation(String request, Grouping result, String expected) { + assertOutput(request, Arrays.asList(result), null, new ContinuationWriter(), expected); + } + + private static void assertContinuation(String request, Grouping result, Continuation cont, String expected) { + assertOutput(request, Arrays.asList(result), cont, new ContinuationWriter(), expected); + } + + private static void assertContinuation(String request, List result, Continuation cont, String expected) { + assertOutput(request, result, cont, new ContinuationWriter(), expected); + } + + private static void assertLayout(String request, Grouping result, String expected) { + assertOutput(request, Arrays.asList(result), null, new LayoutWriter(), expected); + } + + private static void assertLayout(String request, List result, String expected) { + assertOutput(request, result, null, new LayoutWriter(), expected); + } + + private static void assertOutput(String request, List result, Continuation continuation, + OutputWriter writer, String expected) { + ResultTest test = new ResultTest(); + test.result.addAll(result); + test.request = request; + test.outputWriter = writer; + test.continuation = continuation; + test.expectedOutput = expected; + assertOutput(test); + } + + private static void assertOutput(ResultTest test) { + RequestBuilder reqBuilder = new RequestBuilder(REQUEST_ID); + reqBuilder.setRootOperation(GroupingOperation.fromString(test.request)); + reqBuilder.addContinuations(Arrays.asList(test.continuation)); + reqBuilder.build(); + assertEquals(reqBuilder.getRequestList().size(), test.result.size()); + + ResultBuilder resBuilder = new ResultBuilder(); + resBuilder.setHitConverter(new MyHitConverter()); + resBuilder.setTransform(reqBuilder.getTransform()); + resBuilder.setRequestId(REQUEST_ID); + for (int i = 0, len = test.result.size(); i < len; ++i) { + Grouping grouping = test.result.get(i); + grouping.setId(i); + resBuilder.addGroupingResult(grouping); + } + try { + resBuilder.build(); + if (test.expectedException != null) { + fail("Expected exception '" + test.expectedException + "'."); + } + } catch (RuntimeException e) { + if (test.expectedException == null) { + throw e; + } + assertEquals(test.expectedException, e.getMessage()); + return; + } + if (test.outputWriter != null) { + String output = test.outputWriter.write(resBuilder); + assertEquals(test.expectedOutput, output); + } + } + + private static String getCanonicalId(com.yahoo.search.result.Hit hit) { + String str = hit.getId().toString(); + if (!str.startsWith("group:")) { + return str; + } + if (str.startsWith("group:root:")) { + return "group:root"; + } + int pos = str.indexOf(':', 6); + if (pos < 0) { + return str; + } + return "group:" + str.substring(pos + 1); + } + + private static class ResultTest { + + List result = new LinkedList<>(); + String request; + String expectedOutput; + String expectedException; + OutputWriter outputWriter; + Continuation continuation; + } + + private static interface OutputWriter { + + String write(ResultBuilder builder); + } + + private static class ResultContWriter implements OutputWriter { + + @Override + public String write(ResultBuilder builder) { + return toString(builder.getContinuation()); + } + + String toString(Continuation cnt) { + if (cnt instanceof OffsetContinuation) { + OffsetContinuation off = (OffsetContinuation)cnt; + String id = off.getResultId().toString().replace(", ", "."); + return id.substring(5, id.length() - 1) + "=" + off.getOffset(); + } else if (cnt instanceof CompositeContinuation) { + List children = new LinkedList<>(); + for (Continuation child : (CompositeContinuation)cnt) { + children.add(toString(child)); + } + return children.toString(); + } else { + throw new UnsupportedOperationException(cnt.getClass().getName()); + } + } + } + + private static class ContinuationWriter implements OutputWriter { + + @Override + public String write(ResultBuilder builder) { + return toString(builder.getRoot()); + } + + String toString(com.yahoo.search.result.Hit hit) { + Map conts = new TreeMap<>(); + if (hit instanceof AbstractList) { + for (Map.Entry entry : ((AbstractList)hit).continuations().entrySet()) { + conts.put(entry.getKey(), toString(entry.getValue())); + } + } + List children = new LinkedList<>(); + if (hit instanceof HitGroup) { + for (com.yahoo.search.result.Hit childHit : (HitGroup)hit) { + if (childHit instanceof HitGroup) { + children.add(toString(childHit)); + } else { + children.add(childHit.getId().toString()); + } + } + } + return "{ '" + getCanonicalId(hit) + "', " + conts + ", " + children + " }"; + } + + String toString(Continuation cnt) { + if (cnt instanceof OffsetContinuation) { + return String.valueOf(((OffsetContinuation)cnt).getOffset()); + } else if (cnt instanceof CompositeContinuation) { + List children = new LinkedList<>(); + for (Continuation child : (CompositeContinuation)cnt) { + children.add(toString(child)); + } + Collections.sort(children); + return children.toString(); + } else { + throw new UnsupportedOperationException(cnt.getClass().getName()); + } + } + } + + private static class LayoutWriter implements OutputWriter { + + @Override + public String write(ResultBuilder builder) { + return toString(builder.getRoot()); + } + + String toString(com.yahoo.search.result.Hit hit) { + StringBuilder ret = new StringBuilder(); + ret.append(hit.getClass().getSimpleName()); + + Map members = new LinkedHashMap<>(); + if (hit instanceof GroupList) { + members.put("label", ((GroupList)hit).getLabel()); + } else if (hit instanceof HitList) { + members.put("label", ((HitList)hit).getLabel()); + } else { + members.put("id", getCanonicalId(hit)); + } + for (Map.Entry entry : hit.fields().entrySet()) { + members.put(entry.getKey(), String.valueOf(entry.getValue())); + } + ret.append(members); + + if (hit instanceof HitGroup) { + List children = new LinkedList<>(); + for (com.yahoo.search.result.Hit childHit : (HitGroup)hit) { + children.add(toString(childHit)); + } + ret.append(children); + } + return ret.toString(); + } + } + + private static class MyHitConverter implements ResultBuilder.HitConverter { + + @Override + public com.yahoo.search.result.Hit toSearchHit(String summaryClass, com.yahoo.searchlib.aggregation.Hit hit) { + return new com.yahoo.search.result.Hit("hit:" + ((FS4Hit)hit).getPath(), new Relevance(0)); + } + } + + private static class MyAggregationResult extends AggregationResult { + + @Override + public ResultNode getRank() { + return null; + } + + @Override + protected void onMerge(AggregationResult result) { + + } + + @Override + protected boolean equalsAggregation(AggregationResult obj) { + return false; + } + } + + private static class MyResultNode extends ResultNode { + + @Override + protected void set(ResultNode rhs) { + + } + + @Override + protected int onCmp(ResultNode rhs) { + return 0; + } + + @Override + public long getInteger() { + return 0; + } + + @Override + public double getFloat() { + return 0; + } + + @Override + public String getString() { + return null; + } + + @Override + public byte[] getRaw() { + return new byte[0]; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java new file mode 100644 index 00000000000..8124b748f1f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Simon Thoresen + */ +public class ResultIdTestCase { + + @Test + public void requireThatStartsWithWorks() { + assertFalse(ResultId.valueOf().startsWith(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1).startsWith(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1, 1).startsWith(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1, 1, 2).startsWith(1, 1, 2, 3)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1, 1, 2, 3)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1, 1, 2)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1, 1)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith()); + } + + @Test + public void requireThatChildIdStartsWithParentId() { + ResultId parentId = ResultId.valueOf(1, 1, 2); + ResultId childId = parentId.newChildId(3); + assertTrue(childId.startsWith(1, 1, 2)); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(ResultId.valueOf(1, 1, 2, 3).hashCode(), ResultId.valueOf(1, 1, 2, 3).hashCode()); + assertFalse(ResultId.valueOf(1, 1, 2, 3).hashCode() == ResultId.valueOf(5, 8, 13, 21).hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + assertEquals(ResultId.valueOf(1, 1, 2, 3), ResultId.valueOf(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1, 1, 2, 3).equals(ResultId.valueOf(5, 8, 13, 21))); + } + + @Test + public void requireThatResultIdCanBeEncoded() { + assertEncode("BIBCBCBEBG", ResultId.valueOf(1, 1, 2, 3)); + assertEncode("BIBKCBACBKCCK", ResultId.valueOf(5, 8, 13, 21)); + } + + @Test + public void requireThatResultIdCanBeDecoded() { + assertDecode(ResultId.valueOf(1, 1, 2, 3), "BIBCBCBEBG"); + assertDecode(ResultId.valueOf(5, 8, 13, 21), "BIBKCBACBKCCK"); + } + + private static void assertEncode(String expected, ResultId toEncode) { + IntegerEncoder encoder = new IntegerEncoder(); + toEncode.encode(encoder); + assertEquals(expected, encoder.toString()); + } + + private static void assertDecode(ResultId expected, String toDecode) { + IntegerDecoder decoder = new IntegerDecoder(toDecode); + assertTrue(decoder.hasNext()); + assertEquals(expected, ResultId.decode(decoder)); + assertFalse(decoder.hasNext()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java new file mode 100644 index 00000000000..d1cbf403c1a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -0,0 +1,516 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.handler.test; + +import com.yahoo.container.Container; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.container.jdisc.AsyncHttpResponse; +import com.yahoo.container.jdisc.HttpRequest; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.processing.handler.ResponseStatus; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.HttpSearchResponse; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class SearchHandlerTestCase { + + private static final String testDir = "src/test/java/com/yahoo/search/handler/test/config"; + private static String tempDir = ""; + private static String configId = null; + + @Rule + public TemporaryFolder tempfolder = new TemporaryFolder(); + + private RequestHandlerTestDriver driver = null; + private HandlersConfigurerTestWrapper configurer = null; + private SearchHandler searchHandler; + + @Before + public void startUp() throws IOException { + File cfgDir = tempfolder.newFolder("SearchHandlerTestCase"); + tempDir = cfgDir.getAbsolutePath(); + configId = "dir:" + tempDir; + + IOUtils.copyDirectory(new File(testDir), cfgDir, 1); // make configs active + generateComponentsConfigForActive(); + + configurer = new HandlersConfigurerTestWrapper(new Container(), configId); + searchHandler = (SearchHandler)configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + driver = new RequestHandlerTestDriver(searchHandler); + } + + @After + public void shutDown() { + if (configurer != null) configurer.shutdown(); + if (driver != null) driver.close(); + } + + private void generateComponentsConfigForActive() throws IOException { + File activeConfig = new File(tempDir); + SearchChainConfigurerTestCase. + createComponentsConfig(new File(activeConfig, "chains.cfg").getPath(), + new File(activeConfig, "handlers.cfg").getPath(), + new File(activeConfig, "components.cfg").getPath()); + } + + private SearchHandler fetchSearchHandler(HandlersConfigurerTestWrapper configurer) { + return (SearchHandler) configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + } + + @Test + public void testNullQuery() throws Exception { + assertEquals("\n" + + "\n" + + " \n" + + " 1.0\n" + + " testHit\n" + + " \n" + + "\n", + driver.sendRequest("http://localhost?format=xml").readAll() + ); + } + + private String render(AsyncHttpResponse response) throws Exception { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + response.render(stream, null, null); + response.complete(); + return stream.toString(); + } + + @Test + public void testFailing() throws Exception { + assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=classLoadingError").readAll().contains("NoClassDefFoundError")); + } + + @Test + public synchronized void testPluginError() throws Exception { + assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=exceptionInPlugin").readAll().contains("NullPointerException")); + } + + @Test + public synchronized void testWorkingReconfiguration() throws Exception { + assertJsonResult("http://localhost?query=abc", driver); + + // reconfiguration + IOUtils.copyDirectory(new File(testDir, "handlers2"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + // ...and check the resulting config + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Have a new instance of the search handler", searchHandler != newSearchHandler); + assertNotNull("Have the new search chain", fetchSearchHandler(configurer).getSearchChainRegistry().getChain("hello")); + assertNull("Don't have the new search chain", fetchSearchHandler(configurer).getSearchChainRegistry().getChain("classLoadingError")); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(searchHandler)) { + assertJsonResult("http://localhost?query=abc", newDriver); + } + } + + @Test + @Ignore //TODO: Must be done at the ConfiguredApplication level, not handlers configurer? Also, this must be rewritten as the above + public synchronized void testFailedReconfiguration() throws Exception { + assertXmlResult(driver); + + // attempt reconfiguration + IOUtils.copyDirectory(new File(testDir, "handlersInvalid"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + RequestHandler newMockHandler = configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.test.MockHandler"); + assertTrue("Reconfiguration failed: Kept the existing instance of the search handler", searchHandler == newSearchHandler); + assertNull("Reconfiguration failed: No mock handler", newMockHandler); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(searchHandler)) { + assertXmlResult(newDriver); + } + } + + @Test + public void testResponseBasics() { + Query q = new Query("?query=dummy&tracelevel=3"); + q.trace("nalle", 1); + Result r = new Result(q); + r.hits().addError(ErrorMessage.createUnspecifiedError("bamse")); + r.hits().add(new Hit("http://localhost/dummy", 0.5)); + HttpSearchResponse s = new HttpSearchResponse(200, r, q, new DefaultRenderer()); + assertEquals("text/xml", s.getContentType()); + assertNull(s.getCoverage()); + assertEquals("query 'dummy'", s.getParsedQuery()); + assertEquals(5000, s.getTiming().getTimeout()); + } + + @Test + public void testInvalidYqlQuery() throws Exception { + IOUtils.copyDirectory(new File(testDir, "config_yql"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Have a new instance of the search handler", searchHandler != newSearchHandler); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(newSearchHandler)) { + RequestHandlerTestDriver.MockResponseHandler responseHandler = newDriver.sendRequest( + "http://localhost/search/?yql=select%20*%20from%20foo%20where%20bar%20%3E%201453501295%27%3B"); + responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + } + } + + // Query handling takes a different code path when a query profile is active, so we test both paths. + @Test + public void testInvalidQueryParamWithQueryProfile() throws Exception { + try (RequestHandlerTestDriver newDriver = driverWithConfig("config_invalid_param")) { + testInvalidQueryParam(newDriver); + } + } + @Test + public void testInvalidQueryParamWithoutQueryProfile() throws Exception { + testInvalidQueryParam(driver); + } + private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) { + RequestHandlerTestDriver.MockResponseHandler responseHandler = + testDriver.sendRequest("http://localhost/search/?query=status_code%3A0&hits=20&offset=-20"); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + assertThat(response, containsString("offset")); + assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code)); + } + + @Test + public void testWebServiceStatus() { + RequestHandlerTestDriver.MockResponseHandler responseHandler = + driver.sendRequest("http://localhost/search/?query=web_service_status_code"); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(406)); + assertThat(response, containsString("\"code\":" + 406)); + } + + @Test + public void testNormalResultImplicitDefaultRendering() throws Exception { + assertJsonResult("http://localhost?query=abc", driver); + } + + @Test + public void testNormalResultExplicitDefaultRendering() throws Exception { + assertJsonResult("http://localhost?query=abc&format=default", driver); + } + + @Test + public void testNormalResultXmlAliasRendering() throws Exception { + assertXmlResult("http://localhost?query=abc&format=xml", driver); + } + + @Test + public void testNormalResultJsonAliasRendering() throws Exception { + assertJsonResult("http://localhost?query=abc&format=json", driver); + } + + @Test + public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { + assertXmlResult("http://localhost?query=abc&format=DefaultRenderer", driver); + } + + @Test + public void testNormalResultExplicitDefaultRenderingFullRendererName2() throws Exception { + assertJsonResult("http://localhost?query=abc&format=JsonRenderer", driver); + } + + @Test + public void testResultLegacyTiledFormat() throws Exception { + assertTiledResult("http://localhost?query=abc&format=tiled", driver); + } + + @Test + public void testResultLegacyPageFormat() throws Exception { + assertPageResult("http://localhost?query=abc&format=page", driver); + } + + private static final String xmlResult = + "\n" + + "\n" + + " \n" + + " 1.0\n" + + " testHit\n" + + " \n" + + "\n"; + private void assertXmlResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), xmlResult); + } + private void assertXmlResult(RequestHandlerTestDriver driver) throws Exception { + assertXmlResult("http://localhost?query=abc", driver); + } + + private static final String jsonResult = "{\"root\":{" + + "\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0}," + + "\"children\":[" + + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + + "]}}"; + private void assertJsonResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), jsonResult); + } + + private static final String tiledResult = + "\n" + + "\n" + + "\n" + + " \n" + + " testHit\n" + + " testHit\n" + + " \n" + + "\n" + + "\n"; + private void assertTiledResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), tiledResult); + } + + private static final String pageResult = + "\n" + + "\n" + + "\n" + + " \n" + + " \n" + + " testHit\n" + + " testHit\n" + + " \n" + + " \n" + + "\n" + + "\n"; + private void assertPageResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), pageResult); + } + + private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { + assertEquals(expected, response.readAll()); + assertEquals(200, response.getStatus()); + } + + @Test + public void testFaultyHandlers() throws Exception { + assertHandlerResponse(500, null, "NullReturning"); + assertHandlerResponse(500, null, "NullReturningAsync"); + assertHandlerResponse(500, null, "Throwing"); + assertHandlerResponse(500, null, "ThrowingAsync"); + } + + @Test + public void testForwardingHandlers() throws Exception { + assertHandlerResponse(200, jsonResult, "ForwardingAsync"); + + // Fails because we are forwarding from a sync to an async handler - + // the sync handler will respond with status 500 because the async one has not produced a response yet. + // Disabled because this fails due to a race and is therefore unstable + // assertHandlerResponse(500, null, "Forwarding"); + } + + private void assertHandlerResponse(int status, String responseData, String handlerName) throws Exception { + RequestHandler forwardingHandler = configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.test.SearchHandlerTestCase$" + handlerName + "Handler"); + try (RequestHandlerTestDriver forwardingDriver = new RequestHandlerTestDriver(forwardingHandler)) { + RequestHandlerTestDriver.MockResponseHandler response = forwardingDriver.sendRequest("http://localhost/" + handlerName + "?query=test"); + response.awaitResponse(); + assertEquals("Expected HTTP status", status, response.getStatus()); + if (responseData == null) + assertEquals("Connection closed with no data", null, response.read()); + else + assertEquals(responseData, response.readAll()); + } + } + + private RequestHandlerTestDriver driverWithConfig(String configDirectory) throws Exception { + IOUtils.copyDirectory(new File(testDir, configDirectory), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Should have a new instance of the search handler", searchHandler != newSearchHandler); + return new RequestHandlerTestDriver(newSearchHandler); + } + + /** Referenced from config */ + public static class TestSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = new Result(query); + Hit hit = new Hit("testHit"); + hit.setField("uri", "testHit"); + result.hits().add(hit); + + if (query.getModel().getQueryString().contains("web_service_status_code")) + result.hits().addError(new ErrorMessage(406, "Test web service code")); + + return result; + } + } + + /** Referenced from config */ + public static class ClassLoadingErrorSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + throw new NoClassDefFoundError(); // Simulate typical OSGi problem + } + } + + /** Referenced from config */ + public static class ExceptionInPluginSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + try { + result.hits().add(null); // Trigger NullPointerException + } catch (NullPointerException e) { + throw new RuntimeException("Message", e); + } + return result; + } + } + + /** Referenced from config */ + public static class HelloWorldSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("HelloWorld")); + return result; + } + } + + /** Referenced from config */ + public static class ForwardingHandler extends ThreadedHttpRequestHandler { + + private final SearchHandler searchHandler; + + public ForwardingHandler(SearchHandler searchHandler) { + super(Executors.newSingleThreadExecutor(), null, false); + this.searchHandler = searchHandler; + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + try { + HttpRequest forwardRequest = + new HttpRequest.Builder(httpRequest).uri(new URI("http://localhost/search/?query=test")).createDirectRequest(); + return searchHandler.handle(forwardRequest); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + } + + /** Referenced from config */ + public static class ForwardingAsyncHandler extends ThreadedHttpRequestHandler { + + private final SearchHandler searchHandler; + + public ForwardingAsyncHandler(SearchHandler searchHandler) { + super(Executors.newSingleThreadExecutor(), null, true); + this.searchHandler = searchHandler; + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + try { + HttpRequest forwardRequest = + new HttpRequest.Builder(httpRequest).uri(new URI("http://localhost/search/?query=test")).createDirectRequest(); + return searchHandler.handle(forwardRequest); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + } + + /** Referenced from config */ + public static class NullReturningHandler extends ThreadedHttpRequestHandler { + + public NullReturningHandler() { + super(Executors.newSingleThreadExecutor(), null, false); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + return null; + } + + } + + /** Referenced from config */ + public static class NullReturningAsyncHandler extends ThreadedHttpRequestHandler { + + public NullReturningAsyncHandler() { + super(Executors.newSingleThreadExecutor(), null, true); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + return null; + } + + } + + /** Referenced from config */ + public static class ThrowingHandler extends ThreadedHttpRequestHandler { + + public ThrowingHandler() { + super(Executors.newSingleThreadExecutor(), null, false); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + throw new RuntimeException(); + } + + } + + /** Referenced from config */ + public static class ThrowingAsyncHandler extends ThreadedHttpRequestHandler { + + public ThrowingAsyncHandler() { + super(Executors.newSingleThreadExecutor(), null, true); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + throw new RuntimeException(); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/.gitignore b/container-search/src/test/java/com/yahoo/search/handler/test/config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg new file mode 100644 index 00000000000..0336e06f54b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg @@ -0,0 +1,14 @@ +chains[3] +chains[0].id default +chains[0].components[1] +chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +chains[1].id classLoadingError +chains[1].components[1] +chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher +chains[2].id exceptionInPlugin +chains[2].components[1] +chains[2].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher +components[3] +components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher +components[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg new file mode 100644 index 00000000000..a1009f05310 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg @@ -0,0 +1,5 @@ +queryprofile[1] +queryprofile[0].id "default" +queryprofile[0].property[1] +queryprofile[0].property[0].name "maxOffset" +queryprofile[0].property[0].value "1000" diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg new file mode 100644 index 00000000000..72c1af55bc5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg @@ -0,0 +1,6 @@ +chains[1] +chains[0].id default +chains[0].components[1] +chains[0].components[0] com.yahoo.search.yql.MinimalQueryInserter +components[1] +components[0].id com.yahoo.search.yql.MinimalQueryInserter diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg new file mode 100644 index 00000000000..96843d78aae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg @@ -0,0 +1,8 @@ +handler[7] +handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningHandler +handler[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningAsyncHandler +handler[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingHandler +handler[4].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingAsyncHandler +handler[5].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingHandler +handler[6].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingAsyncHandler diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg new file mode 100644 index 00000000000..2437efdec4f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg @@ -0,0 +1,10 @@ +chains[2] +chains[0].id default +chains[0].components[1] +chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +chains[1].id hello +chains[1].components[1] +chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$HelloWorldSearcher +components[2] +components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$HelloWorldSearcher diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg new file mode 100644 index 00000000000..691b37b4955 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg @@ -0,0 +1,3 @@ +handler[2] +handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ErrorOnInitializationHandler diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/index-info.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java b/container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java new file mode 100644 index 00000000000..feddcbde505 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.match.test; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.search.Result; +import com.yahoo.search.match.DocumentDb; +import com.yahoo.text.MapParser; +import org.junit.Ignore; +import org.junit.Test; +import static org.junit.Assert.*; + +public class DocumentDbTest { + + @Test + @Ignore + public void testWand() { + DocumentDb db = new DocumentDb(); + db.put(createFeatureDocument("1","[a:7, b:5, c:3]")); + db.put(createFeatureDocument("2", "[a:2, b:1, c:4]")); + //Result r = execute(createWandQuery("[a:1, b:3, c:5]")); + //assertEquals(67,r.hits().get(0).getRelevance()); + //assertEquals(25, r.hits().get(1).getRelevance()); + } + + private DocumentPut createFeatureDocument(String localId, String features) { + DocumentType type = new DocumentType("withFeature"); + type.addField("features", new WeightedSetDataType(DataType.STRING,true,true)); + Document d = new Document(type, new DocumentId("id:test::" + localId)); + d.setFieldValue("features",fillFromString(new WeightedSet(DataType.STRING), features)); + return new DocumentPut(d); + } + + // TODO: Move to weightedset + // TODO: Don't pass through a map + private WeightedSet fillFromString(WeightedSet s, String values) { + //new IntMapParser().parse(); + return null; + } + + private static class IntMapParser extends MapParser { + + @Override + protected Integer parseValue(String s) { + return Integer.parseInt(s); + } + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java new file mode 100644 index 00000000000..052ea62d4f0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.config.test; + +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.pagetemplates.model.*; + +import java.util.List; + +/** + * @author Jon Bratseth + */ +public class MapPageTemplateXMLReadingTestCase extends junit.framework.TestCase { + + private String root="src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/"; + + public void testMap1() { + PageTemplateRegistry registry=new PageTemplateXMLReader().read(root); + assertCorrectMap1(registry.getComponent("map1")); + } + + private void assertCorrectMap1(PageTemplate page) { + assertNotNull("map1 was read",page); + Section root=page.getSection(); + assertTrue(((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(0)).elements().get(0) instanceof Placeholder); + assertTrue(((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(1)).elements().get(0) instanceof Placeholder); + assertTrue(((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(0)).elements().get(0) instanceof Placeholder); + assertTrue(((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(1)).elements().get(0) instanceof Placeholder); + assertEquals("box1source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(0)).elements().get(0)).getId()); + assertEquals("box2source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(1)).elements().get(0)).getId()); + assertEquals("box3source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(0)).elements().get(0)).getId()); + assertEquals("box4source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(1)).elements().get(0)).getId()); + + MapChoice map=(MapChoice)root.elements().get(2); + assertEquals("box1source",map.placeholderIds().get(0)); + assertEquals("box2source",map.placeholderIds().get(1)); + assertEquals("box3source",map.placeholderIds().get(2)); + assertEquals("box4source",map.placeholderIds().get(3)); + assertEquals("source1",((Source)((List)map.values().get(0)).get(0)).getName()); + assertEquals("source2",((Source)((List)map.values().get(1)).get(0)).getName()); + assertEquals("source3",((Source)((List)map.values().get(2)).get(0)).getName()); + assertEquals("source4",((Source)((List)map.values().get(3)).get(0)).getName()); + + PageTemplateXMLReadingTestCase.assertCorrectSources("source1 source2 source3 source4",page); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java new file mode 100644 index 00000000000..7832719412a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java @@ -0,0 +1,279 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.config.test; + +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import com.yahoo.search.pagetemplates.config.PageTemplateConfigurer; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.Renderer; +import com.yahoo.search.pagetemplates.model.Section; +import com.yahoo.search.pagetemplates.model.Source; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author Jon Bratseth + */ +public class PageTemplateXMLReadingTestCase extends junit.framework.TestCase { + + private String root="src/test/java/com/yahoo/search/pagetemplates/config/test/"; + + public void testExamples() { + PageTemplateRegistry registry=new PageTemplateXMLReader().read(root + "examples"); + assertCorrectSerp(registry.getComponent("serp")); + assertCorrectSlottingSerp(registry.getComponent("slottingSerp")); + assertCorrectRichSerp(registry.getComponent("richSerp")); + assertCorrectRicherSerp(registry.getComponent("richerSerp")); + assertCorrectIncluder(registry.getComponent("includer")); + assertCorrectGeneric(registry.getComponent("generic")); + } + + public void testConfigReading() { + PageTemplatesConfig config = new PageTemplatesConfig(new PageTemplatesConfig.Builder() + .page("\n
\n
\n\n") + .page("\n
\n
\n
\n \n \n \n \n
\n
\n
\n
\n
\n
\n
\n\n") + .page("\n
\n
\n\n") + .page("\n \n
\n
\n
\n \n \n
\n \n \n \n \n \n \n \n \n \n 5\n #ff00ff\n \n \n \n origin=twitter\n \n \n \n \n \n \n \n
\n
\n \n
\n \n
\n
\n
\n
\n \n
\n
\n \n\n") + .page("\n
\n
\n
\n
\n\n") + ); + PageTemplateRegistry registry = PageTemplateConfigurer.toRegistry(config); + assertCorrectSlottingSerp(registry.getComponent("slottingSerp")); + assertCorrectRichSerp(registry.getComponent("richSerp")); + assertCorrectRicherSerp(registry.getComponent("richerSerp")); + } + + public void testInvalidFilename() { + try { + PageTemplateRegistry registry=new PageTemplateXMLReader().read(root + "examples/invalidfilename"); + assertEquals(0,registry.allComponents().size()); + fail("Should have caused an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("The file name of page template 'notinvalid' must be 'notinvalid.xml' but was 'invalid.xml'",e.getMessage()); + } + } + + protected void assertCorrectSerp(PageTemplate page) { + assertNotNull("'serp' was read",page); + Section rootSection=page.getSection(); + assertNotNull(rootSection); + assertEquals("mainAndRight",rootSection.getLayout().getName()); + Section main=(Section)rootSection.elements(Section.class).get(0); + assertEquals("column",main.getLayout().getName()); + assertEquals("main",main.getRegion()); + assertEquals("web",((Source)main.elements(Source.class).get(0)).getName()); + Section right=(Section)rootSection.elements(Section.class).get(1); + assertEquals("column",right.getLayout().getName()); + assertEquals("right",right.getRegion()); + assertEquals("ads",((Source)right.elements(Source.class).get(0)).getName()); + } + + protected void assertCorrectSlottingSerp(PageTemplate page) { + assertNotNull("'slotting serp' was read",page); + Section rootSection=page.getSection(); + Section main=(Section)rootSection.elements(Section.class).get(0); + assertEquals("-[rank]",main.getOrder().toString()); + assertEquals(Source.any,main.elements(Source.class).get(0)); + + assertCorrectSources("* ads",page); + } + + protected void assertCorrectRichSerp(PageTemplate page) { + assertNotNull("'rich serp' was read",page); + Section rootSection=page.getSection(); + assertNotNull(rootSection); + assertEquals("mainAndRight",rootSection.getLayout().getName()); + + Section main=(Section)rootSection.elements(Section.class).get(0); + assertEquals("row",main.getLayout().getName()); + assertEquals("main",main.getRegion()); + Section leftMain=(Section)main.elements(Section.class).get(0); + assertEquals("column",leftMain.getLayout().getName()); + Section imageBar=(Section)leftMain.elements(Section.class).get(0); + assertEquals("row",imageBar.getLayout().getName()); + assertEquals(5,imageBar.getMax()); + assertEquals("annealing",((Choice)imageBar.elements(Source.class).get(0)).getMethod().toString()); + assertEquals("images",((Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(0).get(0)).getName()); + assertEquals("flickr",((Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(1).get(0)).getName()); + Section richElement=(Section)leftMain.elements(Section.class).get(1); + assertEquals(1,richElement.getMax()); + assertEquals("[source 'local', source 'map', source 'video', source 'ticker', source 'weather']",richElement.elements(Source.class).toString()); + Section webResults=(Section)leftMain.elements(Section.class).get(2); + assertEquals("-[rank]",webResults.getOrder().toString()); + assertEquals(10,webResults.getMax()); + assertEquals("[source 'web', source 'news']",webResults.elements(Source.class).toString()); + Section rightMain=(Section)main.elements(Section.class).get(1); + assertEquals("column",rightMain.getLayout().getName()); + assertEquals("+[source]",rightMain.getOrder().toString()); + assertEquals("[source 'answers', source 'blogs', source 'twitter']",rightMain.elements(Source.class).toString()); + + Section right=(Section)rootSection.elements(Section.class).get(1); + assertEquals("column",right.getLayout().getName()); + assertEquals("right",right.getRegion()); + assertEquals("ads",((Source)right.elements(Source.class).get(0)).getName()); + } + + protected void assertCorrectRicherSerp(PageTemplate page) { + assertNotNull("'richer serp' was read",page); + + // Check resolution as we go + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(Choice.createSingleton(page),null,null); + + Section root=page.getSection(); + assertNotNull(root); + assertEquals("column",root.getLayout().getName()); + + assertEquals("Sections was correctly imported and combined with the section in this",4,root.elements(Section.class).size()); + + assertCorrectHeader((Section)root.elements(Section.class).get(0)); + + Section body=(Section)root.elements(Section.class).get(1); + assertEquals("mainAndRight",body.getLayout().getName()); + + Section main=(Section)body.elements(Section.class).get(0); + assertEquals("row",main.getLayout().getName()); + assertEquals("main",main.getRegion()); + + Section leftMain=(Section)main.elements(Section.class).get(0); + assertEquals("column",leftMain.getLayout().getName()); + assertEquals(1,resolution.getResolution((Choice)leftMain.elements(Section.class).get(0))); + + Section imageBar=(Section)((Choice)leftMain.elements(Section.class).get(0)).alternatives().get(0).get(0); + assertEquals("row",imageBar.getLayout().getName()); + assertEquals(5,imageBar.getMax()); + assertEquals(2,((Choice)imageBar.elements(Source.class).get(0)).alternatives().size()); + assertEquals("images",((Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(0).get(0)).getName()); + assertEquals(1,resolution.getResolution((Choice)imageBar.elements(Source.class).get(0))); + assertEquals(1,resolution.getResolution((Choice)imageBar.elements(Renderer.class).get(0))); + + Source flickrSource=(Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(1).get(0); + assertEquals("flickr",flickrSource.getName()); + assertEquals(1,flickrSource.renderers().size()); + assertEquals("mouseOverImage",((Renderer)flickrSource.renderers().get(0)).getName()); + + Source twitpicSource=(Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(1).get(1); + assertEquals("twitpic",twitpicSource.getName()); + assertEquals(1,twitpicSource.parameters().size()); + assertEquals("origin=twitter",twitpicSource.parameters().get("filter")); + assertEquals(2,((Choice)twitpicSource.renderers().get(0)).alternatives().size()); + assertEquals(1,resolution.getResolution((Choice)twitpicSource.renderers().get(0))); + + Renderer mouseOverImageRenderer=(Renderer)((Choice)twitpicSource.renderers().get(0)).alternatives().get(0).get(0); + assertEquals("mouseOverImage", mouseOverImageRenderer.getName()); + assertEquals(2, mouseOverImageRenderer.parameters().size()); + assertEquals("5", mouseOverImageRenderer.parameters().get("hovertime")); + assertEquals("#ff00ff", mouseOverImageRenderer.parameters().get("borderColor")); + assertEquals("regularImage",((Renderer)((Choice)twitpicSource.renderers().get(0)).alternatives().get(1).get(0)).getName()); + assertEquals(2,((Choice)imageBar.elements(Renderer.class).get(0)).alternatives().size()); + assertEquals("regularImageBox",((Renderer)((Choice)imageBar.elements(Renderer.class).get(0)).alternatives().get(0).get(0)).getName()); + assertEquals("newImageBox",((Renderer)((Choice)imageBar.elements(Renderer.class).get(0)).alternatives().get(1).get(0)).getName()); + + Section richElement=(Section)((Choice)leftMain.elements(Section.class).get(0)).get(0).get(1); + assertEquals(1,richElement.getMax()); + assertEquals("[source 'local', source 'map', source 'video', source 'ticker', source 'weather']",richElement.elements(Source.class).toString()); + + Section webResults=(Section)((Choice)leftMain.elements(Section.class).get(0)).get(1).get(0); + assertEquals("+[source]",webResults.getOrder().toString()); + assertEquals(10,webResults.getMax()); + assertEquals("[source 'web', source 'news']",webResults.elements(Source.class).toString()); + + Section rightMain=(Section)main.elements(Section.class).get(1); + assertEquals("column",rightMain.getLayout().getName()); + assertEquals("+[source]",rightMain.getOrder().toString()); + assertEquals("[source 'answers', source 'blogs', source 'twitter']",rightMain.elements(Source.class).toString()); + + Section right=(Section)body.elements(Section.class).get(1); + assertEquals("column",right.getLayout().getName()); + assertEquals("right",right.getRegion()); + assertEquals("ads",((Source)right.elements(Source.class).get(0)).getName()); + assertEquals("newAdBox",((Renderer)right.elements(Renderer.class).get(0)).getName()); + assertEquals("-[rank] +clickProbability",right.getOrder().toString()); + + assertCorrectFooter((Section)root.elements(Section.class).get(2)); + assertEquals("extraFooter",((Section)root.elements(Section.class).get(3)).getId()); + + // Check getSources() + assertCorrectSources("ads answers blogs flickr global images local map news notifications " + + "popularSearches ticker topArticles twitpic twitter video weather web",page); + } + + static void assertCorrectSources(String expectedSourceNameString,PageTemplate page) { + String[] expectedSourceNames=expectedSourceNameString.split(" "); + Set sourceNames=new HashSet<>(); + for (Source source : page.getSources()) + sourceNames.add(source.getName()); + assertEquals("Expected " + expectedSourceNames.length + " elements in " + sourceNames, + expectedSourceNames.length,sourceNames.size()); + for (String expectedSourceName : expectedSourceNames) + assertTrue("Sources did not include '" + expectedSourceName+ "'",sourceNames.contains(expectedSourceName)); + } + + protected void assertCorrectIncluder(PageTemplate page) { + assertNotNull("'includer' was read",page); + + Resolution resolution=new DeterministicResolver().resolve(Choice.createSingleton(page),null,null); + + Section case1=(Section)page.getSection().elements(Section.class).get(0); + assertCorrectHeader((Section)case1.elements(Section.class).get(0)); + assertCorrectFooter((Section)case1.elements(Section.class).get(1)); + + Section case2=(Section)page.getSection().elements(Section.class).get(1); + assertCorrectHeader((Section)((Choice)case2.elements(Section.class).get(0)).get(0).get(0)); + assertCorrectFooter((Section)((Choice)case2.elements(Section.class).get(0)).get(1).get(0)); + assertEquals(1,resolution.getResolution((Choice)case2.elements(Section.class).get(0))); + + Section case3=(Section)page.getSection().elements(Section.class).get(2); + assertCorrectHeader((Section)((Choice)case3.elements(Section.class).get(0)).get(0).get(0)); + assertCorrectFooter((Section)((Choice)case3.elements(Section.class).get(0)).get(1).get(0)); + assertEquals(1,resolution.getResolution((Choice)case3.elements(Section.class).get(0))); + + Section case4=(Section)page.getSection().elements(Section.class).get(3); + assertEquals("first",((Section)((Choice)case4.elements(Section.class).get(0)).get(0).get(0)).getId()); + assertCorrectHeader((Section)((Choice)case4.elements(Section.class).get(0)).get(1).get(0)); + assertEquals("middle",((Section)((Choice)case4.elements(Section.class).get(0)).get(2).get(0)).getId()); + assertCorrectFooter((Section)((Choice)case4.elements(Section.class).get(0)).get(3).get(0)); + assertEquals("last",((Section)((Choice)case4.elements(Section.class).get(0)).get(4).get(0)).getId()); + assertEquals(4,resolution.getResolution((Choice)case4.elements(Section.class).get(0))); + + Section case5=(Section)page.getSection().elements(Section.class).get(4); + assertEquals(2,((Choice)case5.elements(Section.class).get(0)).alternatives().size()); + assertCorrectHeader((Section)((Choice)case5.elements(Section.class).get(0)).get(0).get(0)); + assertEquals("second",((Section)((Choice)case5.elements(Section.class).get(0)).get(1).get(0)).getId()); + assertEquals(1,resolution.getResolution((Choice)case5.elements(Section.class).get(0))); + + // This case - a choice inside a choice - makes little sense. It is included as a reminder - + // what we really want is to be able to include some additional alternatives of a choice, + // but without any magic. That requires allowing alternative as a top-level tag, or something + Section case6=(Section)page.getSection().elements(Section.class).get(5); + Choice includerChoice=(Choice)case6.elements().get(0); + Choice includedChoice=(Choice)includerChoice.alternatives().get(0).get(0); + assertCorrectFooter((Section)includedChoice.alternatives().get(0).get(0)); + } + + private void assertCorrectHeader(Section header) { + assertEquals("row",header.getLayout().getName()); + assertEquals(2,header.elements(Section.class).size()); + assertEquals( "global",((Source)((Section)header.elements(Section.class).get(0)).elements(Source.class).get(0)).getName()); + assertEquals("notifications",((Source)((Section)header.elements(Section.class).get(1)).elements(Source.class).get(0)).getName()); + } + + private void assertCorrectFooter(Section footer) { + assertEquals("row",footer.getLayout().getName()); + assertTrue(footer.elements(Section.class).isEmpty()); + assertEquals("popularSearches",((Source)footer.elements(Source.class).get(0)).getName()); + } + + private void assertCorrectGeneric(PageTemplate page) { + assertEquals("image", ((Source)((Section)page.getSection().elements(Section.class).get(0)).elements(Source.class).get(0)).getName()); + assertEquals("flickr", ((Source)((Section)page.getSection().elements(Section.class).get(0)).elements(Source.class).get(1)).getName()); + assertEquals(Source.any,((Section)page.getSection().elements(Section.class).get(1)).elements(Source.class).get(0)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml new file mode 100644 index 00000000000..9ebdaeb9302 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml @@ -0,0 +1,6 @@ + + + +
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml new file mode 100644 index 00000000000..36b0ae6430c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml @@ -0,0 +1,10 @@ + + + +
+
+
+
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml new file mode 100644 index 00000000000..0866aaaa583 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml @@ -0,0 +1,5 @@ + + +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml new file mode 100644 index 00000000000..319f3058d24 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml @@ -0,0 +1,5 @@ + + +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml new file mode 100644 index 00000000000..a894e8b9a3e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml @@ -0,0 +1,7 @@ + + +
+
+
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml new file mode 100644 index 00000000000..6d4f6121991 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml @@ -0,0 +1,36 @@ + + +
+ + +
+
+ + + + +
+
+ + + + +
+
+ +
+ +
+ +
+ +
+
+ +
+
+ + + +
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml new file mode 100644 index 00000000000..0e799a472de --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml @@ -0,0 +1,4 @@ + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml new file mode 100644 index 00000000000..c13fdcdbbca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml @@ -0,0 +1,21 @@ + + + +
+
+
+
+
+
+
+
+ + + + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml new file mode 100644 index 00000000000..32ab6086b82 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml @@ -0,0 +1,17 @@ + + +
+
+
+ + + + +
+
+
+
+
+
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml new file mode 100644 index 00000000000..d3e11288ef1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml @@ -0,0 +1,45 @@ + + + +
+
+
+ + +
+ + + + + + + + + + 5 + #ff00ff + + + + origin=twitter + + + + + + + +
+
+ +
+ +
+
+
+
+ +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml new file mode 100644 index 00000000000..194c551f84c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml @@ -0,0 +1,5 @@ + + +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml new file mode 100644 index 00000000000..301f7e77edb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml @@ -0,0 +1,5 @@ + + +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml new file mode 100644 index 00000000000..4a5b6b3a1dd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml @@ -0,0 +1,4 @@ + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml new file mode 100644 index 00000000000..40cc6b6935e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml @@ -0,0 +1,41 @@ + + + + + + source3-1 + + + + source1-1 + + + + source2-1 + + + + source3-2 + + + + source1-2 + + + + source2-2 + + + + source3-3 + + + + source1-3 + + + + source2-3 + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java new file mode 100644 index 00000000000..dc20e5483ca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class AnySourceTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("AnySource.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // all three sources, ordered by relevance, source 3 first in each relevance group + HitGroup hits=result.hits(); + assertEquals(9,hits.size()); + assertEquals("source3-1",hits.get(0).getId().stringValue()); + assertEquals("source1-1",hits.get(1).getId().stringValue()); + assertEquals("source2-1",hits.get(2).getId().stringValue()); + assertEquals("source3-2",hits.get(3).getId().stringValue()); + assertEquals("source1-2",hits.get(4).getId().stringValue()); + assertEquals("source2-2",hits.get(5).getId().stringValue()); + assertEquals("source3-3",hits.get(6).getId().stringValue()); + assertEquals("source1-3",hits.get(7).getId().stringValue()); + assertEquals("source2-3",hits.get(8).getId().stringValue()); + + // Check rendering + assertRendered(result,"AnySourceResult.xml"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml new file mode 100644 index 00000000000..e0306872149 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + #ff00ff + true + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml new file mode 100644 index 00000000000..47ed1bd2f12 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml @@ -0,0 +1,36 @@ + + + + + + + + #ff00ff + true + + + + source1-1 + + + + source2-1 + + + + source1-2 + + + + source2-2 + + + + source1-3 + + + + source2-3 + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java new file mode 100644 index 00000000000..58d05971805 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.Renderer; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Jon Bratseth + */ +public class ChoiceOfRenderersTestCase extends ExecutionAbstractTestCase { + + //This test is order dependent. Fix this!! + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("ChoiceOfRenderers.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + assertEquals(1,resolution.getResolution((Choice)((PageTemplate)page.get(0).get(0)).getSection().elements(Renderer.class).get(0))); + assertEquals(2,resolution.getResolution((Choice)((PageTemplate)page.get(0).get(0)).getSection().elements(Renderer.class).get(1))); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + assertEquals(6,result.getConcreteHitCount()); + assertEquals(6,result.getHitCount()); + + // Check rendering + assertRendered(result,"ChoiceOfRenderersResult.xml"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml new file mode 100644 index 00000000000..f7323ba094d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml @@ -0,0 +1,20 @@ + + + + +
+ + +
+ + +
+
+ + + + +
+ + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml new file mode 100644 index 00000000000..b1d29995312 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml @@ -0,0 +1,29 @@ + + + + +
+ + source2-1 + + + source2-2 + + + source2-3 + +
+ +
+ + source4-1 + + + source4-2 + + + source4-3 + +
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java new file mode 100644 index 00000000000..3d92a721f0d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Jon Bratseth + */ +public class ChoiceOfSubsectionsTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("ChoiceOfSubsections.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + result.hits().add(createHits("source4",3)); + + new Organizer().organize(page,new DeterministicResolverAssertingMethod().resolve(page,query,result),result); + + // Check execution: + // Two subsections with one source each + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertEqualHitGroups(createHits("source2",3),section1); + assertEqualHitGroups(createHits("source4",3),section2); + + // Check rendering + assertRendered(result,"ChoiceOfSubsectionsResult.xml"); + } + + /** Same as deterministic resolver, but asserts that it received the correct method names for each choice */ + private static class DeterministicResolverAssertingMethod extends DeterministicResolver { + + private int invocationNumber=0; + + /** Chooses the last alternative of any choice */ + @Override + public void resolve(Choice choice, Query query, Result result, Resolution resolution) { + invocationNumber++; + if (invocationNumber==2) + assertEquals("method1",choice.getMethod()); + else if (invocationNumber==3) + assertEquals("method2",choice.getMethod()); + else if (invocationNumber>3) + throw new IllegalStateException("Unexpected number of resolver invocations"); + + super.resolve(choice,query,result,resolution); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml new file mode 100644 index 00000000000..c6a0af9ddd2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml new file mode 100644 index 00000000000..35d913fd9fa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml @@ -0,0 +1,17 @@ + + + + + + source2-1 + + + + source2-2 + + + + source2-3 + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java new file mode 100644 index 00000000000..facffb50649 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.Source; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Jon Bratseth + */ +public class ChoiceOfTwoSourcesTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("ChoiceOfTwoSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + assertEquals(1,resolution.getResolution((Choice)((PageTemplate)page.get(0).get(0)).getSection().elements(Source.class).get(0))); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // No subsections: Last choice (source2) used + assertEqualHitGroups(createHits("source2",3),result.hits()); + + // Check rendering + assertRendered(result,"ChoiceOfTwoSourcesResult.xml"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml new file mode 100644 index 00000000000..e8d1736f46c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml @@ -0,0 +1,45 @@ + + + + + +
+
+ + + + +
+
+
+
+ + + +
+ + + +
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml new file mode 100644 index 00000000000..ab995365f22 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml @@ -0,0 +1,55 @@ + + + + +
+
+ + news-1 + + + news-2 + + + news-3 + +
+
+ + web-1 + + + web-2 + + + web-3 + +
+
+ +
+
+ + blog-1 + + + blog-2 + + + blog-3 + +
+
+ + images-1 + + + images-2 + + + images-3 + +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java new file mode 100644 index 00000000000..a646823c8cb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.PageElement; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Jon Bratseth + */ +public class ChoicesTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template (second alternative will be chosen) + List pages=new ArrayList<>(); + pages.add(importPage("AnySource.xml")); + pages.add(importPage("Choices.xml")); + Choice page=Choice.createSingletons(pages); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",3)); + result.hits().add(createHits("web",3)); + result.hits().add(createHits("blog",3)); + result.hits().add(createHits("images",3)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"ChoicesResult.xml"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java new file mode 100644 index 00000000000..544366758f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.io.IOUtils; +import com.yahoo.prelude.templates.TiledTemplateSet; +import com.yahoo.prelude.templates.UserTemplate; +import com.yahoo.prelude.templates.test.TilingTestCase; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.io.*; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class ExecutionAbstractTestCase { + + private static final String root="src/test/java/com/yahoo/search/pagetemplates/engine/test/"; + + protected PageTemplate importPage(String name) { + PageTemplate template=new PageTemplateXMLReader().readFile(root + name); + assertNotNull("Could look up page template '" + name + "'",template); + return template; + } + + protected void assertEqualHitGroups(HitGroup expected,HitGroup actual) { + assertEquals(expected.size(),actual.size()); + int i=0; + for (Hit expectedHit : expected.asList()) { + Hit actualHit=actual.get(i++); + assertEquals(expectedHit.getId(),actualHit.getId()); + assertEquals(expectedHit.getSource(),actualHit.getSource()); + } + } + + protected HitGroup createHits(String sourceName,int hitCount) { + HitGroup source=new HitGroup("source:" + sourceName); + for (int i=1; i<=hitCount; i++) { + Hit hit=new Hit(sourceName + "-" + i,1/(double)i); + hit.setSource(sourceName); + source.add(hit); + } + return source; + } + + protected void assertRendered(Result result,String resultFileName) { + assertRendered(result,resultFileName,false); + } + + protected void assertRendered(Result result,String resultFileName, UserTemplate template) { + assertRendered(result,resultFileName,template,false); + } + + protected void assertRendered(Result result,String resultFileName,boolean print) { + assertRendered(result,resultFileName,new TiledTemplateSet(),print); + } + + @SuppressWarnings("deprecation") + protected void assertRendered(Result result,String resultFileName,UserTemplate template, boolean print) { + result.getTemplating().setTemplates(template); + try { + TilingTestCase.assertRendered(IOUtils.readFile(new File(root + resultFileName)), result); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml new file mode 100644 index 00000000000..2bc75fba5f4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml @@ -0,0 +1,28 @@ + + + +
+ + +
+
+ + +
+ + + +
+
+ +
+
+ +
+ + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml new file mode 100644 index 00000000000..3a163e5f804 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml @@ -0,0 +1,96 @@ + + + + +
+
+ + source1-1 + + + source1-2 + + + source1-3 + +
+
+ + source2-1 + + + source2-2 + + + source2-3 + + + source2-4 + +
+
+ +
+
+ + source3-1 + + + source3-2 + + + source3-3 + + + source3-4 + + + source3-5 + +
+
+ + source5-1 + + + source5-2 + + + source5-3 + + + source5-4 + + + source5-5 + + + source5-6 + + + source5-7 + +
+
+ + source4-1 + + + source4-2 + + + source4-3 + + + source4-4 + + + source4-5 + + + source4-6 + +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java new file mode 100644 index 00000000000..54fc342aa22 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.MapChoice; +import com.yahoo.search.pagetemplates.model.PageElement; +import com.yahoo.search.pagetemplates.model.Section; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author Jon Bratseth + */ +public class MapSectionsToSectionsTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("MapSectionsToSections.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",4)); + result.hits().add(createHits("source3",5)); + result.hits().add(createHits("source4",6)); + result.hits().add(createHits("source5",7)); + + // Resolve + Resolver resolver=new DeterministicResolverAssertingMethod(); + Resolution resolution=resolver.resolve(page,query,result); + Map> mapping= + resolution.getResolution((MapChoice)((PageTemplate)page.get(0).get(0)).getSection().elements().get(2)); + assertNotNull(mapping); + assertEquals("box1",((Section)mapping.get("box1holder").get(0)).getId()); + assertEquals("box2",((Section)mapping.get("box2holder").get(0)).getId()); + assertEquals("box3",((Section)mapping.get("box3holder").get(0)).getId()); + assertEquals("box4",((Section)mapping.get("box4holder").get(0)).getId()); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // Two subsections, each containing two sub-subsections with one source each + assertEquals(2,result.hits().size()); + HitGroup row1=(HitGroup)result.hits().get(0); + HitGroup column11=(HitGroup)row1.get(0); + HitGroup column12=(HitGroup)row1.get(1); + HitGroup row2=(HitGroup)result.hits().get(1); + HitGroup column21a=(HitGroup)row2.get(0); + HitGroup column21b=(HitGroup)row2.get(1); + HitGroup column22=(HitGroup)row2.get(2); + assertEqualHitGroups(createHits("source1",3),column11); + assertEqualHitGroups(createHits("source2",4),column12); + assertEqualHitGroups(createHits("source3",5),column21a); + assertEqualHitGroups(createHits("source5",7),column21b); + assertEqualHitGroups(createHits("source4",6),column22); + + // Check rendering + assertRendered(result,"MapSectionsToSectionsResult.xml"); + } + + /** Same as deterministic resolver, but asserts that it received the correct method names for each map choice */ + private static class DeterministicResolverAssertingMethod extends DeterministicResolver { + + /** Chooses the last alternative of any choice */ + @Override + public void resolve(MapChoice mapChoice, Query query, Result result, Resolution resolution) { + assertEquals("myMethod",mapChoice.getMethod()); + super.resolve(mapChoice,query,result,resolution); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml new file mode 100644 index 00000000000..7b5c0770096 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml @@ -0,0 +1,22 @@ + + + +
+
+
+
+
+
+
+
+ + + + + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml new file mode 100644 index 00000000000..034330c071c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml @@ -0,0 +1,73 @@ + + + + +
+
+ + source1-1 + + + source1-2 + + + source1-3 + +
+
+ + source2-1 + + + source2-2 + + + source2-3 + + + source2-4 + +
+
+ +
+
+ + source3-1 + + + source3-2 + + + source3-3 + + + source3-4 + + + source3-5 + +
+
+ + source4-1 + + + source4-2 + + + source4-3 + + + source4-4 + + + source4-5 + + + source4-6 + +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java new file mode 100644 index 00000000000..49cc0411ac5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.MapChoice; +import com.yahoo.search.pagetemplates.model.PageElement; +import com.yahoo.search.pagetemplates.model.Source; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author Jon Bratseth + */ +public class MapSourcesToSectionsTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("MapSourcesToSections.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",4)); + result.hits().add(createHits("source3",5)); + result.hits().add(createHits("source4",6)); + result.hits().add(createHits("source5",7)); + + // Resolve + Resolver resolver=new DeterministicResolverAssertingMethod(); + Resolution resolution=resolver.resolve(page,query,result); + Map> mapping= + resolution.getResolution((MapChoice)((PageTemplate)page.get(0).get(0)).getSection().elements().get(2)); + assertNotNull(mapping); + assertEquals("source1",((Source)mapping.get("box1source").get(0)).getName()); + assertEquals("source2",((Source)mapping.get("box2source").get(0)).getName()); + assertEquals("source3",((Source)mapping.get("box3source").get(0)).getName()); + assertEquals("source4",((Source)mapping.get("box4source").get(0)).getName()); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // Two subsections, each containing two sub-subsections with one source each + assertEquals(2,result.hits().size()); + HitGroup row1=(HitGroup)result.hits().get(0); + HitGroup column11=(HitGroup)row1.get(0); + HitGroup column12=(HitGroup)row1.get(1); + HitGroup row2=(HitGroup)result.hits().get(1); + HitGroup column21=(HitGroup)row2.get(0); + HitGroup column22=(HitGroup)row2.get(1); + assertEqualHitGroups(createHits("source1",3),column11); + assertEqualHitGroups(createHits("source2",4),column12); + assertEqualHitGroups(createHits("source3",5),column21); + assertEqualHitGroups(createHits("source4",6),column22); + + // Check rendering + assertRendered(result,"MapSourcesToSectionsResult.xml"); + } + + /** Same as deterministic resolver, but asserts that it received the correct method names for each map choice */ + private static class DeterministicResolverAssertingMethod extends DeterministicResolver { + + /** Chooses the last alternative of any choice */ + @Override + public void resolve(MapChoice mapChoice, Query query, Result result, Resolution resolution) { + assertEquals("myMethod",mapChoice.getMethod()); + super.resolve(mapChoice,query,result,resolution); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml new file mode 100644 index 00000000000..967da527b6e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml @@ -0,0 +1,31 @@ + + + + + +
+ + + blue + +
+ +
+ + 3 + + +
+ +
+ +
+ + +
+ +
+ +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml new file mode 100644 index 00000000000..95b86ef1f4d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml @@ -0,0 +1,43 @@ + + + + + + +
+ + + blue + +
+ +
+ + 3 + +
+ + + + news-1 + + + news-2 + + +
+
+ + +
+
+ + + + htmlSource-1 + + +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java new file mode 100644 index 00000000000..33930f3d0e0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * Tests an example page. + * + * @author bratseth + */ +public class PageTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("Page.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",2)); + result.hits().add(createHits("htmlSource",1)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"PageResult.xml",new PageTemplateSet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml new file mode 100644 index 00000000000..dca31eb42e1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml @@ -0,0 +1,37 @@ + + + + + +
+ + + blue + +
+ +
+ + 3 + + +
+ + + + + + + +
+ +
+ + +
+ +
+ +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml new file mode 100644 index 00000000000..7ac78f3e820 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml @@ -0,0 +1,45 @@ + + + + + + +
+ + + blue + +
+ +
+ + 3 + +
+ + + + + + news-1 + + + news-2 + + +
+
+ + +
+
+ + + + htmlSource-1 + + +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java new file mode 100644 index 00000000000..445105cfd2f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * Tests an exapnded example. + * + * @author bratseth + */ +public class PageWithBlendingTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("PageWithBlending.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",2)); + result.hits().add(createHits("htmlSource",1)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"PageWithBlendingResult.xml",new PageTemplateSet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml new file mode 100644 index 00000000000..5d3e38c2beb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml @@ -0,0 +1,36 @@ + + + + + +
+ + + + + + blue + +
+ +
+ + 3 + + +
+ + + +
+ +
+ + +
+ +
+ +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml new file mode 100644 index 00000000000..00656399331 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml @@ -0,0 +1,43 @@ + + + + + + +
+ + + blue + +
+ +
+ + 3 + +
+ + + + news-1 + + + news-2 + + +
+
+ + +
+
+ + + + htmlSource-1 + + +
+
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java new file mode 100644 index 00000000000..e7dbae21d47 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * Tests an example with two data sources with a renderer each. + * + * @author bratseth + */ +public class PageWithSourceRendererTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("PageWithSourceRenderer.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",2)); + result.hits().add(createHits("htmlSource",1)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"PageWithSourceRendererResult.xml",new PageTemplateSet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml new file mode 100644 index 00000000000..ff39ef1d3d1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml new file mode 100644 index 00000000000..c9e0909a476 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml @@ -0,0 +1,17 @@ + + + + + + + web-1 + + + web-2 + + + web-3 + + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java new file mode 100644 index 00000000000..04e550a631c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * @author Jon Bratseth + */ +public class SourceChoiceTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("SourceChoice.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("web",3)); + result.hits().add(createHits("news",3)); + result.hits().add(createHits("blog",3)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"SourceChoiceResult.xml",new PageTemplateSet(),true); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml new file mode 100644 index 00000000000..36cace66ff7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml @@ -0,0 +1,5 @@ + + +
+
+ diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml new file mode 100644 index 00000000000..0ca5bfc7ea0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml @@ -0,0 +1,65 @@ + + + + +
+ + source3-1 + + + source3-2 + + + source3-3 + + + source3-4 + + + source3-5 + + + source1-1 + + + source1-2 + + + source1-3 + +
+ +
+ + source4-1 + + + source2-1 + + + source4-2 + + + source2-2 + + + source4-3 + + + source2-3 + + + source4-4 + + + source2-4 + + + source4-5 + + + source4-6 + +
+ + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java new file mode 100644 index 00000000000..3fff2103332 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author Jon Bratseth + */ +public class TwoSectionsFourSourcesTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",4)); + result.hits().add(createHits("source3",12)); + result.hits().add(createHits("source4",13)); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertGroupedSource3Source1(section1.asList()); + assertBlendedSource4Source2(section2.asList()); + + // Check rendering + assertRendered(result,"TwoSectionsFourSourcesResult.xml"); + } + + @Test + public void testExecutionMissingOneSource() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source3",12)); + result.hits().add(createHits("source4",13)); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertGroupedSource3Source1(section1.asList()); + assertEqualHitGroups(createHits("source4",10),section2); + } + + @Test + public void testExecutionMissingTwoSources() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source3",12)); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertGroupedSource3Source1(section1.asList()); + assertEquals(0,section2.size()); + } + + @Test + public void testExecutionMissingAllSources() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertEquals(0,section1.size()); + assertEquals(0,section2.size()); + } + + private void assertGroupedSource3Source1(List hits) { + assertEquals(8,hits.size()); + assertEquals("source3-1",hits.get(0).getId().stringValue()); + assertEquals("source3-2",hits.get(1).getId().stringValue()); + assertEquals("source3-3",hits.get(2).getId().stringValue()); + assertEquals("source3-4",hits.get(3).getId().stringValue()); + assertEquals("source3-5",hits.get(4).getId().stringValue()); + assertEquals("source1-1",hits.get(5).getId().stringValue()); + assertEquals("source1-2",hits.get(6).getId().stringValue()); + assertEquals("source1-3",hits.get(7).getId().stringValue()); + } + + private void assertBlendedSource4Source2(List hits) { + assertEquals(10,hits.size()); + assertEquals("source4-1",hits.get(0).getId().stringValue()); + assertEquals("source2-1",hits.get(1).getId().stringValue()); + assertEquals("source4-2",hits.get(2).getId().stringValue()); + assertEquals("source2-2",hits.get(3).getId().stringValue()); + assertEquals("source4-3",hits.get(4).getId().stringValue()); + assertEquals("source2-3",hits.get(5).getId().stringValue()); + assertEquals("source4-4",hits.get(6).getId().stringValue()); + assertEquals("source2-4",hits.get(7).getId().stringValue()); + assertEquals("source4-5",hits.get(8).getId().stringValue()); + assertEquals("source4-6",hits.get(9).getId().stringValue()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java new file mode 100644 index 00000000000..22c269e761f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.intent.model.*; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.PageTemplateSearcher; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.PageElement; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.interpretation.Interpretation; + +import java.util.*; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class PageTemplateSearcherTestCase extends junit.framework.TestCase { + + public void testSearcher() { + PageTemplateSearcher s = new PageTemplateSearcher(createPageTemplateRegistry(), new FirstChoiceResolver()); + Chain chain = new Chain<>(s,new MockFederator()); + + { + // No template specified, should use default + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=foo&page.resolver=native.deterministic")); + assertSources("source1 source2",result); + } + + { + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=foo&page.id=oneSource&page.resolver=native.deterministic")); + assertSources("source1",result); + } + + { + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=foo&page.id=twoSources&model.sources=source1&page.resolver=native.deterministic")); + assertSources("source1",result); + } + + { + Query query=new Query("?query=foo&page.resolver=native.deterministic"); + addIntentModelSpecifyingSource3(query); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2",result); + } + + { + Query query=new Query("?query=foo&page.id=twoSourcesAndAny&page.resolver=native.deterministic"); + addIntentModelSpecifyingSource3(query); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2 source3",result); + } + + { + Query query=new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); + addIntentModelSpecifyingSource3(query); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source3",result); + } + + { + Query query=new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertTrue(query.getModel().getSources().isEmpty()); + assertNotNull(result.hits().get("source1")); + assertNotNull(result.hits().get("source2")); + assertNotNull(result.hits().get("source3")); + } + + { + Query query=new Query("?query=foo&page.id=choiceOfSources&page.resolver=native.deterministic"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2","source2",result); + } + + { + Query query=new Query("?query=foo&page.id=choiceOfSources&page.resolver=test.firstChoice"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2","source1",result); + } + + { // Specifying two templates, should pick the last + Query query=new Query("?query=foo&page.id=threeSources+oneSource&page.resolver=native.deterministic"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2 source3","source1",result); + } + + { // Specifying two templates as a list, should override the page.id setting + Query query=new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); + query.properties().set("page.idList",Arrays.asList("oneSource","threeSources")); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2 source3","source1 source2 source3",result); + } + + { + try { + Query query=new Query("?query=foo&page.id=oneSource+choiceOfSources&page.resolver=noneSuch"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("No page template resolver 'noneSuch'",e.getMessage()); + } + } + + } + + private PageTemplateRegistry createPageTemplateRegistry() { + PageTemplateRegistry registry=new PageTemplateRegistry(); + + PageTemplate twoSources=new PageTemplate(new ComponentId("default")); + twoSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + twoSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source2")); + registry.register(twoSources); + + PageTemplate oneSource=new PageTemplate(new ComponentId("oneSource")); + oneSource.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + registry.register(oneSource); + + PageTemplate threeSources=new PageTemplate(new ComponentId("threeSources")); + threeSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + threeSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source2")); + threeSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source3")); + registry.register(threeSources); + + PageTemplate twoSourcesAndAny=new PageTemplate(new ComponentId("twoSourcesAndAny")); + twoSourcesAndAny.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + twoSourcesAndAny.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source2")); + twoSourcesAndAny.getSection().elements().add(com.yahoo.search.pagetemplates.model.Source.any); + registry.register(twoSourcesAndAny); + + PageTemplate anySource=new PageTemplate(new ComponentId("anySource")); + anySource.getSection().elements().add(com.yahoo.search.pagetemplates.model.Source.any); + registry.register(anySource); + + PageTemplate choiceOfSources=new PageTemplate(new ComponentId("choiceOfSources")); + List alternatives=new ArrayList<>(); + alternatives.add(new com.yahoo.search.pagetemplates.model.Source("source1")); + alternatives.add(new com.yahoo.search.pagetemplates.model.Source("source2")); + choiceOfSources.getSection().elements().add(Choice.createSingletons(alternatives)); + registry.register(choiceOfSources); + + registry.freeze(); + return registry; + } + + private void addIntentModelSpecifyingSource3(Query query) { + IntentModel intentModel=new IntentModel(); + InterpretationNode interpretation=new InterpretationNode(new Interpretation("ignored")); + IntentNode intent=new IntentNode(new Intent("ignored"),1.0); + intent.children().add(new SourceNode(new com.yahoo.search.intent.model.Source("source3"),1.0)); + interpretation.children().add(intent); + intentModel.children().add(interpretation); + intentModel.setTo(query); + } + + private void assertSources(String expectedSourceString,Result result) { + assertSources(expectedSourceString,expectedSourceString,result); + } + + private void assertSources(String expectedQuerySourceString,String expectedResultSourceString,Result result) { + Set expectedQuerySources=new HashSet<>(Arrays.asList(expectedQuerySourceString.split(" "))); + assertEquals(expectedQuerySources,result.getQuery().getModel().getSources()); + + Set expectedResultSources=new HashSet<>(Arrays.asList(expectedResultSourceString.split(" "))); + for (String sourceName : Arrays.asList("source1 source2 source3".split(" "))) { + if (expectedResultSources.contains(sourceName)) + assertNotNull("Result contains '" + sourceName + "'",result.hits().get(sourceName)); + else + assertNull("Result does not contain '" + sourceName + "'",result.hits().get(sourceName)); + } + } + + private static class MockFederator extends Searcher { + + @Override + public Result search(Query query,Execution execution) { + Result result=new Result(query); + for (String sourceName : Arrays.asList("source1 source2 source3".split(" "))) + if (query.getModel().getSources().isEmpty() || query.getModel().getSources().contains(sourceName)) + result.hits().add(createSource(sourceName)); + return result; + } + + private HitGroup createSource(String sourceName) { + HitGroup hitGroup=new HitGroup("source:" + sourceName); + Hit hit=new Hit(sourceName); + hit.setSource(sourceName); + hitGroup.add(hit); + return hitGroup; + } + + } + + /** Like the deterministic resolver except that it takes the first option of all choices */ + private static class FirstChoiceResolver extends DeterministicResolver { + + public FirstChoiceResolver() { + super("test.firstChoice"); + } + + /** Chooses the first alternative of any choice */ + @Override + public void resolve(Choice choice, Query query, Result result, Resolution resolution) { + resolution.addChoiceResolution(choice,0); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml new file mode 100644 index 00000000000..2a98ef6918f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml @@ -0,0 +1,16 @@ + + + + source1p1Value + source1p2Value + + + + source2p1Value + source2p3Value + + + source3p1Value + + + diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java new file mode 100644 index 00000000000..1f79637119a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.test; + +import com.yahoo.search.Query; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.PageTemplateSearcher; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class SourceParametersTestCase extends junit.framework.TestCase { + + private static final String root="src/test/java/com/yahoo/search/pagetemplates/test/"; + + public void testSourceParametersWithSourcesDeterminedByTemplate() { + // Create the page template + PageTemplateRegistry pageTemplateRegistry=new PageTemplateRegistry(); + PageTemplate page=importPage("SourceParameters.xml"); + pageTemplateRegistry.register(page); + PageTemplateSearcher s=new PageTemplateSearcher(pageTemplateRegistry); + Query query=new Query("?query=foo&page.id=SourceParameters&page.resolver=native.deterministic"); + new Execution(s, Execution.Context.createContextStub()).search(query); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source2p1Value",query.properties().get("source.source2.p1")); + assertEquals("source2p3Value",query.properties().get("source.source2.p3")); + assertEquals("source3p1Value",query.properties().get("source.source3.p1")); + assertEquals("We get the correct number of parameters",5,query.properties().listProperties("source").size()); + } + + public void testSourceParametersWithSourcesDeterminedByParameter() { + // Create the page template + PageTemplateRegistry pageTemplateRegistry=new PageTemplateRegistry(); + PageTemplate page=importPage("SourceParameters.xml"); + pageTemplateRegistry.register(page); + PageTemplateSearcher s=new PageTemplateSearcher(pageTemplateRegistry); + Query query=new Query("?query=foo&page.id=SourceParameters&model.sources=source1,source3&page.resolver=native.deterministic"); + new Execution(s, Execution.Context.createContextStub()).search(query); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source3p1Value",query.properties().get("source.source3.p1")); + assertEquals("We get the correct number of parameters",3,query.properties().listProperties("source").size()); + } + + protected PageTemplate importPage(String name) { + PageTemplate template=new PageTemplateXMLReader().readFile(root + name); + assertNotNull("Could look up read template '" + name + "'",template); + return template; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java new file mode 100644 index 00000000000..e7a0c78aa88 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.Collator; +import com.ibm.icu.text.RuleBasedCollator; +import com.ibm.icu.util.ULocale; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +/** + * @author balder + */ +public class SortingTestCase { + @Test + public void validAttributeName() { + assertNotNull(Sorting.fromString("a")); + assertNotNull(Sorting.fromString("_a")); + assertNotNull(Sorting.fromString("+a")); + assertNotNull(Sorting.fromString("-a")); + assertNotNull(Sorting.fromString("a.b")); + try { + assertNotNull(Sorting.fromString("-1")); + fail("'-1' should not be allowed as attribute name."); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal attribute name '1' for sorting. Requires '[\\[]*[a-zA-Z_][\\.a-zA-Z0-9_-]*[\\]]*'"); + } catch (Exception e) { + fail("I only expect 'IllegalArgumentException', not: + " + e.toString()); + } + } + @Test + public void requireThatChineseSortCorrect() { + requireThatChineseHasCorrectRules(Collator.getInstance(new ULocale("zh"))); + Sorting ch = Sorting.fromString("uca(a,zh)"); + assertEquals(1, ch.fieldOrders().size()); + Sorting.FieldOrder fo = ch.fieldOrders().get(0); + assertTrue(fo.getSorter() instanceof Sorting.UcaSorter); + Sorting.UcaSorter uca = (Sorting.UcaSorter) fo.getSorter(); + requireThatChineseHasCorrectRules(uca.getCollator()); + Sorting.AttributeSorter sorter = fo.getSorter(); + assertTrue(sorter.compare("a", "b") < 0); + assertTrue(sorter.compare("a", "a\u81EA") < 0); + assertTrue(sorter.compare("\u81EA", "a") < 0); + } + + private void requireThatArabicHasCorrectRules(Collator col) { + final int reorderCodes [] = {UScript.ARABIC}; + assertEquals("6.2.0.0", col.getUCAVersion().toString()); + assertEquals("58.0.0.6", col.getVersion().toString()); + assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); + assertTrue(col.compare("a", "b") < 0); + assertTrue(col.compare("a", "aس") < 0); + assertFalse(col.compare("س", "a") < 0); + + assertEquals(" [reorder Arab]&ت<<Ø©<<<ﺔ<<<ﺓ&ÙŠ<<Ù‰<<<ﯨ<<<ﯩ<<<ï»°<<<ﻯ<<<ï²<<<ï±", ((RuleBasedCollator) col).getRules()); + assertFalse(col.compare("س", "a") < 0); + } + + private void requireThatChineseHasCorrectRules(Collator col) { + final int reorderCodes [] = {UScript.HAN}; + assertEquals("8.0.0.0", col.getUCAVersion().toString()); + assertEquals("153.64.29.0", col.getVersion().toString()); + assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); + + assertNotEquals("", ((RuleBasedCollator) col).getRules()); + } + @Test + @Ignore + public void requireThatArabicSortCorrect() { + requireThatArabicHasCorrectRules(Collator.getInstance(new ULocale("ar"))); + Sorting ar = Sorting.fromString("uca(a,ar)"); + assertEquals(1, ar.fieldOrders().size()); + Sorting.FieldOrder fo = ar.fieldOrders().get(0); + assertTrue(fo.getSorter() instanceof Sorting.UcaSorter); + Sorting.UcaSorter uca = (Sorting.UcaSorter) fo.getSorter(); + requireThatArabicHasCorrectRules(uca.getCollator()); + Sorting.AttributeSorter sorter = fo.getSorter(); + assertTrue(sorter.compare("a", "b") < 0); + assertTrue(sorter.compare("a", "aس") < 0); + assertTrue(sorter.compare("س", "a") < 0); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java new file mode 100644 index 00000000000..98ed684af17 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.component.chain.Chain; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; + +/** + * Checks it's OK adding more traces to an instance which is being rendered. + * + * @author Arne Bergene Fossaa + */ +@SuppressWarnings("deprecation") +public class ConcurrentTraceTestCase { + class TraceSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + for(int i = 0;i<1000;i++) { + query.trace("Trace", false, 1); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + } + return execution.search(query); + } + } + + class AsyncSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Chain chain = new Chain<>(new TraceSearcher()); + + Result result = new Result(query); + List futures = new ArrayList<>(); + for(int i = 0; i < 100; i++) { + futures.add(new AsyncExecution(chain, execution).searchAndFill(query)); + } + AsyncExecution.waitForAll(futures, 10); + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java new file mode 100644 index 00000000000..bbddae0f7f0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import com.yahoo.processing.execution.Execution; +import com.yahoo.search.Query; +import com.yahoo.search.query.context.QueryContext; + +/** + * @author Jon Bratseth + */ +public class LoggingTestCase extends junit.framework.TestCase { + + public void testLogging() { + Query query=new Query(); + QueryContext queryContext = query.getContext(true); + queryContext.logValue("a","a1"); + queryContext.trace("first message", 2); + queryContext.logValue("a","a2"); + queryContext.logValue("b","b1"); + QueryContext h2 = query.clone().getContext(true); + h2.logValue("b","b2"); + h2.trace("second message", 2); + h2.logValue("b","b3"); + queryContext.logValue("b","b4"); + QueryContext h3 = query.clone().getContext(true); + h3.logValue("b","b5"); + h3.logValue("c","c1"); + h3.trace("third message", 2); + h2.logValue("c","c2"); + queryContext.trace("fourth message", 2); + queryContext.logValue("d","d1"); + h2.trace("fifth message", 2); + h2.logValue("c","c3"); + queryContext.logValue("c","c4"); + + // Assert that all of the above is in the log, in some undefined order + Set logValues=new HashSet<>(); + for (Iterator logValueIterator=queryContext.logValueIterator(); logValueIterator.hasNext(); ) + logValues.add(logValueIterator.next().toString()); + assertEquals(12,logValues.size()); + assertTrue(logValues.contains("a=a1")); + assertTrue(logValues.contains("a=a2")); + assertTrue(logValues.contains("b=b1")); + assertTrue(logValues.contains("b=b2")); + assertTrue(logValues.contains("b=b3")); + assertTrue(logValues.contains("b=b4")); + assertTrue(logValues.contains("b=b5")); + assertTrue(logValues.contains("c=c1")); + assertTrue(logValues.contains("c=c2")); + assertTrue(logValues.contains("d=d1")); + assertTrue(logValues.contains("c=c3")); + assertTrue(logValues.contains("c=c4")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java new file mode 100644 index 00000000000..e9bdb6f60f5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.context.QueryContext; + +/** + * @author Jon Bratseth + */ +public class PropertiesTestCase extends junit.framework.TestCase { + + public void testProperties() { + Query query=new Query(); + QueryContext h = query.getContext(true); + h.setProperty("a","a1"); + h.trace("first message", 2); + h.setProperty("a","a2"); + h.setProperty("b","b1"); + query.clone(); + QueryContext h2 = query.clone().getContext(true); + h2.setProperty("b","b2"); + h2.trace("second message", 2); + h2.setProperty("b","b3"); + h.setProperty("b","b4"); + QueryContext h3 = query.clone().getContext(true); + h3.setProperty("b","b5"); + h3.setProperty("c","c1"); + h3.trace("third message", 2); + h2.setProperty("c","c2"); + h.trace("fourth message", 2); + h.setProperty("d","d1"); + h2.trace("fifth message", 2); + h2.setProperty("c","c3"); + h.setProperty("c","c4"); + + assertEquals("a2",h.getProperty("a")); + assertEquals("b5",h.getProperty("b")); + assertEquals("c4",h.getProperty("c")); + assertEquals("d1",h.getProperty("d")); + assertNull(h.getProperty("e")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java new file mode 100644 index 00000000000..7cc3d939b01 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.context.QueryContext; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Iterator; + +/** + * @author Steinar Knutsen + */ +public class TraceTestCase extends junit.framework.TestCase { + + public void testBasicTracing() { + Query query=new Query(); + QueryContext h = query.getContext(true); + h.trace("first message", 0); + h.trace("second message", 0); + assertEquals("trace: [ [ first message second message ] ]", h.toString()); + } + + public void testCloning() throws IOException { + Query query=new Query(); + QueryContext h = query.getContext(true); + h.trace("first message", 0); + QueryContext h2 = query.clone().getContext(true); + h2.trace("second message", 0); + QueryContext h3 = query.clone().getContext(true); + h3.trace("third message", 0); + h.trace("fourth message", 0); + h2.trace("fifth message", 0); + Writer w = new StringWriter(); + Writer w2 = new StringWriter(); + h2.render(w2); + String finishedBelow = w2.toString(); + h.render(w); + String toplevel = w.toString(); + // check no matter which QueryContext ends up in the final Result, + // all context info is rendered + assertEquals(toplevel, finishedBelow); + // basic sanity test + assertEquals("trace: [ [ " + + "first message second message third message " + + "fourth message fifth message ] ]",h.toString()); + Iterator i = h.getTrace().traceNode().root().descendants(String.class).iterator(); + assertEquals("first message",i.next()); + assertEquals("second message",i.next()); + assertEquals("third message",i.next()); + assertEquals("fourth message",i.next()); + assertEquals("fifth message",i.next()); + } + + public void testEmpty() throws IOException { + Query query=new Query(); + QueryContext h = query.getContext(true); + Writer w = new StringWriter(); + h.render(w); + assertEquals("", w.toString()); + } + + public void testEmptySubSequence() { + Query query=new Query(); + QueryContext h = query.getContext(true); + query.clone().getContext(true); + Writer w = new StringWriter(); + try { + h.render(w); + } catch (IOException e) { + assertTrue("rendering empty subsequence crashed", false); + } + } + + public void testAttachedTraces() throws IOException { + String needle0 = "nalle"; + String needle1 = "tralle"; + String needle2 = "trolle"; + String needle3 = "bamse"; + Query q = new Query("/?tracelevel=1"); + q.trace(needle0, false, 1); + Query q2 = new Query(); + q.attachContext(q2); + q2.trace(needle1, false, 1); + q2.trace(needle2, false, 1); + q.trace(needle3, false, 1); + Writer w = new StringWriter(); + q.getContext(false).render(w); + String trace = w.toString(); + int nalle = trace.indexOf(needle0); + int tralle = trace.indexOf(needle1); + int trolle = trace.indexOf(needle2); + int bamse = trace.indexOf(needle3); + assertTrue("Could not find first manual context to main query.", nalle > 0); + assertTrue("Could not find second manual context to main query.", bamse > 0); + assertTrue("Could not find first manual context to attached query.", tralle > 0); + assertTrue("Could not find second manual context to attached query.", trolle > 0); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java new file mode 100644 index 00000000000..f406552cd30 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; + +/** + * @author bratseth + */ +public class MultiProfileTestCase extends junit.framework.TestCase { + + public void testValid() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"); + + QueryProfile multiprofile1=registry.getComponent("multiprofile1"); + assertNotNull(multiprofile1); + assertGet("general-a","a",new String[] {null,null,null},multiprofile1); + assertGet("us-nokia-test1-a","a",new String[] {"us","nok ia","test1"},multiprofile1); + assertGet("us-nokia-b","b",new String[] {"us","nok ia","test1"},multiprofile1); + assertGet("us-a","a",new String[] {"us",null,null},multiprofile1); + assertGet("us-b","b",new String[] {"us",null,null},multiprofile1); + assertGet("us-nokia-a","a",new String[] {"us","nok ia",null},multiprofile1); + assertGet("us-test1-a","a",new String[] {"us",null,"test1"},multiprofile1); + assertGet("us-test1-b","b",new String[] {"us",null,"test1"},multiprofile1); + + assertGet("us-a","a",new String[] {"us","unspecified","unspecified"},multiprofile1); + assertGet("us-nokia-a","a",new String[] {"us","nok ia","unspecified"},multiprofile1); + assertGet("us-test1-a","a",new String[] {"us","unspecified","test1"},multiprofile1); + assertGet("us-nokia-b","b",new String[] {"us","nok ia","test1"},multiprofile1); + + // ...inherited + assertGet("parent1-value","parent1",new String[] { "us","nok ia","-" }, multiprofile1); + assertGet("parent2-value","parent2",new String[] { "us","nok ia","-" }, multiprofile1); + assertGet(null,"parent1",new String[] { "us","-","-" }, multiprofile1); + assertGet(null,"parent2",new String[] { "us","-","-" }, multiprofile1); + } + + private void assertGet(String expectedValue,String parameter,String[] dimensionValues,QueryProfile profile) { + Map context=new HashMap<>(); + context.put("region",dimensionValues[0]); + context.put("model",dimensionValues[1]); + context.put("bucket",dimensionValues[2]); + assertEquals("Looking up '" + parameter + "' for '" + toString(dimensionValues) + "'",expectedValue,profile.get(parameter,context,null)); + } + + private String toString(String[] array) { + StringBuilder b=new StringBuilder("["); + for (String value : array) { + b.append(value); + b.append(","); + } + b.deleteCharAt(b.length()-1); // Remove last comma :-) + b.append("]"); + return b.toString(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java new file mode 100644 index 00000000000..36fc16b94eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.io.IOUtils; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.test.QueryTestCase; +import com.yahoo.vespa.config.ConfigPayload; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileConfigurationTestCase { + + public final String CONFIG_DIR ="src/test/java/com/yahoo/search/query/profile/config/test/"; + + @Test + public void testConfiguration() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profiles-configuration.cfg"); + QueryProfile profile=configurer.getCurrentRegistry().getComponent("default"); + + assertEquals("a-value",profile.get("a")); + assertEquals("b-value",profile.get("b")); + assertEquals("c.d-value",profile.get("c.d")); + assertFalse(profile.isDeclaredOverridable("c.d", null)); + assertEquals("e-value-inherited1",profile.get("e")); + assertEquals("g.d2-value-inherited1",profile.get("g.d2")); // Even though we make an explicit reference to one not having this value, we still inherit it + assertEquals("a-value-subprofile1",profile.get("sub1.a")); + assertEquals("c.d-value-subprofile1",profile.get("sub1.c.d")); + assertEquals("a-value-subprofile2",profile.get("sub2.a")); + assertEquals("c.d-value-subprofile2",profile.get("sub2.c.d")); + assertEquals("e-value-subprofile3",profile.get("g.e")); + } + + @Test + public void testBug3197426() { + QueryProfileConfigurer configurer = new QueryProfileConfigurer("file:" + CONFIG_DIR + "bug3197426.cfg"); + CompiledQueryProfile profile = configurer.getCurrentRegistry().getComponent("default").compile(null); + Map properties = new QueryProfileProperties(profile).listProperties("source.image"); + assertEquals("yes", properties.get("mlr")); + assertEquals("zh-Hant", properties.get("language")); + assertEquals("tw", properties.get("custid2")); + assertEquals("4", properties.get("hits")); + assertEquals("0", properties.get("offset")); + assertEquals("image", properties.get("catalog")); + assertEquals("yahoo", properties.get("custid1")); + assertEquals("utf-8", properties.get("encoding")); + assertEquals("all", properties.get("imquality")); + assertEquals("all", properties.get("dimensions")); + assertEquals("1", properties.get("flickr")); + assertEquals("yes", properties.get("ocr")); + } + + @Test + public void testVariantConfiguration() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); + + // Variant 1 + QueryProfile variants1 =configurer.getCurrentRegistry().getComponent("variants1"); + assertGet("x1.y1.a","a",new String[] { "x1","y1" }, variants1); + assertGet("x1.y1.b","b",new String[] { "x1","y1" }, variants1); + assertGet("x1.y?.a","a",new String[] { "x1","zz" }, variants1); + assertGet("x?.y1.a","a",new String[] { "zz","y1" }, variants1); + assertGet("a-deflt","a",new String[] { "z1","z2" }, variants1); + // ...inherited + assertGet("parent1-value","parent1",new String[] { "x1","y1" }, variants1); + assertGet("parent2-value","parent2",new String[] { "x1","y1" }, variants1); + assertGet(null,"parent1",new String[] { "x1","y2" }, variants1); + assertGet(null,"parent2",new String[] { "x1","y2" }, variants1); + + // Variant 2 + QueryProfile variants2 =configurer.getCurrentRegistry().getComponent("variants2"); + assertGet("variant2:y1.c","c",new String[] { "*","y1" }, variants2); + assertGet("variant2:y2.c","c",new String[] { "*","y2" }, variants2); + assertGet("variant2:c-df","c",new String[] { "*","z1" }, variants2); + assertGet("variant2:c-df","c",new String[] { }, variants2); + assertGet("variant2:c-df","c",new String[] { "*" }, variants2); + assertGet(null, "d",new String[] { "*","y1" }, variants2); + + // Reference following from variant 1 + assertGet("variant2:y1.c","toVariants.c",new String[] { "**", "y1" } , variants1); + assertGet("variant3:c-df","toVariants.c",new String[] { "x1", "**" } , variants1); + assertGet("variant3:y1.c","toVariants.c",new String[] { "x1", "y1" } , variants1); // variant3 by order priority + assertGet("variant3:y2.c","toVariants.c",new String[] { "x1", "y2" } , variants1); + } + + @Test + public void testVariantConfigurationThroughQueryLookup() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); + + CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); + CompiledQueryProfile variants1 = registry.getComponent("variants1"); + + // Variant 1 + assertEquals("x1.y1.a", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).properties().get("a")); + assertEquals("x1.y1.b", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).properties().get("b")); + assertEquals("x1.y1.defaultIndex", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).getModel().getDefaultIndex()); + assertEquals("x1.y?.a", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=zz"), variants1).properties().get("a")); + assertEquals("x1.y?.defaultIndex", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=zz"),variants1).getModel().getDefaultIndex()); + assertEquals("x?.y1.a", new Query(QueryTestCase.httpEncode("?query=foo&x=zz&y=y1"), variants1).properties().get("a")); + assertEquals("x?.y1.defaultIndex", new Query(QueryTestCase.httpEncode("?query=foo&x=zz&y=y1"), variants1).getModel().getDefaultIndex()); + assertEquals("x?.y1.filter", new Query(QueryTestCase.httpEncode("?query=foo&x=zz&y=y1"), variants1).getModel().getFilter()); + assertEquals("a-deflt", new Query(QueryTestCase.httpEncode("?query=foo&x=z1&y=z2"), variants1).properties().get("a")); + + // Variant 2 + CompiledQueryProfile variants2 = registry.getComponent("variants2"); + assertEquals("variant2:y1.c", new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=y1"), variants2).properties().get("c")); + assertEquals("variant2:y2.c", new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=y2"), variants2).properties().get("c")); + assertEquals("variant2:c-df", new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=z1"), variants2).properties().get("c")); + assertEquals("variant2:c-df", new Query(QueryTestCase.httpEncode("?query=foo"), variants2).properties().get("c")); + assertEquals("variant2:c-df", new Query(QueryTestCase.httpEncode("?query=foo&x=x1"), variants2).properties().get("c")); + assertNull(new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=y1"), variants2).properties().get("d")); + + // Reference following from variant 1 + assertEquals("variant2:y1.c", new Query(QueryTestCase.httpEncode("?query=foo&x=**&y=y1"), variants1).properties().get("toVariants.c")); + assertEquals("variant3:c-df", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=**"), variants1).properties().get("toVariants.c")); + assertEquals("variant3:y1.c", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).properties().get("toVariants.c")); + assertEquals("variant3:y2.c", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y2"), variants1).properties().get("toVariants.c")); + } + + @Test + public void testVariant2ConfigurationThroughQueryLookup() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants2.cfg"); + + CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); + Query query = new Query(QueryTestCase.httpEncode("?query=heh&queryProfile=multi&myindex=default&myquery=lo ve&tracelevel=5"), + registry.findQueryProfile("multi")); + assertEquals("love",query.properties().get("model.queryString")); + assertEquals("default",query.properties().get("model.defaultIndex")); + + assertEquals("-20",query.properties().get("ranking.features.query(scorelimit)")); + assertEquals("-20",query.getRanking().getFeatures().get("query(scorelimit)")); + query.properties().set("rankfeature.query(scorelimit)", -30); + assertEquals("-30",query.properties().get("ranking.features.query(scorelimit)")); + assertEquals("-30",query.getRanking().getFeatures().get("query(scorelimit)")); + } + + private void assertGet(String expectedValue,String parameter,String[] dimensionValues,QueryProfile profile) { + Map context=new HashMap<>(); + for (int i=0; i context = new HashMap<>(); + context.put("x", "x1"); + assertEquals(37, someUser.get("alder", context, null)); + assertEquals(37,someUser.get("anno", context, null)); + assertEquals(37,someUser.get("aLdER", context, null)); + assertEquals(37,someUser.get("ANNO", context, null)); + assertEquals("male",someUser.get("gender", context, null)); + assertEquals("male",someUser.get("sex", context, null)); + assertEquals("male",someUser.get("Sex", context, null)); + assertNull(someUser.get("Gender", context, null)); // Only aliases are case insensitive + } + + @Test + public void testBasicsNoProfile() { + Query q=new Query(HttpRequest.createTestRequest("?query=test", Method.GET)); + assertEquals("test",q.properties().get("query")); + assertEquals("test",q.properties().get("QueRY")); + assertEquals("test",q.properties().get("model.queryString")); + assertEquals("test",q.getModel().getQueryString()); + } + + @Test + public void testBasicsWithProfile() { + QueryProfile p = new QueryProfile("default"); + p.set("a", "foo", null); + Query q=new Query(HttpRequest.createTestRequest("?query=test", Method.GET), p.compile(null)); + assertEquals("test", q.properties().get("query")); + assertEquals("test", q.properties().get("QueRY")); + assertEquals("test", q.properties().get("model.queryString")); + assertEquals("test",q.getModel().getQueryString()); + } + + /** Tests a subset of the configuration in the system test of this */ + @Test + public void testSystemtest() { + String queryString = "?query=test"; + + QueryProfileXMLReader reader = new QueryProfileXMLReader(); + CompiledQueryProfileRegistry registry = reader.read("src/test/java/com/yahoo/search/query/profile/config/test/systemtest/").compile(); + HttpRequest request = HttpRequest.createTestRequest(queryString, Method.GET); + CompiledQueryProfile profile = registry.findQueryProfile("default"); + Query query = new Query(request, profile); + Properties p = query.properties(); + + assertEquals("test", query.getModel().getQueryString()); + assertEquals("test",p.get("query")); + assertEquals("test",p.get("QueRY")); + assertEquals("test",p.get("model.queryString")); + assertEquals("bar",p.get("foo")); + assertEquals(5,p.get("hits")); + assertEquals("tit",p.get("subst")); + assertEquals("le",p.get("subst.end")); + assertEquals("title",p.get("model.defaultIndex")); + + Map ps = p.listProperties(); + assertEquals(6,ps.size()); + assertEquals("bar",ps.get("foo")); + assertEquals("5",ps.get("hits")); + assertEquals("tit",ps.get("subst")); + assertEquals("le",ps.get("subst.end")); + assertEquals("title",ps.get("model.defaultIndex")); + assertEquals("test",ps.get("model.queryString")); + } + + @Test + public void testInvalid1() { + try { + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1"); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("Error reading query profile 'illegalSetting' of type 'native': Could not set 'model.notDeclared' to 'value': 'notDeclared' is not declared in query profile type 'model', and the type is strict", Exceptions.toMessageString(e)); + } + } + + @Test + public void testInvalid2() { + try { + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2"); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not parse 'unparseable.xml', error at line 2, column 21: Element type \"query-profile\" must be followed by either attribute specifications, \">\" or \"/>\".", Exceptions.toMessageString(e)); + } + } + + @Test + public void testInvalid3() { + try { + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/invalidxml3"); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("The file name of query profile 'MyProfile' must be 'MyProfile.xml' but was 'default.xml'", Exceptions.toMessageString(e)); + } + } + + @Test + public void testQueryProfileVariants() { + String query = "?query=test&dim1=yahoo&dim2=uk&dim3=test"; + + QueryProfileXMLReader reader = new QueryProfileXMLReader(); + CompiledQueryProfileRegistry registry = reader.read("src/test/java/com/yahoo/search/query/profile/config/test/news/").compile(); + HttpRequest request = HttpRequest.createTestRequest(query, Method.GET); + CompiledQueryProfile profile = registry.findQueryProfile("default"); + Query q = new Query(request, profile); + + assertEquals("c", q.properties().get("a.c")); + assertEquals("b", q.properties().get("a.b")); + } + + @Test + public void testNewsFE1() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newsfe").compile(); + + String queryString="tiled?vertical=news&query=barack&intl=us&resulttypes=article&testid=&clientintl=us&SpellState=&rss=0&tracelevel=5"; + + Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET), registry.getComponent("default")); + assertEquals("13",query.properties().listProperties().get("source.news.discovery.sources.count")); + assertEquals("13",query.properties().get("source.news.discovery.sources.count")); + assertEquals("sources",query.properties().listProperties().get("source.news.discovery")); + assertEquals("sources",query.properties().get("source.news.discovery")); + } + + @Test + public void testQueryProfileVariants2() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2").compile(); + CompiledQueryProfile multi = registry.getComponent("multi"); + + { + Query query=new Query(HttpRequest.createTestRequest("?queryProfile=multi", Method.GET), multi); + query.validate(); + assertEquals("best",query.properties().get("model.queryString")); + assertEquals("best",query.getModel().getQueryString()); + } + { + Query query=new Query(HttpRequest.createTestRequest("?queryProfile=multi&myindex=default", Method.GET), multi); + query.validate(); + assertEquals("best", query.properties().get("model.queryString")); + assertEquals("best", query.getModel().getQueryString()); + assertEquals("default", query.getModel().getDefaultIndex()); + } + { + Query query=new Query(HttpRequest.createTestRequest("?queryProfile=multi&myindex=default&myquery=love", Method.GET), multi); + query.validate(); + assertEquals("love", query.properties().get("model.queryString")); + assertEquals("love", query.getModel().getQueryString()); + assertEquals("default", query.getModel().getDefaultIndex()); + } + { + Query query=new Query(HttpRequest.createTestRequest("?model=querybest", Method.GET), multi); + query.validate(); + assertEquals("best",query.getModel().getQueryString()); + assertEquals("title",query.properties().get("model.defaultIndex")); + assertEquals("title",query.getModel().getDefaultIndex()); + } + } + + @Test + public void testKlee() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/klee"); + + QueryProfile pv=registry.getComponent("twitter_dd-us:0.2.4"); + assertEquals("0.2.4",pv.getId().getVersion().toString()); + assertEquals("[query profile 'production']",pv.inherited().toString()); + + QueryProfile p=registry.getComponent("twitter_dd-us:0.0.0"); + assertEquals("",p.getId().getVersion().toString()); // that is 0.0.0 + assertEquals("[query profile 'twitter_dd']",p.inherited().toString()); + } + + @Test + public void testVersions() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/versions"); + registry.freeze(); + + assertEquals("1.20.100",registry.findQueryProfile("testprofile:1.20.100").getId().getVersion().toString()); + assertEquals("1.20.100",registry.findQueryProfile("testprofile:1.20").getId().getVersion().toString()); + assertEquals("1.20.100",registry.findQueryProfile("testprofile:1").getId().getVersion().toString()); + assertEquals("1.20.100",registry.findQueryProfile("testprofile").getId().getVersion().toString()); + } + + @Test + public void testNewsFE2() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newsfe2").compile(); + + String queryString="tiled?query=a&intl=tw&mode=adv&mode=adv"; + + Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET),registry.getComponent("default")); + assertEquals("news_adv",query.properties().listProperties().get("provider")); + assertEquals("news_adv",query.properties().get("provider")); + } + + @Test + public void testSourceProvider() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider").compile(); + + String queryString="tiled?query=india&queryProfile=myprofile&source.common.intl=tw&source.common.mode=adv"; + + Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET), registry.getComponent("myprofile")); + for (Map.Entry e : query.properties().listProperties().entrySet()) + System.out.println(e); + assertEquals("news",query.properties().listProperties().get("source.common.provider")); + assertEquals("news",query.properties().get("source.common.provider")); + } + + @Test + public void testNewsCase1() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase1").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("ranking.features.b")); + assertEquals("0.0",query.properties().listProperties().get("ranking.features.b")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("ranking.features.b")); + assertEquals("0.1",query.properties().listProperties().get("ranking.features.b")); + } + + @Test + public void testNewsCase2() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase2").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("a.features.b")); + assertEquals("0.0",query.properties().listProperties().get("a.features.b")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("a.features.b")); + assertEquals("0.1",query.properties().listProperties().get("a.features.b")); + } + + @Test + public void testNewsCase3() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase3").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("a.features")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("a.features")); + } + + // Should cause an exception on the first line as we are trying to create a profile setting an illegal value in "ranking" + @Test + public void testNewsCase4() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase4").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("ranking.features")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("ranking.features")); + } + + @Test + public void testVersionRefs() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/versionrefs").compile(); + + Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); + assertEquals("MyProfile:1.0.2",query.properties().get("profile1.name")); + } + + @Test + public void testRefOverride() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverride").compile(); + + { + // Original reference + Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile1",query.properties().get("profileRef.name")); + assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + assertNull(query.properties().get("profileRef.myProfile2Only")); + } + + { + // Overridden reference + Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + assertNull(query.properties().get("profileRef.myProfile1Only")); + + // later assignment + query.properties().set("profileRef.name","newName"); + assertEquals("newName",query.properties().get("profileRef.name")); + // ...will not impact others + query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + } + + } + + @Test + public void testRefOverrideTyped() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); + + { + // Original reference + Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile1",query.properties().get("profileRef.name")); + assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + assertNull(query.properties().get("profileRef.myProfile2Only")); + } + + { + // Overridden reference + Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + assertNull(query.properties().get("profileRef.myProfile1Only")); + + // later assignment + query.properties().set("profileRef.name","newName"); + assertEquals("newName",query.properties().get("profileRef.name")); + // ...will not impact others + query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg new file mode 100644 index 00000000000..03422bc1020 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg @@ -0,0 +1,45 @@ +queryprofile[4] +queryprofile[0].id "default" +queryprofile[0].reference[1] +queryprofile[0].reference[0].name "source" +queryprofile[0].reference[0].value "source" +queryprofile[1].id "source" +queryprofile[1].reference[1] +queryprofile[1].reference[0].name "image" +queryprofile[1].reference[0].value "imageProfileTW" +queryprofile[2].id "imageProfileBase" +queryprofile[2].property[11] +queryprofile[2].property[0].name "hits" +queryprofile[2].property[0].value "4" +queryprofile[2].property[1].name "offset" +queryprofile[2].property[1].value "0" +queryprofile[2].property[2].name "catalog" +queryprofile[2].property[2].value "image" +queryprofile[2].property[3].name "custid1" +queryprofile[2].property[3].value "yahoo" +queryprofile[2].property[4].name "custid2" +queryprofile[2].property[4].value "us" +queryprofile[2].property[5].name "language" +queryprofile[2].property[5].value "en" +queryprofile[2].property[6].name "encoding" +queryprofile[2].property[6].value "utf-8" +queryprofile[2].property[7].name "imquality" +queryprofile[2].property[7].value "all" +queryprofile[2].property[8].name "dimensions" +queryprofile[2].property[8].value "all" +queryprofile[2].property[9].name "flickr" +queryprofile[2].property[9].value "1" +queryprofile[2].property[10].name "ocr" +queryprofile[2].property[10].value "yes" +queryprofile[3].id "imageProfileTW" +queryprofile[3].inherit[1] +queryprofile[3].inherit[0] imageProfileBase +queryprofile[3].property[4] +queryprofile[3].property[0].name "hits" +queryprofile[3].property[0].value "4" +queryprofile[3].property[1].name "custid2" +queryprofile[3].property[1].value "tw" +queryprofile[3].property[2].name "language" +queryprofile[3].property[2].value "zh-Hant" +queryprofile[3].property[3].name "mlr" +queryprofile[3].property[3].value "yes" \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml new file mode 100644 index 00000000000..cb1592e405b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml @@ -0,0 +1,4 @@ + + + value + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml new file mode 100644 index 00000000000..4bc6cb4e464 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml @@ -0,0 +1,2 @@ + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml new file mode 100644 index 00000000000..5d55d9626b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml @@ -0,0 +1,6 @@ + + + + production + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml new file mode 100644 index 00000000000..2946f581533 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml @@ -0,0 +1,14 @@ + + + 3 + true + freshness-mlrrecency4 + cosine + +yst_tweet_adult_score:0 + twitter_dd + -created_at + 21600 + true + 52 + en + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml new file mode 100644 index 00000000000..c96918f97f8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml @@ -0,0 +1,4 @@ + + + +yst_tweet_language:en +yst_tweet_adult_score:0 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml new file mode 100644 index 00000000000..588229d21c6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml @@ -0,0 +1,14 @@ + + + 3 + true + unranked + user,cosine + +yst_tweet_adult_score:0 + twitter_dd + + -created_at + 21600 + true + 52 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml new file mode 100644 index 00000000000..37354db337a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml @@ -0,0 +1,30 @@ + + + + + general-a + + + + us-nokia-test1-a + + + + + us-a + us-b + + + + + us-nokia-a + us-nokia-b + + + us-test1-a + us-test1-b + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml new file mode 100644 index 00000000000..bfb1c08c9e8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml @@ -0,0 +1,7 @@ + + + + region,model,bucket + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml new file mode 100644 index 00000000000..a89701a5720 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml @@ -0,0 +1,4 @@ + + + parent1-value + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml new file mode 100644 index 00000000000..59f08b3ef4c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml @@ -0,0 +1,4 @@ + + + parent2-value + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml new file mode 100644 index 00000000000..a1bd6a57727 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml @@ -0,0 +1,8 @@ + + +dim1,dim2,dim3 + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml new file mode 100644 index 00000000000..2e4dbce956d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml @@ -0,0 +1,4 @@ + + +b + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml new file mode 100644 index 00000000000..fe4ed037532 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml @@ -0,0 +1,4 @@ + + +c + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml new file mode 100644 index 00000000000..2dc8aebd6ce --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml @@ -0,0 +1,15 @@ + + + + + custid_1,custid_2 + + + usrank + + + + 0.1 + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml new file mode 100644 index 00000000000..157b5ae9702 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml @@ -0,0 +1,5 @@ + + + + 0.0 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml new file mode 100644 index 00000000000..5c2ed77211f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml @@ -0,0 +1,15 @@ + + + + + custid_1,custid_2,custid_3,custid_4,custid_5,custid_6 + + + usrank + + + + 0.1 + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml new file mode 100644 index 00000000000..25bff4ada59 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml @@ -0,0 +1,5 @@ + + + + 0.0 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml new file mode 100644 index 00000000000..736ab0020d6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml @@ -0,0 +1,15 @@ + + + + + custid_1,custid_2,custid_3,custid_4,custid_5,custid_6 + + + usrank + + + + 0.1 + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml new file mode 100644 index 00000000000..473fbd9610e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml @@ -0,0 +1,5 @@ + + + + 0.0 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml new file mode 100644 index 00000000000..1efcd6e4d87 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml @@ -0,0 +1,15 @@ + + + + + custid_1,custid_2,custid_3,custid_4,custid_5,custid_6 + + + usrank + + + + 0.1 + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml new file mode 100644 index 00000000000..23c2b657182 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml @@ -0,0 +1,5 @@ + + + + 0.0 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml new file mode 100644 index 00000000000..3585ccd5eda --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml @@ -0,0 +1,9 @@ + + + vertical,sort,offset,resulttypes,rss,age,intl,testid + + sources + article + 13 + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml new file mode 100644 index 00000000000..d8dbe6e929a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml @@ -0,0 +1,4 @@ + + + backend/news + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml new file mode 100644 index 00000000000..b0168f583b4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml @@ -0,0 +1,8 @@ + + + mode + news_basic + + news_adv + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml new file mode 100644 index 00000000000..d43538bd106 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml @@ -0,0 +1,5 @@ + + + intl + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg new file mode 100644 index 00000000000..21d036080e1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg @@ -0,0 +1,95 @@ +queryprofile[5] +queryprofile[0].id "variants1" +queryprofile[0].dimensions[2] +queryprofile[0].dimensions[0] x +queryprofile[0].dimensions[1] y +queryprofile[0].property[2] +queryprofile[0].property[0].name "a" +queryprofile[0].property[0].value "a-deflt" +queryprofile[0].property[1].name "model.defaultIndex" +queryprofile[0].property[1].value "defaultIndex-default" +queryprofile[0].queryprofilevariant[3] +queryprofile[0].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[0].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[0].queryprofilevariant[0].inherit[2] +queryprofile[0].queryprofilevariant[0].inherit[0] "parent1" +queryprofile[0].queryprofilevariant[0].inherit[1] "parent2" +queryprofile[0].queryprofilevariant[0].property[3] +queryprofile[0].queryprofilevariant[0].property[0].name "a" +queryprofile[0].queryprofilevariant[0].property[0].value "x1.y1.a" +queryprofile[0].queryprofilevariant[0].property[1].name "b" +queryprofile[0].queryprofilevariant[0].property[1].value "x1.y1.b" +queryprofile[0].queryprofilevariant[0].property[2].name "model.defaultIndex" +queryprofile[0].queryprofilevariant[0].property[2].value "x1.y1.defaultIndex" +queryprofile[0].queryprofilevariant[1].fordimensionvalues[1] +queryprofile[0].queryprofilevariant[1].fordimensionvalues[0] "x1" +queryprofile[0].queryprofilevariant[1].property[2] +queryprofile[0].queryprofilevariant[1].property[0].name "a" +queryprofile[0].queryprofilevariant[1].property[0].value "x1.y?.a" +queryprofile[0].queryprofilevariant[1].property[1].name "model.defaultIndex" +queryprofile[0].queryprofilevariant[1].property[1].value "x1.y?.defaultIndex" +queryprofile[0].queryprofilevariant[1].reference[1] +queryprofile[0].queryprofilevariant[1].reference[0].name "toVariants" +queryprofile[0].queryprofilevariant[1].reference[0].value "variants3" +queryprofile[0].queryprofilevariant[2].fordimensionvalues[2] +queryprofile[0].queryprofilevariant[2].fordimensionvalues[0] "*" +queryprofile[0].queryprofilevariant[2].fordimensionvalues[1] "y1" +queryprofile[0].queryprofilevariant[2].property[3] +queryprofile[0].queryprofilevariant[2].property[0].name "a" +queryprofile[0].queryprofilevariant[2].property[0].value "x?.y1.a" +queryprofile[0].queryprofilevariant[2].property[1].name "model.filter" +queryprofile[0].queryprofilevariant[2].property[1].value "x?.y1.filter" +queryprofile[0].queryprofilevariant[2].property[2].name "model.defaultIndex" +queryprofile[0].queryprofilevariant[2].property[2].value "x?.y1.defaultIndex" +queryprofile[0].queryprofilevariant[2].reference[1] +queryprofile[0].queryprofilevariant[2].reference[0].name "toVariants" +queryprofile[0].queryprofilevariant[2].reference[0].value "variants2" +queryprofile[1].id "variants2" +queryprofile[1].dimensions[2] +queryprofile[1].dimensions[0] x +queryprofile[1].dimensions[1] y +queryprofile[1].property[1] +queryprofile[1].property[0].name "c" +queryprofile[1].property[0].value "variant2:c-df" +queryprofile[1].queryprofilevariant[2] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[1].queryprofilevariant[0].property[1] +queryprofile[1].queryprofilevariant[0].property[0].name "c" +queryprofile[1].queryprofilevariant[0].property[0].value "variant2:y1.c" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[1] "y2" +queryprofile[1].queryprofilevariant[1].property[1] +queryprofile[1].queryprofilevariant[1].property[0].name "c" +queryprofile[1].queryprofilevariant[1].property[0].value "variant2:y2.c" +queryprofile[2].id "variants3" +queryprofile[2].dimensions[2] +queryprofile[2].dimensions[0] x +queryprofile[2].dimensions[1] y +queryprofile[2].property[1] +queryprofile[2].property[0].name "c" +queryprofile[2].property[0].value "variant3:c-df" +queryprofile[2].queryprofilevariant[2] +queryprofile[2].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[2].queryprofilevariant[0].fordimensionvalues[0] "*" +queryprofile[2].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[2].queryprofilevariant[0].property[1] +queryprofile[2].queryprofilevariant[0].property[0].name "c" +queryprofile[2].queryprofilevariant[0].property[0].value "variant3:y1.c" +queryprofile[2].queryprofilevariant[1].fordimensionvalues[2] +queryprofile[2].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[2].queryprofilevariant[1].fordimensionvalues[1] "y2" +queryprofile[2].queryprofilevariant[1].property[1] +queryprofile[2].queryprofilevariant[1].property[0].name "c" +queryprofile[2].queryprofilevariant[1].property[0].value "variant3:y2.c" +queryprofile[3].id "parent1" +queryprofile[3].property[1] +queryprofile[3].property[0].name "parent1" +queryprofile[3].property[0].value "parent1-value" +queryprofile[4].id "parent2" +queryprofile[4].property[1] +queryprofile[4].property[0].name "parent2" +queryprofile[4].property[0].value "parent2-value" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg new file mode 100644 index 00000000000..ec091ecf2ea --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg @@ -0,0 +1,63 @@ +queryprofile[4] +queryprofile[0].id "default" +queryprofile[0].property[5] +queryprofile[0].property[0].name "hits" +queryprofile[0].property[0].value "5" +queryprofile[0].property[1].name "model.defaultIndex" +queryprofile[0].property[1].value "title" +queryprofile[0].property[2].name "ranking.features.query(scorelimit)" +queryprofile[0].property[2].value "-20" +queryprofile[0].property[3].name "ranking.profile" +queryprofile[0].property[3].value "production1" +queryprofile[0].property[4].name "ranking.properties.dotProduct.X" +queryprofile[0].property[4].value "(a:1,b:2)" +queryprofile[1].id "multi" +queryprofile[1].inherit[1] +queryprofile[1].inherit[0] "default" +queryprofile[1].dimensions[2] +queryprofile[1].dimensions[0] "myquery" +queryprofile[1].dimensions[1] "myindex" +queryprofile[1].reference[1] +queryprofile[1].reference[0].name "model" +queryprofile[1].reference[0].value "querybest" +queryprofile[1].property[1] +queryprofile[1].property[0].name "model.defaultIndex" +queryprofile[1].property[0].value "default-default" +queryprofile[1].queryprofilevariant[3] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "lo ve" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[1] "default" +queryprofile[1].queryprofilevariant[0].reference[1] +queryprofile[1].queryprofilevariant[0].reference[0].name "model" +queryprofile[1].queryprofilevariant[0].reference[0].value "querylove" +queryprofile[1].queryprofilevariant[0].property[1] +queryprofile[1].queryprofilevariant[0].property[0].name "model.defaultIndex" +queryprofile[1].queryprofilevariant[0].property[0].value "default" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[1] "default" +queryprofile[1].queryprofilevariant[1].property[1] +queryprofile[1].queryprofilevariant[1].property[0].name "model.defaultIndex" +queryprofile[1].queryprofilevariant[1].property[0].value "default" +queryprofile[1].queryprofilevariant[2].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[2].fordimensionvalues[0] "lo ve" +queryprofile[1].queryprofilevariant[2].fordimensionvalues[1] "*" +queryprofile[1].queryprofilevariant[2].reference[1] +queryprofile[1].queryprofilevariant[2].reference[0].name "model" +queryprofile[1].queryprofilevariant[2].reference[0].value "querylove" +queryprofile[2].id "querybest" +queryprofile[2].type "model" +queryprofile[2].property[2] +queryprofile[2].property[0].name "defaultIndex" +queryprofile[2].property[0].value "title" +queryprofile[2].property[1].name "queryString" +queryprofile[2].property[1].value "best" +queryprofile[2].property[1].overridable false +queryprofile[3].id "querylove" +queryprofile[3].type "model" +queryprofile[3].property[2] +queryprofile[3].property[0].name "defaultIndex" +queryprofile[3].property[0].value "title" +queryprofile[3].property[1].name "queryString" +queryprofile[3].property[1].value "love" +queryprofile[3].property[1].overridable false diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg new file mode 100644 index 00000000000..6d3e957a722 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg @@ -0,0 +1,52 @@ +queryprofile[6] +queryprofile[0].id "subprofile3" +queryprofile[0].property[1] +queryprofile[0].property[0].name "e" +queryprofile[0].property[0].value "e-value-subprofile3" +queryprofile[1].id "inherited1" +queryprofile[1].property[4] +queryprofile[1].property[0].name "a" +queryprofile[1].property[0].value "a-value-inherited1" +queryprofile[1].property[1].name "e" +queryprofile[1].property[1].value "e-value-inherited1" +queryprofile[1].property[2].name "c.d" +queryprofile[1].property[2].value "c.d-value-inherited1" +queryprofile[1].property[3].name "g.d2" +queryprofile[1].property[3].value "g.d2-value-inherited1" +queryprofile[2].id "inherited2" +queryprofile[2].property[2] +queryprofile[2].property[0].name "a" +queryprofile[2].property[0].value "a-value-inherited2" +queryprofile[2].property[1].name "c.d2" +queryprofile[2].property[1].value "c.d2-value-inherited2" +queryprofile[3].id "subprofile1" +queryprofile[3].property[2] +queryprofile[3].property[0].name "a" +queryprofile[3].property[0].value "a-value-subprofile1" +queryprofile[3].property[1].name "c.d" +queryprofile[3].property[1].value "c.d-value-subprofile1" +queryprofile[4].id "subprofile2" +queryprofile[4].property[2] +queryprofile[4].property[0].name "a" +queryprofile[4].property[0].value "a-value-subprofile2" +queryprofile[4].property[1].name "c.d" +queryprofile[4].property[1].value "c.d-value-subprofile2" +queryprofile[5].id "default" +queryprofile[5].inherit[2] +queryprofile[5].inherit[0] inherited1 +queryprofile[5].inherit[1] inherited2 +queryprofile[5].property[3] +queryprofile[5].property[0].name "a" +queryprofile[5].property[0].value "a-value" +queryprofile[5].property[1].name "b" +queryprofile[5].property[1].value "b-value" +queryprofile[5].property[2].name "c.d" +queryprofile[5].property[2].value "c.d-value" +queryprofile[5].property[2].overridable "false" +queryprofile[5].reference[3] +queryprofile[5].reference[0].name "sub1" +queryprofile[5].reference[0].value "subprofile1" +queryprofile[5].reference[1].name "sub2" +queryprofile[5].reference[1].value "subprofile2" +queryprofile[5].reference[2].name "g" +queryprofile[5].reference[2].value "subprofile3" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml new file mode 100644 index 00000000000..ce40b67e3de --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml @@ -0,0 +1,10 @@ + + + 5 + %{subst}%{subst.end} + production1 + -20 + (a:1,b:2) + tit + le + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml new file mode 100644 index 00000000000..5b67e6682c8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml @@ -0,0 +1,3 @@ + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml new file mode 100644 index 00000000000..39b835cb536 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml @@ -0,0 +1,5 @@ + + + 1377 + 37 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml new file mode 100644 index 00000000000..a4feef724b4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml @@ -0,0 +1,22 @@ + + + querybest + + + querylove + default + + + + default + + + + querylove + + + + +me + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml new file mode 100644 index 00000000000..e4abc4a2202 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml @@ -0,0 +1,4 @@ + + + myquery, myindex + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml new file mode 100644 index 00000000000..b6e5031705b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml @@ -0,0 +1,5 @@ + + + title + best + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml new file mode 100644 index 00000000000..e7864977804 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml @@ -0,0 +1,5 @@ + + + title + love + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml new file mode 100644 index 00000000000..ceb1d0302c6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml @@ -0,0 +1,4 @@ + + + querybest + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml new file mode 100644 index 00000000000..e8412121d67 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml @@ -0,0 +1,5 @@ + + + default + 1 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml new file mode 100644 index 00000000000..60e5026bc5f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml @@ -0,0 +1,7 @@ + + + %{indexname} + test1 + 10 + default + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml new file mode 100644 index 00000000000..b7060093d74 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml @@ -0,0 +1,3 @@ + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml new file mode 100644 index 00000000000..f942e5c3cb5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml @@ -0,0 +1,3 @@ + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml new file mode 100644 index 00000000000..1cb98d12ba8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml @@ -0,0 +1,4 @@ + + + +best + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml new file mode 100644 index 00000000000..6146a6ef7d0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml @@ -0,0 +1,5 @@ + + + 3 + true + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml new file mode 100644 index 00000000000..6b1f666929f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml new file mode 100644 index 00000000000..ea6180f7379 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml new file mode 100644 index 00000000000..74077baafff --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml new file mode 100644 index 00000000000..9b257e1c8a6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml @@ -0,0 +1,4 @@ + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml new file mode 100644 index 00000000000..4b81164309d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml @@ -0,0 +1,5 @@ + + + MyProfile1 + myProfile1Only + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml new file mode 100644 index 00000000000..14bb544b744 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml @@ -0,0 +1,5 @@ + + + MyProfile2 + myProfile2Only + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml new file mode 100644 index 00000000000..252b84600ed --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml @@ -0,0 +1,5 @@ + + + default + MyProfile1 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml new file mode 100644 index 00000000000..4b81164309d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml @@ -0,0 +1,5 @@ + + + MyProfile1 + myProfile1Only + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml new file mode 100644 index 00000000000..14bb544b744 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml @@ -0,0 +1,5 @@ + + + MyProfile2 + myProfile2Only + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml new file mode 100644 index 00000000000..cf425a819e0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml @@ -0,0 +1,5 @@ + + + default + MyProfile1 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml new file mode 100644 index 00000000000..78a1db68661 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml @@ -0,0 +1,4 @@ + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml new file mode 100755 index 00000000000..eb6cc805bc4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml @@ -0,0 +1,5 @@ + + + source.common.intl + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml new file mode 100755 index 00000000000..6260c08e9fa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml @@ -0,0 +1,6 @@ + + + common + source + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml new file mode 100755 index 00000000000..71ed8000a9d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml @@ -0,0 +1,9 @@ + + + source.common.mode + yst + + news + + test + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml new file mode 100755 index 00000000000..a33a3cb455a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml @@ -0,0 +1,4 @@ + + + common + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml new file mode 100644 index 00000000000..873233042e4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml @@ -0,0 +1,8 @@ + + + bar + 5 + %{subst}%{subst.end} + tit + le + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg new file mode 100644 index 00000000000..27964bbd94f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg @@ -0,0 +1,42 @@ +queryprofiletype[4] + +queryprofiletype[0].id testtype +queryprofiletype[0].field[7] +queryprofiletype[0].field[0].name "myString" +queryprofiletype[0].field[0].type "string" +queryprofiletype[0].field[0].mandatory true +queryprofiletype[0].field[1].name "myInteger" +queryprofiletype[0].field[1].type "integer" +queryprofiletype[0].field[1].overridable false +queryprofiletype[0].field[2].name "myLong" +queryprofiletype[0].field[2].type "long" +queryprofiletype[0].field[3].name "myFloat" +queryprofiletype[0].field[3].type "float" +queryprofiletype[0].field[4].name "myDouble" +queryprofiletype[0].field[4].type "double" +queryprofiletype[0].field[5].name "myQueryProfile" +queryprofiletype[0].field[5].type "query-profile" +queryprofiletype[0].field[6].name "myUserQueryProfile" +queryprofiletype[0].field[6].type "query-profile:user" +queryprofiletype[0].field[6].alias "myqp user-profile" + +queryprofiletype[1].id testtypestrict +queryprofiletype[1].strict true +queryprofiletype[1].matchaspath true +queryprofiletype[1].inherit[1] +queryprofiletype[1].inherit[0] "testtype" +queryprofiletype[1].field[1] +queryprofiletype[1].field[0].name "myUserQueryProfile" +queryprofiletype[1].field[0].type "query-profile:userstrict" + +queryprofiletype[2].id user +queryprofiletype[2].field[2] +queryprofiletype[2].field[0].name "myUserString" +queryprofiletype[2].field[0].type "string" +queryprofiletype[2].field[1].name "myUserInteger" +queryprofiletype[2].field[1].type "integer" + +queryprofiletype[3].id userstrict +queryprofiletype[3].strict true +queryprofiletype[3].inherit[1] +queryprofiletype[3].inherit[0] "user" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore new file mode 100644 index 00000000000..0a1c2c442c1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore @@ -0,0 +1,10 @@ +bundles.cfg +container-mbus.cfg +diagnostics.cfg +documentdb-info.cfg +documentmanager.cfg +int.cfg +qr-templates.cfg +renderers.cfg +schemamapping.cfg +string.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg new file mode 100644 index 00000000000..99af9283ea8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg @@ -0,0 +1,16 @@ +chains[2] +chains[0].id test +chains[0].components[3] +chains[0].components[0] SettingSearcher +chains[0].components[1] ReceivingSearcher +chains[0].components[2] TestSearcher +chains[1].id default +chains[1].components[3] +chains[1].components[0] SettingSearcher +chains[1].components[1] ReceivingSearcher +chains[1].components[2] DefaultSearcher +components[4] +components[0].id SettingSearcher +components[1].id ReceivingSearcher +components[2].id TestSearcher +components[3].id DefaultSearcher diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg new file mode 100644 index 00000000000..1110d76f887 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg @@ -0,0 +1,11 @@ +components[6] +components[0].id SettingSearcher +components[0].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$SettingSearcher +components[1].id ReceivingSearcher +components[1].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$ReceivingSearcher +components[2].id TestSearcher +components[2].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$TestSearcher +components[3].id DefaultSearcher +components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$DefaultSearcher +components[4].id com.yahoo.search.handler.SearchHandler +components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/index-info.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg new file mode 100644 index 00000000000..7c0b22a3606 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg @@ -0,0 +1,49 @@ +queryprofile[5] +queryprofile[0].id "default" +queryprofile[1].type "root" +queryprofile[0].property[1] +queryprofile[0].property[0].name "searchChain" +queryprofile[0].property[0].value "test" +queryprofile[1].id "test" +queryprofile[1].type "root" +queryprofile[1].property[1] +queryprofile[1].property[0].name "searchChain" +queryprofile[1].property[0].value "default" +queryprofile[2].id "hitsoffset" +queryprofile[2].type "root" +queryprofile[2].property[2] +queryprofile[2].property[0].name "hits" +queryprofile[2].property[0].value "22" +queryprofile[2].property[1].name "offset" +queryprofile[2].property[1].value "80" +queryprofile[3].id "root" +queryprofile[3].type "root" +queryprofile[3].property[2] +queryprofile[3].property[0].name "sub.value1" +queryprofile[3].property[0].value "subvalue1" +queryprofile[3].property[1].name "sub.value2" +queryprofile[3].property[1].value "subvalue2" +queryprofile[4].id "newsub" +queryprofile[4].type "sub" +queryprofile[4].property[2] +queryprofile[4].property[0].name "value1" +queryprofile[4].property[0].value "newsubvalue1" +queryprofile[4].property[1].name "value2" +queryprofile[4].property[1].value "newsubvalue2" + +queryprofiletype[2] +queryprofiletype[0].id "root" +queryprofiletype[0].inherit[1] +queryprofiletype[0].inherit[0] "native" +queryprofiletype[0].strict true +queryprofiletype[0].matchaspath true +queryprofiletype[0].field[1] +queryprofiletype[0].field[0].name "sub" +queryprofiletype[0].field[0].type "query-profile:sub" +queryprofiletype[1].id "sub" +queryprofiletype[1].strict true +queryprofiletype[1].field[2] +queryprofiletype[1].field[0].name "value1" +queryprofiletype[1].field[0].type "string" +queryprofiletype[1].field[1].name "value2" +queryprofiletype[1].field[1].type "string" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore new file mode 100644 index 00000000000..0a1c2c442c1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore @@ -0,0 +1,10 @@ +bundles.cfg +container-mbus.cfg +diagnostics.cfg +documentdb-info.cfg +documentmanager.cfg +int.cfg +qr-templates.cfg +renderers.cfg +schemamapping.cfg +string.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg new file mode 100644 index 00000000000..99af9283ea8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg @@ -0,0 +1,16 @@ +chains[2] +chains[0].id test +chains[0].components[3] +chains[0].components[0] SettingSearcher +chains[0].components[1] ReceivingSearcher +chains[0].components[2] TestSearcher +chains[1].id default +chains[1].components[3] +chains[1].components[0] SettingSearcher +chains[1].components[1] ReceivingSearcher +chains[1].components[2] DefaultSearcher +components[4] +components[0].id SettingSearcher +components[1].id ReceivingSearcher +components[2].id TestSearcher +components[3].id DefaultSearcher diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg new file mode 100644 index 00000000000..70f5452b74d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg @@ -0,0 +1,11 @@ +components[6] +components[0].id SettingSearcher +components[0].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$SettingSearcher +components[1].id ReceivingSearcher +components[1].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$ReceivingSearcher +components[2].id TestSearcher +components[2].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$TestSearcher +components[3].id DefaultSearcher +components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$DefaultSearcher +components[4].id com.yahoo.search.handler.SearchHandler +components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/index-info.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg new file mode 100644 index 00000000000..e5fe53e4e0a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg @@ -0,0 +1,29 @@ +queryprofile[5] +queryprofile[0].id "default" +queryprofile[0].property[1] +queryprofile[0].property[0].name "searchChain" +queryprofile[0].property[0].value "test" +queryprofile[1].id "test" +queryprofile[1].property[2] +queryprofile[1].property[0].name "searchChain" +queryprofile[1].property[0].value "default" +queryprofile[1].property[1].name "model.defaultIndex" +queryprofile[1].property[1].value "index" +queryprofile[2].id "hitsoffset" +queryprofile[2].property[2] +queryprofile[2].property[0].name "hits" +queryprofile[2].property[0].value "20" +queryprofile[2].property[1].name "offset" +queryprofile[2].property[1].value "80" +queryprofile[3].id "root" +queryprofile[3].property[2] +queryprofile[3].property[0].name "sub.value1" +queryprofile[3].property[0].value "subvalue1" +queryprofile[3].property[1].name "sub.value2" +queryprofile[3].property[1].value "subvalue2" +queryprofile[4].id "newsub" +queryprofile[4].property[2] +queryprofile[4].property[0].name "value1" +queryprofile[4].property[0].value "newsubvalue1" +queryprofile[4].property[1].name "value2" +queryprofile[4].property[1].value "newsubvalue2" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml new file mode 100644 index 00000000000..a93771e68cb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml @@ -0,0 +1,10 @@ + + + + + + 20 + false + default + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml new file mode 100644 index 00000000000..a045635966e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml @@ -0,0 +1,4 @@ + + + some query + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml new file mode 100644 index 00000000000..fe07ae55a1f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml @@ -0,0 +1,7 @@ + + + + modelSettings + aDefaultIndex + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml new file mode 100644 index 00000000000..05606e2be8d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml @@ -0,0 +1,9 @@ + + + + + + 30 + 3 + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml new file mode 100644 index 00000000000..42175fd6368 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml @@ -0,0 +1,12 @@ + + + + + x + 18 + 5 + + 37 + male + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml new file mode 100644 index 00000000000..bf49c03e57f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml new file mode 100644 index 00000000000..96f702a43eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml new file mode 100644 index 00000000000..7aad3ee2df7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml @@ -0,0 +1,4 @@ + + + MyProfile:1.0.0 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml new file mode 100644 index 00000000000..0314223b732 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml @@ -0,0 +1,4 @@ + + + MyProfile:1.0.2.a + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml new file mode 100644 index 00000000000..debd93850ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml @@ -0,0 +1,4 @@ + + + MyProfile:1.0.2 + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml new file mode 100644 index 00000000000..67dc76c851f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml @@ -0,0 +1,5 @@ + + + default + MyProfile + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml new file mode 100644 index 00000000000..08ae7dd9ee0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml @@ -0,0 +1,3 @@ + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml new file mode 100644 index 00000000000..12309a78075 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml @@ -0,0 +1,3 @@ + + + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java new file mode 100644 index 00000000000..4e68ac92adc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.jdisc.http.HttpRequest.Method; + +/** + * @author bratseth + */ +public class CloningTestCase extends junit.framework.TestCase { + + public void testCloningWithVariants() { + QueryProfile test = new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a","a1"); + Query q2 = q1.clone(); + q2.properties().set("a","a2"); + assertEquals("a1",q1.properties().get("a")); + assertEquals("a2",q2.properties().get("a")); + } + + public void testShallowCloning() { + QueryProfile test = new QueryProfile("test"); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + q1.properties().set("a",new MutableString("a1")); + Query q2 = q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a2",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testShallowCloningWithVariants() { + QueryProfile test = new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a",new MutableString("a1")); + Query q2 = q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a2",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testDeepCloning() { + QueryProfile test=new QueryProfile("test"); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + q1.properties().set("a",new CloneableMutableString("a1")); + Query q2=q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a1",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testDeepCloningWithVariants() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a",new CloneableMutableString("a1")); + Query q2=q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a1",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testReAssignment() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a","a1"); + q1.properties().set("a","a2"); + assertEquals("a2",q1.properties().get("a")); + } + + public void testThreeLevelsOfCloning() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + q1.properties().set("a","q1-a"); + Query q2=q1.clone(); + q2.properties().set("a","q2-a"); + Query q31=q2.clone(); + q31.properties().set("a","q31-a"); + Query q32=q2.clone(); + q32.properties().set("a","q32-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + } + + public void testThreeLevelsOfCloningReverseSetOrder() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + Query q2=q1.clone(); + Query q31=q2.clone(); + Query q32=q2.clone(); + q32.properties().set("a","q32-a"); + q31.properties().set("a","q31-a"); + q2.properties().set("a","q2-a"); + q1.properties().set("a","q1-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + } + + public void testThreeLevelsOfCloningMiddleFirstSetOrder1() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + Query q2=q1.clone(); + Query q31=q2.clone(); + Query q32=q2.clone(); + q2.properties().set("a","q2-a"); + q32.properties().set("a","q32-a"); + q31.properties().set("a","q31-a"); + q1.properties().set("a","q1-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + } + + public void testThreeLevelsOfCloningMiddleFirstSetOrder2() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + Query q2=q1.clone(); + Query q31=q2.clone(); + Query q32=q2.clone(); + q2.properties().set("a","q2-a"); + q31.properties().set("a","q31-a"); + q1.properties().set("a","q1-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("config-a",q32.properties().get("a").toString()); + q1.properties().set("a","q1-a-2"); + assertEquals("q1-a-2",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("config-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a-2",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("config-a",q32.properties().get("a").toString()); + } + + public static class MutableString { + + private String string; + + public MutableString(String string) { + this.string=string; + } + + public void set(String string) { this.string=string; } + + public @Override String toString() { return string; } + + public @Override int hashCode() { return string.hashCode(); } + + public @Override boolean equals(Object other) { + if (other==this) return true; + if ( ! (other instanceof MutableString)) return false; + return ((MutableString)other).string.equals(string); + } + + } + + public static class CloneableMutableString extends MutableString implements Cloneable { + + public CloneableMutableString(String string) { + super(string); + } + + public @Override CloneableMutableString clone() { + try { + return (CloneableMutableString)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java new file mode 100644 index 00000000000..28cd0f4daf9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.search.query.profile.DimensionBinding; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class DimensionBindingTestCase { + + @Test + public void testCombining() { + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("c", "c=1"))); + + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("a, c", "a=1", "c=1"))); + + assertEquals(binding("c, a, b", "c=1", "a=1", "b=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("c, a", "a=1", "c=1"))); + + assertEquals(binding("a, b", "a=1", "b=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("a, b", "a=1", "b=1"))); + + assertEquals(DimensionBinding.invalidBinding, + binding("a, b", "a=1", "b=1").combineWith(binding("b, a", "a=1", "b=1"))); + + assertEquals(binding("a, b", "a=1", "b=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("b", "b=1"))); + + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, b, c", "a=1", "c=1").combineWith(binding("a, b, c", "a=1", "b=1", "c=1"))); + + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, c", "a=1", "c=1").combineWith(binding("a, b, c", "a=1", "b=1", "c=1"))); + } + + // found DimensionBinding [custid_1=yahoo, custid_2=ca, custid_3=sc, custid_4=null, custid_5=null, custid_6=null], combined with DimensionBinding [custid_1=yahoo, custid_2=null, custid_3=sc, custid_4=null, custid_5=null, custid_6=null] to Invalid DimensionBinding + @Test + public void testCombiningBindingsWithNull() { + List dimensions = list("a,b"); + + Map map1 = new HashMap<>(); + map1.put("a","a1"); + map1.put("b","b1"); + + Map map2 = new HashMap<>(); + map2.put("a","a1"); + map2.put("b",null); + + assertEquals(DimensionBinding.createFrom(dimensions, map1), + DimensionBinding.createFrom(dimensions, map1).combineWith(DimensionBinding.createFrom(dimensions, map2))); + } + + private DimensionBinding binding(String dimensions, String ... dimensionPoints) { + return DimensionBinding.createFrom(list(dimensions), QueryProfileVariantsTestCase.toMap(dimensionPoints)); + } + + private List list(String listString) { + List l = new ArrayList<>(); + for (String s : listString.split(",")) + l.add(s.trim()); + return l; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java new file mode 100644 index 00000000000..576605b8fa4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.search.query.profile.DumpTool; + +/** + * @author bratseth + */ +public class DumpToolTestCase extends junit.framework.TestCase { + + String profileDir="src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"; + + public void testNoParameters() { + assertTrue(new DumpTool().resolveAndDump().startsWith("Dumps all resolved")); + } + + public void testHelpParameter() { + assertTrue(new DumpTool().resolveAndDump("-help").startsWith("Dumps all resolved")); + } + + public void testNoDimensionValues() { + assertTrue(new DumpTool().resolveAndDump("multiprofile1",profileDir).startsWith("a=general-a\n")); + } + + public void testAllParametersSet() { + assertTrue(new DumpTool().resolveAndDump("multiprofile1",profileDir,"").startsWith("a=general-a\n")); + } + + //This test is order dependent. Fix this!! + public void testVariant() { + System.out.println(new DumpTool().resolveAndDump("multiprofile1",profileDir,"region=us")); + assertTrue(new DumpTool().resolveAndDump("multiprofile1",profileDir,"region=us").startsWith("a=us-a\nb=us-b\nregion=us")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java new file mode 100644 index 00000000000..e67d7723932 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * Test using the profile to set the query to execute + * + * @author bratseth + */ +public class QueryFromProfileTestCase extends junit.framework.TestCase { + + public void testQueryFromProfile1() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile topLevel = new QueryProfile("topLevel"); + topLevel.setType(registry.getTypeRegistry().getComponent("native")); + registry.register(topLevel); + + QueryProfile queryBest = new QueryProfile("querybest"); + queryBest.setType(registry.getTypeRegistry().getComponent("model")); + queryBest.set("queryString", "best", registry); + registry.register(queryBest); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?model=querybest", Method.GET), cRegistry.getComponent("topLevel")); + assertEquals("best", query.properties().get("model.queryString")); + assertEquals("best", query.getModel().getQueryTree().toString()); + } + + public void testQueryFromProfile2() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType rootType = new QueryProfileType("root"); + rootType.inherited().add(registry.getTypeRegistry().getComponent("native")); + registry.getTypeRegistry().register(rootType); + + QueryProfile root = new QueryProfile("root"); + root.setType(rootType); + registry.register(root); + + QueryProfile queryBest=new QueryProfile("querybest"); + queryBest.setType(registry.getTypeRegistry().getComponent("model")); + queryBest.set("queryString", "best", registry); + registry.register(queryBest); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?query=overrides&model=querybest", Method.GET), cRegistry.getComponent("root")); + assertEquals("overrides", query.properties().get("model.queryString")); + assertEquals("overrides", query.getModel().getQueryTree().toString()); + } + + public void testQueryFromProfile3() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType rootType = new QueryProfileType("root"); + rootType.inherited().add(registry.getTypeRegistry().getComponent("native")); + registry.getTypeRegistry().register(rootType); + + QueryProfile root = new QueryProfile("root"); + root.setType(rootType); + registry.register(root); + + QueryProfile queryBest=new QueryProfile("querybest"); + queryBest.setType(registry.getTypeRegistry().getComponent("model")); + queryBest.set("queryString", "best", registry); + registry.register(queryBest); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?query=overrides&model=querybest", Method.GET), cRegistry.getComponent("root")); + assertEquals("overrides", query.properties().get("model.queryString")); + assertEquals("overrides", query.getModel().getQueryTree().toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java new file mode 100644 index 00000000000..684e6072c5d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; + +/** + * @author bratseth + */ +public class QueryProfileCloneMicroBenchmark { + + private final String description; + private final int propertyCount; + private final String propertyPrefix; + private final boolean useDimensions; + + public QueryProfileCloneMicroBenchmark(String description, int propertyCount, String propertyPrefix, boolean useDimensions) { + this.description=description; + this.propertyCount=propertyCount; + this.propertyPrefix=propertyPrefix; + this.useDimensions=useDimensions; + } + + public void benchmark(int clone) { + cloneQueryWithProfile(10000); // warm-up + System.out.println(description); + long startTime=System.currentTimeMillis(); + cloneQueryWithProfile(clone); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float)totalTime/clone + " ms per clone)")); + } + + private void cloneQueryWithProfile(int clones) { + QueryProfile main = new QueryProfile("main"); + main.set("a", "value1", (QueryProfileRegistry)null); + main.set("b", "value2", useDimensions ? new String[] {"x1"} : null, null); + main.set("c", "value3", useDimensions ? new String[] {"x1","y2"} : null, null); + main.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&x=1&y=2", Method.GET), main.compile(null)); + setValues(query); + for (int i=0; i=0"); + this.dotDepth=dotDepth; + if (variantCount<1) throw new IllegalArgumentException("variantCount must be >0"); + this.variantCount=variantCount; + if (variantParentCount<0) throw new IllegalArgumentException("varientParentCount must be >=0"); + this.variantParentCount=variantParentCount; + } + + public void benchmark(int count, boolean useVariant) { + QueryProfile main=createProfile(useVariant); + Query query = new Query(QueryTestCase.httpEncode("?query=test&x=1&y=2"), main.compile(null)); + getValues(100000,query); // warm-up + System.out.print(this + ": "); + long startTime=System.currentTimeMillis(); + getValues(count,query); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float) totalTime * 1000 / (count * 2) + " microsecond per get)")); // *2 because we do 2 gets + } + + private QueryProfile createProfile(boolean useVariant) { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"d0"}); + String prefix=generatePrefix(); + for (int i=0; i dimensionValues=createDimensionValueMap(); + String prefix=generatePrefix(); + final int dotInterval=1000000; + final CompoundName found = new CompoundName(prefix + "a"); + final CompoundName notFound = new CompoundName(prefix + "nonexisting"); + for (int i=0; idotInterval && i%(dotInterval)==0) + System.out.print("."); + if (null==query.properties().get(found,dimensionValues)) // request the last variant for worst case + throw new RuntimeException("Expected value"); + if (null!=query.properties().get(notFound,dimensionValues)) // request the last variant for worst case + throw new RuntimeException("Did not expect value"); + } + } + + private Map createDimensionValueMap() { + Map dimensionValueMap=new HashMap<>(); + dimensionValueMap.put("d0","dv" + (variantCount-1)); + return dimensionValueMap; + } + + private String generatePrefix() { + StringBuilder b=new StringBuilder(); + for (int i=0; idotInterval && i%(count/dotInterval)==0) + System.out.print("."); + if (null==query.properties().get(found)) + throw new RuntimeException("Expected value"); + if (null!=query.properties().get(notFound)) + throw new RuntimeException("Expected no value"); + } + } + + public static void main(String[] args) { + int count=10000000; + new QueryProfileGetMicroBenchmark("Getting values in root, no dimensions ","",false).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 1-level nested profiles, no dimensions ","a.",false).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 2-level nested profiles, no dimensions ","a.b.",false).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in root, with dimensions ","",true).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 1-level nested profiles, with dimensions","a.",true).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 2-level nested profiles, with dimensions","a.b.",true).benchmark(count); + System.out.println(""); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java new file mode 100644 index 00000000000..cca4a2833d0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileListPropertiesMicroBenchmark { + + private final String description; + private final String propertyPrefix; + private final boolean useDimensions; + + public QueryProfileListPropertiesMicroBenchmark(String description, String propertyPrefix, boolean useDimensions) { + this.description=description; + this.propertyPrefix=propertyPrefix; + this.useDimensions=useDimensions; + } + + public void benchmark(int count) { + Query query=createQuery(); + listValues(10000, query); // warm-up + System.out.println(description); + long startTime=System.currentTimeMillis(); + listValues(count, query); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float)totalTime*1000/(count) + " microsecond per listProperties)")); + } + + private Query createQuery() { + Map dimensions=null; + if (useDimensions) { + dimensions=new HashMap<>(); + dimensions.put("x","1"); + dimensions.put("y","2"); + } + + QueryProfile main=new QueryProfile("main"); + setValues(10,main,dimensions); + QueryProfile parent=new QueryProfile("parent"); + setValues(5,main,dimensions); + main.addInherited(parent); + main.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&x=1&y=2", Method.GET), main.compile(null)); + return query; + } + + private void setValues(int count,QueryProfile profile,Map dimensions) { + for (int i=0; idotInterval && i%(count/dotInterval)==0) + System.out.print("."); + Map properties = query.properties().listProperties(propertyPrefix); + int expectedSize = 10 + (propertyPrefix.isEmpty() ? 3 : 0); // 3 extra properties on the root + if ( properties.size() != expectedSize ) + throw new RuntimeException("Expected a map of 10 elements, but got " + expectedSize + ": \n" + toString(properties)); + } + } + + private String toString(Map map) { + StringBuilder b=new StringBuilder(); + for (Map.Entry entry : map.entrySet()) + b.append(" ") + .append(entry.getKey()) + .append(" = ") + .append(entry.getValue().toString()) + .append("\n"); + return b.toString(); + } + + public static void main(String[] args) { + int count=1000000; + new QueryProfileListPropertiesMicroBenchmark("Listing values in root, no dimensions ","",false).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 1-level nested profiles, no dimensions ","a",false).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 2-level nested profiles, no dimensions ","a.b",false).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in root, with dimensions ","",true).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 1-level nested profiles, with dimensions","a",true).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 2-level nested profiles, with dimensions","a.b",true).benchmark(count); + System.out.println(""); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java new file mode 100644 index 00000000000..27f48e3ff6d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.processing.request.Properties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +/** + * @author bratseth + */ +public class QueryProfileSubstitutionTestCase extends junit.framework.TestCase { + + public void testSingleSubstitution() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world", "world", (QueryProfileRegistry)null); + assertEquals("Hello world!",p.compile(null).get("message")); + + QueryProfile p2=new QueryProfile("test2"); + p2.addInherited(p); + p2.set("world", "universe", (QueryProfileRegistry)null); + assertEquals("Hello universe!",p2.compile(null).get("message")); + } + + public void testMultipleSubstitutions() { + QueryProfile p=new QueryProfile("test"); + p.set("message","%{greeting} %{entity}%{exclamation}", (QueryProfileRegistry)null); + p.set("greeting","Hola", (QueryProfileRegistry)null); + p.set("entity","local group", (QueryProfileRegistry)null); + p.set("exclamation","?", (QueryProfileRegistry)null); + assertEquals("Hola local group?",p.compile(null).get("message")); + + QueryProfile p2=new QueryProfile("test2"); + p2.addInherited(p); + p2.set("entity","milky way", (QueryProfileRegistry)null); + assertEquals("Hola milky way?",p2.compile(null).get("message")); + } + + public void testUnclosedSubstitution1() { + try { + QueryProfile p=new QueryProfile("test"); + p.set("message1","%{greeting} %{entity}%{exclamation", (QueryProfileRegistry)null); + fail("Should have produced an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'message1' to '%{greeting} %{entity}%{exclamation': Unterminated value substitution '%{exclamation'", + Exceptions.toMessageString(e)); + } + } + + public void testUnclosedSubstitution2() { + try { + QueryProfile p=new QueryProfile("test"); + p.set("message1","%{greeting} %{entity%{exclamation}", (QueryProfileRegistry)null); + fail("Should have produced an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'message1' to '%{greeting} %{entity%{exclamation}': Unterminated value substitution '%{entity%{exclamation}'", + Exceptions.toMessageString(e)); + } + } + + public void testNullSubstitution() { + QueryProfile p=new QueryProfile("test"); + p.set("message","%{greeting} %{entity}%{exclamation}", (QueryProfileRegistry)null); + p.set("greeting","Hola", (QueryProfileRegistry)null); + assertEquals("Hola ", p.compile(null).get("message")); + + QueryProfile p2=new QueryProfile("test2"); + p2.addInherited(p); + p2.set("greeting","Hola", (QueryProfileRegistry)null); + p2.set("exclamation", "?", (QueryProfileRegistry)null); + assertEquals("Hola ?",p2.compile(null).get("message")); + } + + public void testNoOverridingOfPropertiesSetAtRuntime() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","world", (QueryProfileRegistry)null); + p.freeze(); + + Properties runtime=new QueryProfileProperties(p.compile(null)); + runtime.set("runtimeMessage","Hello %{world}!"); + assertEquals("Hello world!", runtime.get("message")); + assertEquals("Hello %{world}!",runtime.get("runtimeMessage")); + } + + public void testButPropertiesSetAtRuntimeAreUsedInSubstitutions() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","world", (QueryProfileRegistry)null); + + Properties runtime=new QueryProfileProperties(p.compile(null)); + runtime.set("world","Earth"); + assertEquals("Hello Earth!",runtime.get("message")); + } + + public void testInspection() { + QueryProfile p=new QueryProfile("test"); + p.set("message", "%{greeting} %{entity}%{exclamation}", (QueryProfileRegistry)null); + assertEquals("message","%{greeting} %{entity}%{exclamation}", + p.declaredContent().entrySet().iterator().next().getValue().toString()); + } + + public void testVariants() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","world", (QueryProfileRegistry)null); + p.setDimensions(new String[] {"x"}); + p.set("message","Halo %{world}!",new String[] {"x1"}, null); + p.set("world","Europe",new String[] {"x2"}, null); + + CompiledQueryProfile cp = p.compile(null); + assertEquals("Hello world!", cp.get("message", QueryProfileVariantsTestCase.toMap("x=x?"))); + assertEquals("Halo world!", cp.get("message", QueryProfileVariantsTestCase.toMap("x=x1"))); + assertEquals("Hello Europe!", cp.get("message", QueryProfileVariantsTestCase.toMap("x=x2"))); + } + + public void testRecursion() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","sol planet number %{number}", (QueryProfileRegistry)null); + p.set("number",3, (QueryProfileRegistry)null); + assertEquals("Hello sol planet number 3!",p.compile(null).get("message")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java new file mode 100644 index 00000000000..ba066367fb0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -0,0 +1,562 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +/** + * Tests untyped query profiles + * + * @author bratseth + */ +public class QueryProfileTestCase extends junit.framework.TestCase { + + public void testBasics() { + QueryProfile profile=new QueryProfile("test"); + profile.set("a","a-value", (QueryProfileRegistry)null); + profile.set("b.c","b.c-value", (QueryProfileRegistry)null); + profile.set("d.e.f","d.e.f-value", (QueryProfileRegistry)null); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("a-value",cprofile.get("a")); + assertEquals("b.c-value",cprofile.get("b.c")); + assertEquals("d.e.f-value",cprofile.get("d.e.f")); + + assertNull(cprofile.get("nonexistent")); + assertNull(cprofile.get("nested.nonexistent")); + + assertTrue(profile.lookup("b",null).getClass()==QueryProfile.class); + assertTrue(profile.lookup("b",null).getClass()==QueryProfile.class); + } + + /** Tests cloning, with wrappers used in production in place */ + public void testCloning() { + QueryProfile classProfile=new QueryProfile("test"); + classProfile.set("a","aValue", (QueryProfileRegistry)null); + classProfile.set("b",3, (QueryProfileRegistry)null); + + Properties properties = new QueryProfileProperties(classProfile.compile(null)); + + Properties propertiesClone=properties.clone(); + assertEquals("aValue",propertiesClone.get("a")); + assertEquals(3,propertiesClone.get("b")); + properties.set("a","aNewValue"); + assertEquals("aNewValue",properties.get("a")); + assertEquals("aValue",propertiesClone.get("a")); + } + + public void testFreezing() { + QueryProfile profile=new QueryProfile("test"); + profile.set("a","a-value", (QueryProfileRegistry)null); + profile.set("b.c","b.c-value", (QueryProfileRegistry)null); + profile.set("d.e.f","d.e.f-value", (QueryProfileRegistry)null); + + assertFalse(profile.isFrozen()); + assertEquals("a-value",profile.get("a")); + + profile.freeze(); + + assertTrue(profile.isFrozen()); + assertTrue(((QueryProfile)profile.lookup("b",null)).isFrozen()); + assertTrue(((QueryProfile)profile.lookup("d.e",null)).isFrozen()); + + try { + profile.set("a","value", (QueryProfileRegistry)null); + fail("Expected exception"); + } + catch (IllegalStateException e) { + } + } + + private void assertSameObjects(CompiledQueryProfile profile, String path, List expectedKeys) { + Map subObjects = profile.listValues(path); + assertEquals("Sub-objects list equal for path " + path, new HashSet<>(expectedKeys), subObjects.keySet()); + for(String key : expectedKeys) { + assertEquals("Equal for key " + key, profile.get(key),subObjects.get(path + "." + key)); + } + + } + + public void testGetSubObjects() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + barn.addInherited(mor); + barn.addInherited(far); + mormor.set("a.mormor","a.mormor", (QueryProfileRegistry)null); + barn.set("a.barn","a.barn", (QueryProfileRegistry)null); + mor.set("b.mor", "b.mor", (QueryProfileRegistry)null); + far.set("b.far", "b.far", (QueryProfileRegistry)null); + far.set("a.far","a.far", (QueryProfileRegistry)null); + CompiledQueryProfile cbarn = barn.compile(null); + + assertSameObjects(cbarn, "a", Arrays.asList("mormor","far","barn")); + + assertEquals("b.mor", cbarn.get("b.mor")); + assertEquals("b.far", cbarn.get("b.far")); + } + + public void testInheritance() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("a","morfar-a", (QueryProfileRegistry)null); + mormor.set("a","mormor-a", (QueryProfileRegistry)null); + farfar.set("a","farfar-a", (QueryProfileRegistry)null); + mor.set("a","mor-a", (QueryProfileRegistry)null); + far.set("a","far-a", (QueryProfileRegistry)null); + barn.set("a","barn-a", (QueryProfileRegistry)null); + + mormor.set("b","mormor-b", (QueryProfileRegistry)null); + far.set("b","far-b", (QueryProfileRegistry)null); + + mor.set("c","mor-c", (QueryProfileRegistry)null); + far.set("c","far-c", (QueryProfileRegistry)null); + + mor.set("d.a","mor-d.a", (QueryProfileRegistry)null); + barn.set("d.b","barn-d.b", (QueryProfileRegistry)null); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn",barn, (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + assertEquals("barn-a", cbarn.get("a")); + assertEquals("mormor-b", cbarn.get("b")); + assertEquals("mor-c", cbarn.get("c")); + + assertEquals("barn-a", cannetBarn.get("venn.a")); + assertEquals("mormor-b", cannetBarn.get("venn.b")); + assertEquals("mor-c", cannetBarn.get("venn.c")); + + assertEquals("barn-d.b", cbarn.get("d.b")); + assertEquals("mor-d.a", cbarn.get("d.a")); + } + + public void testInheritance2Level() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("a.x","morfar-a", (QueryProfileRegistry)null); + mormor.set("a.x","mormor-a", (QueryProfileRegistry)null); + farfar.set("a.x","farfar-a", (QueryProfileRegistry)null); + mor.set("a.x","mor-a", (QueryProfileRegistry)null); + far.set("a.x","far-a", (QueryProfileRegistry)null); + barn.set("a.x","barn-a", (QueryProfileRegistry)null); + + mormor.set("b.x","mormor-b", (QueryProfileRegistry)null); + far.set("b.x","far-b", (QueryProfileRegistry)null); + + mor.set("c.x","mor-c", (QueryProfileRegistry)null); + far.set("c.x","far-c", (QueryProfileRegistry)null); + + mor.set("d.a.x","mor-d.a", (QueryProfileRegistry)null); + barn.set("d.b.x","barn-d.b", (QueryProfileRegistry)null); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn",barn, (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + assertEquals("barn-a", cbarn.get("a.x")); + assertEquals("mormor-b", cbarn.get("b.x")); + assertEquals("mor-c", cbarn.get("c.x")); + + assertEquals("barn-a", cannetBarn.get("venn.a.x")); + assertEquals("mormor-b", cannetBarn.get("venn.b.x")); + assertEquals("mor-c", cannetBarn.get("venn.c.x")); + + assertEquals("barn-d.b", cbarn.get("d.b.x")); + assertEquals("mor-d.a", cbarn.get("d.a.x")); + } + + public void testInheritance3Level() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("y.a.x","morfar-a", (QueryProfileRegistry)null); + mormor.set("y.a.x","mormor-a", (QueryProfileRegistry)null); + farfar.set("y.a.x","farfar-a", (QueryProfileRegistry)null); + mor.set("y.a.x","mor-a", (QueryProfileRegistry)null); + far.set("y.a.x","far-a", (QueryProfileRegistry)null); + barn.set("y.a.x","barn-a", (QueryProfileRegistry)null); + + mormor.set("y.b.x","mormor-b", (QueryProfileRegistry)null); + far.set("y.b.x","far-b", (QueryProfileRegistry)null); + + mor.set("y.c.x","mor-c", (QueryProfileRegistry)null); + far.set("y.c.x","far-c", (QueryProfileRegistry)null); + + mor.set("y.d.a.x","mor-d.a", (QueryProfileRegistry)null); + barn.set("y.d.b.x","barn-d.b", (QueryProfileRegistry)null); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn",barn, (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + assertEquals("barn-a", cbarn.get("y.a.x")); + assertEquals("mormor-b", cbarn.get("y.b.x")); + assertEquals("mor-c", cbarn.get("y.c.x")); + + assertEquals("barn-a", cannetBarn.get("venn.y.a.x")); + assertEquals("mormor-b", cannetBarn.get("venn.y.b.x")); + assertEquals("mor-c", cannetBarn.get("venn.y.c.x")); + + assertEquals("barn-d.b", cbarn.get("y.d.b.x")); + assertEquals("mor-d.a", cbarn.get("y.d.a.x")); + } + + public void testListProperties() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("a","morfar-a", (QueryProfileRegistry)null); + morfar.set("model.b","morfar-model.b", (QueryProfileRegistry)null); + mormor.set("a","mormor-a", (QueryProfileRegistry)null); + mormor.set("model.b","mormor-model.b", (QueryProfileRegistry)null); + farfar.set("a","farfar-a", (QueryProfileRegistry)null); + mor.set("a","mor-a", (QueryProfileRegistry)null); + far.set("a","far-a", (QueryProfileRegistry)null); + barn.set("a","barn-a", (QueryProfileRegistry)null); + mormor.set("b","mormor-b", (QueryProfileRegistry)null); + far.set("b","far-b", (QueryProfileRegistry)null); + mor.set("c","mor-c", (QueryProfileRegistry)null); + far.set("c","far-c", (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + + QueryProfileProperties properties = new QueryProfileProperties(cbarn); + + assertEquals("barn-a", cbarn.get("a")); + assertEquals("mormor-b", cbarn.get("b")); + + Map rootMap = properties.listProperties(); + assertEquals("barn-a", rootMap.get("a")); + assertEquals("mormor-b", rootMap.get("b")); + assertEquals("mor-c", rootMap.get("c")); + + Map modelMap = properties.listProperties("model"); + assertEquals("mormor-model.b", modelMap.get("b")); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn", barn, (QueryProfileRegistry)null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + Map annetBarnMap = new QueryProfileProperties(cannetBarn).listProperties(); + assertEquals("barn-a", annetBarnMap.get("venn.a")); + assertEquals("mormor-b", annetBarnMap.get("venn.b")); + assertEquals("mor-c", annetBarnMap.get("venn.c")); + assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b")); + } + + /** Tests that dots are followed when setting overridability */ + public void testInstanceOverridable() { + QueryProfile profile=new QueryProfile("root/unoverridableIndex"); + profile.set("model.defaultIndex","default", (QueryProfileRegistry)null); + profile.setOverridable("model.defaultIndex",false,null); + + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue()); + + // Parameters should be ignored + Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + + // Parameters should be ignored + query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title&model.language=de", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + assertEquals("de",query.getModel().getLanguage().languageCode()); + } + + /** Tests that dots are followed when setting overridability...also with variants */ + public void testInstanceOverridableWithVariants() { + QueryProfile profile=new QueryProfile("root/unoverridableIndex"); + profile.setDimensions(new String[] {"x"}); + profile.set("model.defaultIndex","default", (QueryProfileRegistry)null); + profile.setOverridable("model.defaultIndex",false,null); + + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null)); + + // Parameters should be ignored + Query query = new Query(HttpRequest.createTestRequest("?x=x1&model.defaultIndex=title", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + + // Parameters should be ignored + query = new Query(HttpRequest.createTestRequest("?x=x1&model.default-index=title&model.language=de", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + assertEquals("de",query.getModel().getLanguage().languageCode()); + } + + public void testSimpleInstanceOverridableWithVariants1() { + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a","original", (QueryProfileRegistry)null); + profile.setOverridable("a",false,null); + + assertFalse(profile.isDeclaredOverridable("a",null)); + + Query query = new Query(HttpRequest.createTestRequest("?x=x1&a=overridden", Method.GET), profile.compile(null)); + assertEquals("original",query.properties().get("a")); + } + + public void testSimpleInstanceOverridableWithVariants2() { + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a","original",new String[] {"x1"}, null); + profile.setOverridable("a",false,null); + + assertFalse(profile.isDeclaredOverridable("a",null)); + + Query query = new Query(HttpRequest.createTestRequest("?x=x1&a=overridden", Method.GET), profile.compile(null)); + assertEquals("original",query.properties().get("a")); + } + + /** Tests having both an explicit reference and an override */ + public void testExplicitReferenceOverride() { + QueryProfile a1=new QueryProfile("a1"); + a1.set("b","a1.b", (QueryProfileRegistry)null); + QueryProfile profile=new QueryProfile("test"); + profile.set("a",a1, (QueryProfileRegistry)null); + profile.set("a.b","a.b", (QueryProfileRegistry)null); + assertEquals("a.b",profile.compile(null).get("a.b")); + } + + public void testSettingNonLeaf1() { + QueryProfile p=new QueryProfile("test"); + p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + assertEquals("a-value", cp.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + } + + public void testSettingNonLeaf2() { + QueryProfile p=new QueryProfile("test"); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a","a-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + assertEquals("a-value", cp.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + } + + public void testSettingNonLeaf3a() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a","a-value",new String[] {"x1"}, null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertNull(p.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + assertEquals("a-value", cp.get("a", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + assertEquals("a.b-value", cp.get("a.b", new String[] {"x1"})); + } + + public void testSettingNonLeaf3b() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a","a-value",new String[] {"x1"}, null); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertNull(cp.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + assertEquals("a-value", cp.get("a", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + assertEquals("a.b-value", cp.get("a.b",new String[] {"x1"})); + } + + public void testSettingNonLeaf4a() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a.b","a.b-value",new String[] {"x1"}, null); + p.set("a","a-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertEquals("a-value", cp.get("a")); + assertNull(cp.get("a.b")); + assertEquals("a-value", cp.get("a",new String[] {"x1"})); + assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + } + + public void testSettingNonLeaf4b() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value",new String[] {"x1"}, null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertEquals("a-value", cp.get("a")); + assertNull(cp.get("a.b")); + assertEquals("a-value", cp.get("a",new String[] {"x1"})); + assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + } + + public void testSettingNonLeaf5() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a.b","a.b-value",new String[] {"x1"}, null); + p.set("a","a-value",new String[] {"x1"}, null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertNull(cp.get("a")); + assertNull(cp.get("a.b")); + assertEquals("a-value", cp.get("a", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + } + + public void testListingWithNonLeafs() { + QueryProfile p=new QueryProfile("test"); + p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + Map values = p.compile(null).listValues("a"); + assertEquals(1,values.size()); + assertEquals("a.b-value",values.get("b")); + } + + public void testRankTypeNames() { + QueryProfile p=new QueryProfile("test"); + p.set("a.$b","foo", (QueryProfileRegistry)null); + p.set("a.query(b)","bar", (QueryProfileRegistry)null); + p.set("a.b.default-index","fuu", (QueryProfileRegistry)null); + CompiledQueryProfile cp = p.compile(null); + + assertEquals("foo", cp.get("a.$b")); + assertEquals("bar", cp.get("a.query(b)")); + assertEquals("fuu", cp.get("a.b.default-index")); + + Map p1 = cp.listValues(""); + assertEquals("foo", p1.get("a.$b")); + assertEquals("bar", p1.get("a.query(b)")); + assertEquals("fuu", p1.get("a.b.default-index")); + + Map p2 = cp.listValues("a"); + assertEquals("foo", p2.get("$b")); + assertEquals("bar", p2.get("query(b)")); + assertEquals("fuu", p2.get("b.default-index")); + } + + public void testQueryProfileInlineValueReassignment() { + QueryProfile p=new QueryProfile("test"); + p.set("source.rel.params.query","%{model.queryString}", (QueryProfileRegistry)null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=foo", Method.GET), p.compile(null)); + assertEquals("foo",q.properties().get("source.rel.params.query")); + assertEquals("foo",q.properties().listProperties().get("source.rel.params.query")); + q.getModel().setQueryString("bar"); + assertEquals("bar",q.properties().get("source.rel.params.query")); + assertEquals("foo",q.properties().listProperties().get("source.rel.params.query")); // Is still foo because model variables are not supported with the list function + } + + public void testQueryProfileInlineValueReassignmentSimpleName() { + QueryProfile p=new QueryProfile("test"); + p.set("key","%{model.queryString}", (QueryProfileRegistry)null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=foo", Method.GET), p.compile(null)); + assertEquals("foo",q.properties().get("key")); + assertEquals("foo",q.properties().listProperties().get("key")); + q.getModel().setQueryString("bar"); + assertEquals("bar",q.properties().get("key")); + assertEquals("foo",q.properties().listProperties().get("key")); // Is still bar because model variables are not supported with the list function + } + + public void testQueryProfileInlineValueReassignmentSimpleNameGenericProperty() { + QueryProfile p=new QueryProfile("test"); + p.set("key","%{value}", (QueryProfileRegistry)null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=test&value=foo", Method.GET), p.compile(null)); + assertEquals("foo",q.properties().get("key")); + assertEquals("foo",q.properties().listProperties().get("key")); + q.properties().set("value","bar"); + assertEquals("bar",q.properties().get("key")); + assertEquals("bar",q.properties().listProperties().get("key")); + } + + public void testQueryProfileModelValueListing() { + QueryProfile p=new QueryProfile("test"); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=bar", Method.GET), p.compile(null)); + assertEquals("bar",q.properties().get("model.queryString")); + assertEquals("bar",q.properties().listProperties().get("model.queryString")); + q.getModel().setQueryString("baz"); + assertEquals("baz",q.properties().get("model.queryString")); + assertEquals("bar",q.properties().listProperties().get("model.queryString")); // Is still bar because model variables are not supported with the list function + } + + public void testEmptyBoolean() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x","y"}); + p.set("clustering.something","bar", (QueryProfileRegistry)null); + p.set("clustering.something","bar",new String[] {"x1","y1"}, null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?x=x1&y=y1&query=bar&clustering.timeline.kano=tur&" + + "clustering.enable=true&clustering.timeline.bucketspec=-" + + "7d/3h&clustering.timeline.tophit=false&clustering.timeli" + + "ne=true", Method.GET),p.compile(null)); + assertEquals(true,q.properties().getBoolean("clustering.timeline",false)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java new file mode 100644 index 00000000000..de8f9dab985 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + + +import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author tonytv + */ +public class QueryProfileVariantsCloneTestCase { + + /** + * Test for Ticket 4882480. + */ + @Test + public void test_that_interior_and_leaf_values_on_a_path_are_preserved_when_cloning() { + Map dimensionBinding = createDimensionBinding("location", "norway"); + + QueryProfile profile = new QueryProfile("profile"); + profile.setDimensions(keys(dimensionBinding)); + + DimensionValues dimensionValues = DimensionValues.createFrom(values(dimensionBinding)); + profile.set("interior.leaf", "leafValue", dimensionValues, null); + profile.set("interior", "interiorValue", dimensionValues, null); + + CompiledQueryProfile clone = profile.compile(null).clone(); + + assertEquals(profile.get("interior", dimensionBinding, null), + clone.get("interior", dimensionBinding)); + + assertEquals(profile.get("interior.leaf", dimensionBinding, null), + clone.get("interior.leaf", dimensionBinding)); + } + + + private static Map createDimensionBinding(String dimension, String value) { + Map dimensionBinding = new HashMap<>(); + dimensionBinding.put(dimension, value); + return Collections.unmodifiableMap(dimensionBinding); + } + + private static String[] keys(Map map) { + return map.keySet().toArray(new String[0]); + } + + private static String[] values(Map map) { + return map.values().toArray(new String[0]); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java new file mode 100644 index 00000000000..95f121adab9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -0,0 +1,1052 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.Properties; +import com.yahoo.search.query.profile.BackedOverridableQueryProfile; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileVariantsTestCase extends junit.framework.TestCase { + + public void testSimple() { + QueryProfile profile=new QueryProfile("a"); + profile.set("a","a.deflt", (QueryProfileRegistry)null); + profile.setDimensions(new String[] {"x","y","z"}); + profile.set("a","a.1.*.*",new String[] {"x1",null,null}, null); + profile.set("a","a.1.*.1",new String[] {"x1",null,"z1"}, null); + profile.set("a","a.1.*.5",new String[] {"x1",null,"z5"}, null); + profile.set("a","a.1.1.*",new String[] {"x1","y1",null}, null); + profile.set("a","a.1.5.*",new String[] {"x1","y5",null}, null); + profile.set("a","a.1.1.1",new String[] {"x1","y1","z1"}, null); + profile.set("a","a.2.1.1",new String[] {"x2","y1","z1"}, null); + profile.set("a","a.1.2.2",new String[] {"x1","y2","z2"}, null); + profile.set("a","a.1.2.3",new String[] {"x1","y2","z3"}, null); + profile.set("a","a.2.*.*",new String[] {"x2" }, null); // Same as ,null,null + CompiledQueryProfile cprofile = profile.compile(null); + + // Perfect matches + assertGet("a.deflt","a",new String[] {null,null,null}, profile, cprofile); + assertGet("a.1.*.*","a",new String[] {"x1",null,null}, profile, cprofile); + assertGet("a.1.1.*","a",new String[] {"x1","y1",null}, profile, cprofile); + assertGet("a.1.5.*","a",new String[] {"x1","y5",null}, profile, cprofile); + assertGet("a.1.*.1","a",new String[] {"x1",null,"z1"}, profile, cprofile); + assertGet("a.1.*.5","a",new String[] {"x1",null,"z5"}, profile, cprofile); + assertGet("a.1.1.1","a",new String[] {"x1","y1","z1"}, profile, cprofile); + assertGet("a.2.1.1","a",new String[] {"x2","y1","z1"}, profile, cprofile); + assertGet("a.1.2.2","a",new String[] {"x1","y2","z2"}, profile, cprofile); + assertGet("a.1.2.3","a",new String[] {"x1","y2","z3"}, profile, cprofile); + assertGet("a.2.*.*","a",new String[] {"x2",null,null}, profile, cprofile); + + // Wildcard matches + assertGet("a.deflt","a",new String[] {"x?","y?","z?"}, profile, cprofile); + assertGet("a.deflt","a",new String[] {"x?","y1","z1"}, profile, cprofile); + assertGet("a.1.*.*","a",new String[] {"x1","y?","z?"}, profile, cprofile); + assertGet("a.1.*.*","a",new String[] {"x1","y?","z?"}, profile, cprofile); + assertGet("a.1.1.*","a",new String[] {"x1","y1","z?"}, profile, cprofile); + assertGet("a.1.*.1","a",new String[] {"x1","y?","z1"}, profile, cprofile); + assertGet("a.1.5.*","a",new String[] {"x1","y5","z?"}, profile, cprofile); + assertGet("a.1.*.5","a",new String[] {"x1","y?","z5"}, profile, cprofile); + assertGet("a.1.5.*","a",new String[] {"x1","y5","z5"}, profile, cprofile); // Left dimension gets precedence + assertGet("a.2.*.*","a",new String[] {"x2","y?","z?"}, profile, cprofile); + } + + public void testVariantsOfInlineCompound() { + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a.b","a.b", (QueryProfileRegistry)null); + profile.set("a.b","a.b.x1",new String[] {"x1"}, null); + profile.set("a.b","a.b.x2",new String[] {"x2"}, null); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("a.b",cprofile.get("a.b")); + assertEquals("a.b.x1",cprofile.get("a.b", toMap("x=x1"))); + assertEquals("a.b.x2",cprofile.get("a.b", toMap("x=x2"))); + } + + public void testVariantsOfExplicitCompound() { + QueryProfile a1=new QueryProfile("a1"); + a1.set("b","a.b", (QueryProfileRegistry)null); + + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a",a1, (QueryProfileRegistry)null); + profile.set("a.b","a.b.x1",new String[] {"x1"}, null); + profile.set("a.b","a.b.x2",new String[] {"x2"}, null); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("a.b",cprofile.get("a.b")); + assertEquals("a.b.x1",cprofile.get("a.b", toMap("x=x1"))); + assertEquals("a.b.x2",cprofile.get("a.b", toMap("x=x2"))); + } + + public void testCompound() { + // Configuration phase + + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x","y"}); + + QueryProfile a1=new QueryProfile("a1"); + a1.set("b","a1.b.default", (QueryProfileRegistry)null); + a1.set("c","a1.c.default", (QueryProfileRegistry)null); + a1.set("d","a1.d.default", (QueryProfileRegistry)null); + a1.set("e","a1.e.default", (QueryProfileRegistry)null); + + QueryProfile a2=new QueryProfile("a2"); + a2.set("b","a2.b.default", (QueryProfileRegistry)null); + a2.set("c","a2.c.default", (QueryProfileRegistry)null); + a2.set("d","a2.d.default", (QueryProfileRegistry)null); + a2.set("e","a2.e.default", (QueryProfileRegistry)null); + + profile.set("a",a1, (QueryProfileRegistry)null); // Must set profile references before overrides + profile.set("a.b","a.b.default-override", (QueryProfileRegistry)null); + profile.set("a.c","a.c.default-override", (QueryProfileRegistry)null); + profile.set("a.d","a.d.default-override", (QueryProfileRegistry)null); + profile.set("a.g","a.g.default-override", (QueryProfileRegistry)null); + + String[] d1=new String[] { "x1","y1" }; + profile.set("a",a1,d1, null); + profile.set("a.b","x1.y1.a.b.default-override",d1, null); + profile.set("a.c","x1.y1.a.c.default-override",d1, null); + profile.set("a.g","x1.y1.a.g.default-override",d1, null); // This value is never manifest because the runtime override overrides all variants + + String[] d2=new String[] { "x1","y2" }; + profile.set("a.b","x1.y2.a.b.default-override",d2, null); + profile.set("a.c","x1.y2.a.c.default-override",d2, null); + + String[] d3=new String[] { "x2","y1" }; + profile.set("a",a2,d3, null); + profile.set("a.b","x2.y1.a.b.default-override",d3, null); + profile.set("a.c","x2.y1.a.c.default-override",d3, null); + + + // Runtime phase - four simultaneous requests using different variants makes their own overrides + QueryProfileProperties defaultRuntimeProfile = new QueryProfileProperties(profile.compile(null)); + defaultRuntimeProfile.set("a.f", "a.f.runtime-override"); + defaultRuntimeProfile.set("a.g", "a.g.runtime-override"); + + QueryProfileProperties d1RuntimeProfile = new QueryProfileProperties(profile.compile(null)); + d1RuntimeProfile.set("a.f", "a.f.d1.runtime-override", toMap("x=x1", "y=y1")); + d1RuntimeProfile.set("a.g", "a.g.d1.runtime-override", toMap("x=x1", "y=y1")); + + QueryProfileProperties d2RuntimeProfile = new QueryProfileProperties(profile.compile(null)); + d2RuntimeProfile.set("a.f", "a.f.d2.runtime-override",toMap("x=x1", "y=y2")); + d2RuntimeProfile.set("a.g", "a.g.d2.runtime-override",toMap("x=x1", "y=y2")); + + QueryProfileProperties d3RuntimeProfile = new QueryProfileProperties(profile.compile(null)); + d3RuntimeProfile.set("a.f", "a.f.d3.runtime-override", toMap("x=x2", "y=y1")); + d3RuntimeProfile.set("a.g", "a.g.d3.runtime-override", toMap("x=x2", "y=y1")); + + // Lookups + assertEquals("a.b.default-override", defaultRuntimeProfile.get("a.b")); + assertEquals("a.c.default-override", defaultRuntimeProfile.get("a.c")); + assertEquals("a.d.default-override", defaultRuntimeProfile.get("a.d")); + assertEquals("a1.e.default", defaultRuntimeProfile.get("a.e")); + assertEquals("a.f.runtime-override", defaultRuntimeProfile.get("a.f")); + assertEquals("a.g.runtime-override", defaultRuntimeProfile.get("a.g")); + + assertEquals("x1.y1.a.b.default-override", d1RuntimeProfile.get("a.b", toMap("x=x1", "y=y1"))); + assertEquals("x1.y1.a.c.default-override", d1RuntimeProfile.get("a.c", toMap("x=x1", "y=y1"))); + assertEquals("a1.d.default", d1RuntimeProfile.get("a.d", toMap("x=x1", "y=y1"))); + assertEquals("a1.e.default", d1RuntimeProfile.get("a.e", toMap("x=x1", "y=y1"))); + assertEquals("a.f.d1.runtime-override", d1RuntimeProfile.get("a.f", toMap("x=x1", "y=y1"))); + assertEquals("a.g.d1.runtime-override", d1RuntimeProfile.get("a.g", toMap("x=x1", "y=y1"))); + + assertEquals("x1.y2.a.b.default-override", d2RuntimeProfile.get("a.b", toMap("x=x1", "y=y2"))); + assertEquals("x1.y2.a.c.default-override", d2RuntimeProfile.get("a.c", toMap("x=x1", "y=y2"))); + assertEquals("a.d.default-override", d2RuntimeProfile.get("a.d", toMap("x=x1", "y=y2"))); // Because this variant does not itself refer to a + assertEquals("a1.e.default", d2RuntimeProfile.get("a.e", toMap("x=x1", "y=y2"))); + assertEquals("a.f.d2.runtime-override", d2RuntimeProfile.get("a.f", toMap("x=x1", "y=y2"))); + assertEquals("a.g.d2.runtime-override", d2RuntimeProfile.get("a.g", toMap("x=x1", "y=y2"))); + + assertEquals("x2.y1.a.b.default-override", d3RuntimeProfile.get("a.b", toMap("x=x2", "y=y1"))); + assertEquals("x2.y1.a.c.default-override", d3RuntimeProfile.get("a.c", toMap("x=x2", "y=y1"))); + assertEquals("a2.d.default", d3RuntimeProfile.get("a.d", toMap("x=x2", "y=y1"))); + assertEquals("a2.e.default", d3RuntimeProfile.get("a.e", toMap("x=x2", "y=y1"))); + assertEquals("a.f.d3.runtime-override", d3RuntimeProfile.get("a.f", toMap("x=x2", "y=y1"))); + assertEquals("a.g.d3.runtime-override", d3RuntimeProfile.get("a.g", toMap("x=x2", "y=y1"))); + } + + public void testVariantNotInBase() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("InX1Only","x1",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("x1",ctest.get("InX1Only", toMap("x=x1"))); + assertEquals(null,ctest.get("InX1Only", toMap("x=x2"))); + assertEquals(null,ctest.get("InX1Only")); + } + + public void testVariantNotInBaseSpaceVariantValue() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("InX1Only","x1",new String[] {"x 1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals("x1",ctest.get("InX1Only", toMap("x=x 1"))); + assertEquals(null,ctest.get("InX1Only", toMap("x=x 2"))); + assertEquals(null,ctest.get("InX1Only")); + } + + public void testDimensionsInSuperType() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + QueryProfile child=new QueryProfile("child"); + child.addInherited(parent); + child.set("a","a.default", (QueryProfileRegistry)null); + child.set("a","a.x1.y1",new String[] {"x1","y1"}, null); + child.set("a","a.x1.y2",new String[] {"x1","y2"}, null); + + CompiledQueryProfile cchild = child.compile(null); + + assertEquals("a.default",cchild.get("a")); + assertEquals("a.x1.y1",cchild.get("a", toMap("x=x1","y=y1"))); + assertEquals("a.x1.y2",cchild.get("a", toMap("x=x1","y=y2"))); + } + + public void testDimensionsInSuperTypeRuntime() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + QueryProfile child=new QueryProfile("child"); + child.addInherited(parent); + child.set("a","a.default", (QueryProfileRegistry)null); + child.set("a", "a.x1.y1", new String[]{"x1", "y1"}, null); + child.set("a", "a.x1.y2", new String[]{"x1", "y2"}, null); + Properties overridable=new QueryProfileProperties(child.compile(null)); + + assertEquals("a.default", child.get("a")); + assertEquals("a.x1.y1", overridable.get("a", toMap("x=x1", "y=y1"))); + assertEquals("a.x1.y2", overridable.get("a", toMap("x=x1", "y=y2"))); + } + + public void testVariantsAreResolvedBeforeInheritance() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + parent.set("a","p.a.default", (QueryProfileRegistry)null); + parent.set("a","p.a.x1.y1",new String[] {"x1","y1"}, null); + parent.set("a","p.a.x1.y2",new String[] {"x1","y2"}, null); + parent.set("b","p.b.default", (QueryProfileRegistry)null); + parent.set("b","p.b.x1.y1",new String[] {"x1","y1"}, null); + parent.set("b","p.b.x1.y2",new String[] {"x1","y2"}, null); + QueryProfile child=new QueryProfile("child"); + child.setDimensions(new String[] {"x","y"}); + child.addInherited(parent); + child.set("a","c.a.default", (QueryProfileRegistry)null); + child.set("a","c.a.x1.y1",new String[] {"x1","y1"}, null); + + CompiledQueryProfile cchild = child.compile(null); + assertEquals("c.a.default",cchild.get("a")); + assertEquals("c.a.x1.y1",cchild.get("a", toMap("x=x1", "y=y1"))); + assertEquals("c.a.default",cchild.get("a", toMap("x=x1", "y=y2"))); + assertEquals("p.b.default",cchild.get("b")); + assertEquals("p.b.x1.y1",cchild.get("b", toMap("x=x1", "y=y1"))); + assertEquals("p.b.x1.y2",cchild.get("b", toMap("x=x1", "y=y2"))); + } + + public void testVariantsAreResolvedBeforeInheritanceSimplified() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + parent.set("a","p.a.x1.y2",new String[] {"x1","y2"}, null); + + QueryProfile child=new QueryProfile("child"); + child.setDimensions(new String[] {"x","y"}); + child.addInherited(parent); + child.set("a","c.a.default", (QueryProfileRegistry)null); + + assertEquals("c.a.default",child.compile(null).get("a", toMap("x=x1", "y=y2"))); + } + + public void testVariantInheritance() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x","y"}); + QueryProfile defaultParent=new QueryProfile("defaultParent"); + defaultParent.set("a","a-default", (QueryProfileRegistry)null); + QueryProfile x1Parent=new QueryProfile("x1Parent"); + x1Parent.set("a","a-x1", (QueryProfileRegistry)null); + x1Parent.set("d","d-x1", (QueryProfileRegistry)null); + x1Parent.set("e","e-x1", (QueryProfileRegistry)null); + QueryProfile x1y1Parent=new QueryProfile("x1y1Parent"); + x1y1Parent.set("a","a-x1y1", (QueryProfileRegistry)null); + QueryProfile x1y2Parent=new QueryProfile("x1y2Parent"); + x1y2Parent.set("a","a-x1y2", (QueryProfileRegistry)null); + x1y2Parent.set("b","b-x1y2", (QueryProfileRegistry)null); + x1y2Parent.set("c","c-x1y2", (QueryProfileRegistry)null); + test.addInherited(defaultParent); + test.addInherited(x1Parent,new String[] {"x1"}); + test.addInherited(x1y1Parent,new String[] {"x1","y1"}); + test.addInherited(x1y2Parent,new String[] {"x1","y2"}); + test.set("c","c-x1",new String[] {"x1"}, null); + test.set("e","e-x1y2",new String[] {"x1","y2"}, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals("a-default",ctest.get("a")); + assertEquals("a-x1",ctest.get("a", toMap("x=x1"))); + assertEquals("a-x1y1",ctest.get("a", toMap("x=x1", "y=y1"))); + assertEquals("a-x1y2",ctest.get("a", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("b")); + assertEquals(null,ctest.get("b", toMap("x=x1"))); + assertEquals(null,ctest.get("b", toMap("x=x1", "y=y1"))); + assertEquals("b-x1y2",ctest.get("b", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("c")); + assertEquals("c-x1",ctest.get("c", toMap("x=x1"))); + assertEquals("c-x1",ctest.get("c", toMap("x=x1", "y=y1"))); + assertEquals("c-x1y2",ctest.get("c", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("d")); + assertEquals("d-x1",ctest.get("d", toMap("x=x1"))); + + assertEquals("d-x1",ctest.get("d", toMap("x=x1", "y=y1"))); + assertEquals("d-x1",ctest.get("d", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("d")); + assertEquals("e-x1",ctest.get("e", toMap("x=x1"))); + assertEquals("e-x1",ctest.get("e", toMap("x=x1", "y=y1"))); + assertEquals("e-x1y2",ctest.get("e", toMap("x=x1", "y=y2"))); + } + + public void testVariantInheritanceSimplified() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x","y"}); + QueryProfile x1y2Parent=new QueryProfile("x1y2Parent"); + x1y2Parent.set("c","c-x1y2", (QueryProfileRegistry)null); + test.addInherited(x1y2Parent,new String[] {"x1","y2"}); + test.set("c","c-x1",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals(null,ctest.get("c")); + assertEquals("c-x1",ctest.get("c", toMap("x=x1"))); + assertEquals("c-x1", ctest.get("c", toMap("x=x1", "y=y1"))); + assertEquals("c-x1y2",ctest.get("c", toMap("x=x1", "y=y2"))); + } + + public void testVariantInheritanceWithCompoundReferences() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("a.b","default-a.b", (QueryProfileRegistry)null); + + QueryProfile ac=new QueryProfile("ac"); + ac.set("a.c","referenced-a.c", (QueryProfileRegistry)null); + test.addInherited(ac,new String[] {"x1"}); + test.set("a.b","x1-a.b",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("Basic functionality","default-a.b",ctest.get("a.b")); + assertEquals("Inherited variance reference works","referenced-a.c",ctest.get("a.c", toMap("x=x1"))); + assertEquals("Inherited variance reference overriding works","x1-a.b",ctest.get("a.b", toMap("x=x1"))); + } + + public void testVariantInheritanceWithTwoLevelCompoundReferencesVariantAtFirstLevel() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("o.a.b","default-a.b", (QueryProfileRegistry)null); + + QueryProfile ac=new QueryProfile("ac"); + ac.set("o.a.c","referenced-a.c", (QueryProfileRegistry)null); + test.addInherited(ac,new String[] {"x1"}); + test.set("o.a.b","x1-a.b",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("Basic functionality","default-a.b",ctest.get("o.a.b")); + assertEquals("Inherited variance reference works","referenced-a.c",ctest.get("o.a.c", toMap("x=x1"))); + assertEquals("Inherited variance reference overriding works","x1-a.b",ctest.get("o.a.b", toMap("x=x1"))); + } + + public void testVariantInheritanceWithTwoLevelCompoundReferencesVariantAtSecondLevel() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + + QueryProfile ac=new QueryProfile("ac"); + ac.set("a.c","referenced-a.c", (QueryProfileRegistry)null); + test.addInherited(ac,new String[] {"x1"}); + test.set("a.b","x1-a.b",new String[] {"x1"}, null); + + QueryProfile top=new QueryProfile("top"); + top.set("o.a.b","default-a.b", (QueryProfileRegistry)null); + top.set("o",test, (QueryProfileRegistry)null); + + CompiledQueryProfile ctop = top.compile(null); + assertEquals("Basic functionality","default-a.b",ctop.get("o.a.b")); + assertEquals("Inherited variance reference works","referenced-a.c",ctop.get("o.a.c", toMap("x=x1"))); + assertEquals("Inherited variance reference does not override value set in referent","default-a.b",ctop.get("o.a.b", toMap("x=x1"))); // Note: Changed from x1-a.b in 4.2.3 + } + + public void testVariantInheritanceOverridesBaseInheritance1() { + QueryProfile test=new QueryProfile("test"); + QueryProfile baseInherited=new QueryProfile("baseInherited"); + baseInherited.set("a.b","baseInherited-a.b", (QueryProfileRegistry)null); + QueryProfile variantInherited=new QueryProfile("variantInherited"); + variantInherited.set("a.b","variantInherited-a.b", (QueryProfileRegistry)null); + test.setDimensions(new String[] {"x"}); + test.addInherited(baseInherited); + test.addInherited(variantInherited,new String[] {"x1"}); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("baseInherited-a.b",ctest.get("a.b")); + assertEquals("variantInherited-a.b",ctest.get("a.b",toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritance2() { + QueryProfile test=new QueryProfile("test"); + QueryProfile baseInherited=new QueryProfile("baseInherited"); + baseInherited.set("a.b","baseInherited-a.b", (QueryProfileRegistry)null); + QueryProfile variantInherited=new QueryProfile("variantInherited"); + variantInherited.set("a.b","variantInherited-a.b", (QueryProfileRegistry)null); + test.setDimensions(new String[] {"x"}); + test.addInherited(baseInherited); + test.addInherited(variantInherited,new String[] {"x1"}); + test.set("a.c","variant-a.c",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("baseInherited-a.b",ctest.get("a.b")); + assertEquals("variantInherited-a.b",ctest.get("a.b", toMap("x=x1"))); + assertEquals("variant-a.c",ctest.get("a.c", toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritanceComplex() { + QueryProfile defaultQP=new QueryProfile("default"); + defaultQP.set("model.defaultIndex","title", (QueryProfileRegistry)null); + + QueryProfile root=new QueryProfile("root"); + root.addInherited(defaultQP); + root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + + QueryProfile querybest=new QueryProfile("querybest"); + querybest.set("defaultIndex","title", (QueryProfileRegistry)null); + querybest.set("queryString","best", (QueryProfileRegistry)null); + + QueryProfile multi=new QueryProfile("multi"); + multi.setDimensions(new String[] {"x"}); + multi.addInherited(defaultQP); + multi.set("model",querybest, (QueryProfileRegistry)null); + multi.addInherited(root,new String[] {"x1"}); + multi.set("model.queryString","love",new String[] {"x1"}, null); + + // Rumtimize + defaultQP.freeze(); + root.freeze(); + querybest.freeze(); + multi.freeze(); + Properties runtime = new QueryProfileProperties(multi.compile(null)); + + assertEquals("default",runtime.get("model.defaultIndex", toMap("x=x1"))); + assertEquals("love",runtime.get("model.queryString", toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritanceComplexSimplified() { + QueryProfile root=new QueryProfile("root"); + root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + + QueryProfile multi=new QueryProfile("multi"); + multi.setDimensions(new String[] {"x"}); + multi.set("model.defaultIndex","title", (QueryProfileRegistry)null); + multi.addInherited(root,new String[] {"x1"}); + + assertEquals("default",multi.compile(null).get("model.defaultIndex", toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritanceMixed() { + QueryProfile root=new QueryProfile("root"); + root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + + QueryProfile multi=new QueryProfile("multi"); + multi.setDimensions(new String[] {"x"}); + multi.set("model.defaultIndex","title", (QueryProfileRegistry)null); + multi.set("model.queryString","modelQuery", (QueryProfileRegistry)null); + multi.addInherited(root,new String[] {"x1"}); + multi.set("model.queryString","modelVariantQuery",new String[] {"x1"}, null); + + CompiledQueryProfile cmulti = multi.compile(null); + assertEquals("default",cmulti.get("model.defaultIndex", toMap("x=x1"))); + assertEquals("modelVariantQuery",cmulti.get("model.queryString", toMap("x=x1"))); + } + + public void testListVariantPropertiesNoCompounds() { + QueryProfile parent1=new QueryProfile("parent1"); + parent1.set("a","parent1-a", (QueryProfileRegistry)null); // Defined everywhere + parent1.set("b","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants + parent1.set("c","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + + QueryProfile parent2=new QueryProfile("parent2"); + parent2.set("a","parent2-a", (QueryProfileRegistry)null); + parent2.set("b","parent2-b", (QueryProfileRegistry)null); + parent2.set("c","parent2-c", (QueryProfileRegistry)null); + parent2.set("d","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.addInherited(parent1); + main.addInherited(parent2); + main.set("a","main-a", (QueryProfileRegistry)null); + main.set("a","main-a-x1",new String[] {"x1"}, null); + main.set("e","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only + main.set("f","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only + main.set("a","main-a-x1.y1",new String[] {"x1","y1"}, null); + main.set("a","main-a-x1.y2",new String[] {"x1","y2"}, null); + main.set("e","main-e-x1.y2",new String[] {"x1","y2"}, null); + main.set("g","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only + main.set("b","main-b", (QueryProfileRegistry)null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("a","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("h","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + + QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); + inheritedVariant2.set("a","inheritedVariant2-a", (QueryProfileRegistry)null); + inheritedVariant2.set("h","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant2.set("i","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + + QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); + inheritedVariant3.set("j","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + + main.addInherited(inheritedVariant1,new String[] {"x1"}); + main.addInherited(inheritedVariant3,new String[] {"x1"}); + main.addInherited(inheritedVariant2,new String[] {"x1","y2"}); + main.addInherited(inheritedVariant3,new String[] {"x1","y2"}); + + // Runtime-ify + Properties properties=new QueryProfileProperties(main.compile(null)); + + int expectedBaseSize=4; + + // No context + Map listed=properties.listProperties(); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + + // Context x=x1 + listed=properties.listProperties(toMap(main, new String[] {"x1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("inheritedVariant1-h",listed.get("h")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x1,y=y1 + listed=properties.listProperties(toMap(main, new String[] {"x1","y1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1.y1",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("inheritedVariant1-h",listed.get("h")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x1,y=y2 + listed=properties.listProperties(toMap(main, new String[] {"x1","y2"})); + assertEquals(expectedBaseSize+6,listed.size()); + assertEquals("main-a-x1.y2",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1.y2",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("main-g-x1.y2",listed.get("g")); + assertEquals("inheritedVariant2-h",listed.get("h")); + assertEquals("inheritedVariant2-i",listed.get("i")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x1,y=y3 + listed=properties.listProperties(toMap(main, new String[] {"x1","y3"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("inheritedVariant1-h",listed.get("h")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x2,y=y1 + listed=properties.listProperties(toMap(main, new String[] {"x2","y1"})); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + } + + public void testListVariantPropertiesCompounds1Simplified() { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.set("a.p1","main-a-x1",new String[] {"x1"}, null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("a.p1","inheritedVariant1-a", (QueryProfileRegistry)null); + main.addInherited(inheritedVariant1,new String[] {"x1"}); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + // Context x=x1 + Map listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals("main-a-x1",listed.get("a.p1")); + } + + public void testListVariantPropertiesCompounds1() { + QueryProfile parent1=new QueryProfile("parent1"); + parent1.set("a.p1","parent1-a", (QueryProfileRegistry)null); // Defined everywhere + parent1.set("b.p1","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants + parent1.set("c.p1","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + + QueryProfile parent2=new QueryProfile("parent2"); + parent2.set("a.p1","parent2-a", (QueryProfileRegistry)null); + parent2.set("b.p1","parent2-b", (QueryProfileRegistry)null); + parent2.set("c.p1","parent2-c", (QueryProfileRegistry)null); + parent2.set("d.p1","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.addInherited(parent1); + main.addInherited(parent2); + main.set("a.p1","main-a", (QueryProfileRegistry)null); + main.set("a.p1","main-a-x1",new String[] {"x1"}, null); + main.set("e.p1","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only + main.set("f.p1","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only + main.set("a.p1","main-a-x1.y1",new String[] {"x1","y1"}, null); + main.set("a.p1","main-a-x1.y2",new String[] {"x1","y2"}, null); + main.set("e.p1","main-e-x1.y2",new String[] {"x1","y2"}, null); + main.set("g.p1","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only + main.set("b.p1","main-b", (QueryProfileRegistry)null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("a.p1","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("h.p1","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + + QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); + inheritedVariant2.set("a.p1","inheritedVariant2-a", (QueryProfileRegistry)null); + inheritedVariant2.set("h.p1","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant2.set("i.p1","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + + QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); + inheritedVariant3.set("j.p1","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + + main.addInherited(inheritedVariant1,new String[] {"x1"}); + main.addInherited(inheritedVariant3,new String[] {"x1"}); + main.addInherited(inheritedVariant2,new String[] {"x1","y2"}); + main.addInherited(inheritedVariant3,new String[] {"x1","y2"}); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + int expectedBaseSize=4; + + // No context + Map listed=properties.listProperties(); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("inheritedVariant1-h",listed.get("h.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x1,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x1","y1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1.y1",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("inheritedVariant1-h",listed.get("h.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x1,y=y2 + listed=properties.listProperties(toMap(main,new String[] {"x1","y2"})); + assertEquals(expectedBaseSize+6,listed.size()); + assertEquals("main-a-x1.y2",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1.y2",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("main-g-x1.y2",listed.get("g.p1")); + assertEquals("inheritedVariant2-h",listed.get("h.p1")); + assertEquals("inheritedVariant2-i",listed.get("i.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x1,y=y3 + listed=properties.listProperties(toMap(main,new String[] {"x1","y3"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("inheritedVariant1-h",listed.get("h.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x2,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x2","y1"})); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + } + + public void testListVariantPropertiesCompounds2() { + QueryProfile parent1=new QueryProfile("parent1"); + parent1.set("p1.a","parent1-a", (QueryProfileRegistry)null); // Defined everywhere + parent1.set("p1.b","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants + parent1.set("p1.c","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + + QueryProfile parent2=new QueryProfile("parent2"); + parent2.set("p1.a","parent2-a", (QueryProfileRegistry)null); + parent2.set("p1.b","parent2-b", (QueryProfileRegistry)null); + parent2.set("p1.c","parent2-c", (QueryProfileRegistry)null); + parent2.set("p1.d","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.addInherited(parent1); + main.addInherited(parent2); + main.set("p1.a","main-a", (QueryProfileRegistry)null); + main.set("p1.a","main-a-x1",new String[] {"x1"}, null); + main.set("p1.e","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only + main.set("p1.f","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only + main.set("p1.a","main-a-x1.y1",new String[] {"x1","y1"}, null); + main.set("p1.a","main-a-x1.y2",new String[] {"x1","y2"}, null); + main.set("p1.e","main-e-x1.y2",new String[] {"x1","y2"}, null); + main.set("p1.g","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only + main.set("p1.b","main-b", (QueryProfileRegistry)null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("p1.a","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("p1.h","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + + QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); + inheritedVariant2.set("p1.a","inheritedVariant2-a", (QueryProfileRegistry)null); + inheritedVariant2.set("p1.h","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant2.set("p1.i","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + + QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); + inheritedVariant3.set("p1.j","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + + main.addInherited(inheritedVariant1,new String[] {"x1"}); + main.addInherited(inheritedVariant3,new String[] {"x1"}); + main.addInherited(inheritedVariant2,new String[] {"x1","y2"}); + main.addInherited(inheritedVariant3,new String[] {"x1","y2"}); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + int expectedBaseSize=4; + + // No context + Map listed=properties.listProperties(); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("inheritedVariant1-h",listed.get("p1.h")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x1,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x1","y1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1.y1",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("inheritedVariant1-h",listed.get("p1.h")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x1,y=y2 + listed=properties.listProperties(toMap(main,new String[] {"x1","y2"})); + assertEquals(expectedBaseSize+6,listed.size()); + assertEquals("main-a-x1.y2",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1.y2",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("main-g-x1.y2",listed.get("p1.g")); + assertEquals("inheritedVariant2-h",listed.get("p1.h")); + assertEquals("inheritedVariant2-i",listed.get("p1.i")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x1,y=y3 + listed=properties.listProperties(toMap(main,new String[] {"x1","y3"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("inheritedVariant1-h",listed.get("p1.h")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x2,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x2","y1"})); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + } + + public void testQueryProfileReferences() { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x1"}); + QueryProfile referencedMain=new QueryProfile("referencedMain"); + referencedMain.set("r1","mainReferenced-r1", (QueryProfileRegistry)null); // In both + referencedMain.set("r2","mainReferenced-r2", (QueryProfileRegistry)null); // Only in this + QueryProfile referencedVariant=new QueryProfile("referencedVariant"); + referencedVariant.set("r1","variantReferenced-r1", (QueryProfileRegistry)null); // In both + referencedVariant.set("r3","variantReferenced-r3", (QueryProfileRegistry)null); // Only in this + + main.set("a",referencedMain, (QueryProfileRegistry)null); + main.set("a",referencedVariant,new String[] {"x1"}, null); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + // No context + Map listed=properties.listProperties(); + assertEquals(2,listed.size()); + assertEquals("mainReferenced-r1",listed.get("a.r1")); + assertEquals("mainReferenced-r2",listed.get("a.r2")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(3,listed.size()); + assertEquals("variantReferenced-r1",listed.get("a.r1")); + assertEquals("mainReferenced-r2",listed.get("a.r2")); + assertEquals("variantReferenced-r3",listed.get("a.r3")); + } + + public void testQueryProfileReferencesWithSubstitution() { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x1"}); + QueryProfile referencedMain=new QueryProfile("referencedMain"); + referencedMain.set("r1","%{prefix}mainReferenced-r1", (QueryProfileRegistry)null); // In both + referencedMain.set("r2","%{prefix}mainReferenced-r2", (QueryProfileRegistry)null); // Only in this + QueryProfile referencedVariant=new QueryProfile("referencedVariant"); + referencedVariant.set("r1","%{prefix}variantReferenced-r1", (QueryProfileRegistry)null); // In both + referencedVariant.set("r3","%{prefix}variantReferenced-r3", (QueryProfileRegistry)null); // Only in this + + main.set("a",referencedMain, (QueryProfileRegistry)null); + main.set("a",referencedVariant,new String[] {"x1"}, null); + main.set("prefix","mainPrefix:", (QueryProfileRegistry)null); + main.set("prefix","variantPrefix:",new String[] {"x1"}, null); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + // No context + Map listed=properties.listProperties(); + assertEquals(3,listed.size()); + assertEquals("mainPrefix:mainReferenced-r1",listed.get("a.r1")); + assertEquals("mainPrefix:mainReferenced-r2",listed.get("a.r2")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(4,listed.size()); + assertEquals("variantPrefix:variantReferenced-r1",listed.get("a.r1")); + assertEquals("variantPrefix:mainReferenced-r2",listed.get("a.r2")); + assertEquals("variantPrefix:variantReferenced-r3",listed.get("a.r3")); + } + + public void testNewsCase1() { + QueryProfile shortcuts=new QueryProfile("shortcuts"); + shortcuts.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); + shortcuts.set("testout","outside", (QueryProfileRegistry)null); + shortcuts.set("test.out","dotoutside", (QueryProfileRegistry)null); + shortcuts.set("testin","inside",new String[] {"yahoo","ca","sc"}, null); + shortcuts.set("test.in","dotinside",new String[] {"yahoo","ca","sc"}, null); + + QueryProfile profile=new QueryProfile("default"); + profile.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); + profile.addInherited(shortcuts, new String[] {"yahoo",null,"sc"}); + + profile.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=yahoo&custid_2=ca&custid_3=sc", Method.GET), profile.compile(null)); + + assertEquals("outside",query.properties().get("testout")); + assertEquals("dotoutside",query.properties().get("test.out")); + assertEquals("inside",query.properties().get("testin")); + assertEquals("dotinside",query.properties().get("test.in")); + } + + public void testNewsCase2() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions("sort,resulttypes,rss,age,intl,testid".split(",")); + String[] dimensionValues=new String[] {null,null,"0"}; + test.set("discovery","sources",dimensionValues, null); + test.set("discoverytypes","article",dimensionValues, null); + test.set("discovery.sources.count","10",dimensionValues, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals("sources",ctest.get("discovery", toMap(test, dimensionValues))); + assertEquals("article",ctest.get("discoverytypes", toMap(test, dimensionValues))); + assertEquals("10",ctest.get("discovery.sources.count", toMap(test, dimensionValues))); + + Map values=ctest.listValues("",toMap(test,dimensionValues)); + assertEquals(3,values.size()); + assertEquals("sources",values.get("discovery")); + assertEquals("article",values.get("discoverytypes")); + assertEquals("10",values.get("discovery.sources.count")); + + Map sourceValues=ctest.listValues("discovery.sources",toMap(test,dimensionValues)); + assertEquals(1,sourceValues.size()); + assertEquals("10",sourceValues.get("count")); + } + + public void testRuntimeAssignmentInClone() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + String[] x1=new String[] {"x1"}; + Map x1m=toMap(test,x1); + test.set("a","30",x1, null); + test.set("a.b","20",x1, null); + test.set("a.b.c","10",x1, null); + + // Setting in one profile works + Query qMain = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), test.compile(null)); + qMain.properties().set("a.b","50",x1m); + assertEquals("50",qMain.properties().get("a.b",x1m)); + + // Cloning + Query qBranch=qMain.clone(); + + // Setting in main still works + qMain.properties().set("a.b","51",x1m); + assertEquals("51",qMain.properties().get("a.b",x1m)); + + // Clone is not affected by change in original + assertEquals("50",qBranch.properties().get("a.b",x1m)); + + // Setting in clone works + qBranch.properties().set("a.b","70",x1m); + assertEquals("70",qBranch.properties().get("a.b",x1m)); + + // Setting in clone does not affect original + assertEquals("51",qMain.properties().get("a.b",x1m)); + } + + public void testIncompatibleDimensions() { + QueryProfile alert = new QueryProfile("alert"); + + QueryProfile backendBase = new QueryProfile("backendBase"); + backendBase.setDimensions(new String[] { "sort", "resulttypes", "rss" }); + backendBase.set("custid", "s", (QueryProfileRegistry)null); + + QueryProfile backend = new QueryProfile("backend"); + backend.setDimensions(new String[] { "sort", "offset", "resulttypes", "rss", "age", "lang", "fr", "entry" }); + backend.addInherited(backendBase); + + QueryProfile web = new QueryProfile("web"); + web.setDimensions(new String[] { "entry", "recency" }); + web.set("fr", "alerts", new String[] { "alert" }, null); + + alert.set("config.backend.vertical.news", backend, (QueryProfileRegistry)null); + alert.set("config.backend.multimedia", web, (QueryProfileRegistry)null); + backend.set("custid", "yahoo/alerts", new String[] { null, null, null, null, null, "en-US", null, "alert"}, null); + + CompiledQueryProfile cAlert = alert.compile(null); + assertEquals("yahoo/alerts", cAlert.get("config.backend.vertical.news.custid", toMap("entry=alert", "intl=us", "lang=en-US"))); + } + + public void testIncompatibleDimensionsSimplified() { + QueryProfile alert = new QueryProfile("alert"); + + QueryProfile backendBase = new QueryProfile("backendBase"); + backendBase.set("custid", "s", (QueryProfileRegistry)null); + + QueryProfile backend = new QueryProfile("backend"); + backend.setDimensions(new String[] { "sort", "lang", "fr", "entry" }); + backend.set("custid", "yahoo/alerts", new String[] { null, "en-US", null, "alert"}, null); + backend.addInherited(backendBase); + + QueryProfile web = new QueryProfile("web"); + web.setDimensions(new String[] { "entry", "recency" }); + web.set("fr", "alerts", new String[] { "alert" }, null); + + alert.set("vertical", backend, (QueryProfileRegistry)null); + alert.set("multimedia", web, (QueryProfileRegistry)null); + + CompiledQueryProfile cAlert = alert.compile(null); + assertEquals("yahoo/alerts", cAlert.get("vertical.custid", toMap("entry=alert", "intl=us", "lang=en-US"))); + } + + private void assertGet(String expectedValue, String parameter, String[] dimensionValues, QueryProfile profile, CompiledQueryProfile cprofile) { + Map context=toMap(profile,dimensionValues); + assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'",expectedValue,cprofile.get(parameter,context)); + } + + public static Map toMap(QueryProfile profile, String[] dimensionValues) { + Map context=new HashMap<>(); + List dimensions; + if (profile.getVariants()!=null) + dimensions=profile.getVariants().getDimensions(); + else + dimensions=((BackedOverridableQueryProfile)profile).getBacking().getVariants().getDimensions(); + + for (int i=0; i toMap(String... bindings) { + Map context = new HashMap<>(); + for (String binding : bindings) { + String[] entry = binding.split("="); + context.put(entry[0].trim(), entry[1].trim()); + } + return context; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java new file mode 100644 index 00000000000..ae1e39e52d0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * @author bratseth + */ +public class FieldTypeTestCase extends junit.framework.TestCase { + + public void testConvertToFromString() { + QueryProfileTypeRegistry registry=new QueryProfileTypeRegistry(); + registry.register(new QueryProfileType("foo")); + + assertEquals("string", FieldType.fromString("string",registry).stringValue()); + assertEquals("boolean", FieldType.fromString("boolean",registry).stringValue()); + assertEquals("query-profile", FieldType.fromString("query-profile",registry).stringValue()); + assertEquals("query-profile:foo", FieldType.fromString("query-profile:foo",registry).stringValue()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java new file mode 100644 index 00000000000..8e2c465911b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.search.test.QueryTestCase; + +/** + * @author bratseth + */ +public class MandatoryTestCase extends junit.framework.TestCase { + + private QueryProfileTypeRegistry registry; + + private QueryProfileType type, user; + + protected @Override void setUp() { + type=new QueryProfileType(new ComponentId("testtype")); + user=new QueryProfileType(new ComponentId("user")); + registry=new QueryProfileTypeRegistry(); + registry.register(type); + registry.register(user); + + addTypeFields(type); + addUserFields(user); + } + + private void addTypeFields(QueryProfileType type) { + boolean mandatory=true; + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry), mandatory)); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry))); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry))); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry))); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry))); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry))); + type.addField(new FieldDescription("myUserQueryProfile", FieldType.fromString("query-profile:user",registry),mandatory)); + } + + private void addUserFields(QueryProfileType user) { + boolean mandatory=true; + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry),mandatory)); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry),mandatory)); + } + + public void testMandatoryFullySpecifiedQueryProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + test.set("myString","aString", registry); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + myUser.set("myUserString",1, registry); + test.set("myUserQueryProfile", myUser, registry); + registry.register(myUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Fully specified request + assertError(null, new Query(QueryTestCase.httpEncode("?queryProfile=test"), cRegistry.getComponent("test"))); + } + + public void testMandatoryRequestPropertiesNeeded() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'myString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("", Method.GET), cRegistry.getComponent("test"))); + + // Underspecified request 2 + assertError("Incomplete query: Parameter 'myUserQueryProfile.myUserString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString", Method.GET), cRegistry.getComponent("test"))); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString&myUserQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("test"))); + } + + /** Same as above except the whole thing is nested in maps */ + public void testMandatoryNestedInMaps() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile topMap=new QueryProfile("topMap"); + registry.register(topMap); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + registry.register(subMap); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("test",test, registry); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'subMap.test.myString' is mandatory in query profile 'topMap' but is not set", + new Query(HttpRequest.createTestRequest("", Method.GET), cRegistry.getComponent("topMap"))); + + // Underspecified request 2 + assertError("Incomplete query: Parameter 'subMap.test.myUserQueryProfile.myUserString' is mandatory in query profile 'topMap' but is not set", + new Query(HttpRequest.createTestRequest("?subMap.test.myString=aString", Method.GET), cRegistry.getComponent("topMap"))); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?subMap.test.myString=aString&subMap.test.myUserQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("topMap"))); + } + + /** Here, no user query profile is referenced in the query profile, but one is chosen in the request */ + public void testMandatoryUserProfileSetInRequest() { + QueryProfile test=new QueryProfile("test"); + test.setType(type); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, (QueryProfileRegistry)null); + + QueryProfileRegistry registry = new QueryProfileRegistry(); + registry.register(test); + registry.register(myUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'myUserQueryProfile' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString", Method.GET), cRegistry.getComponent("test"))); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'myUserQueryProfile.myUserString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString&myUserQueryProfile=user", Method.GET), cRegistry.getComponent("test"))); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString&myUserQueryProfile=user&myUserQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("test"))); + } + + /** Here, a partially specified query profile is added to a non-mandatory field, making the request underspecified */ + public void testNonMandatoryUnderspecifiedUserProfileSetInRequest() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile test = new QueryProfile("test"); + test.setType(type); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger", 1, registry); + myUser.set("myUserString","userValue", registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + QueryProfile otherUser=new QueryProfile("otherUser"); + otherUser.setType(user); + otherUser.set("myUserInteger", 2, registry); + registry.register(otherUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString", Method.GET), cRegistry.getComponent("test"))); + + // Underspecified because an underspecified profile is added + assertError("Incomplete query: Parameter 'myQueryProfile.myUserString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString&myQueryProfile=otherUser", Method.GET), cRegistry.getComponent("test"))); + + // Back to fully specified + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString&myQueryProfile=otherUser&myQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("test"))); + } + + private void assertError(String message,Query query) { + assertEquals(message, query.validate()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java new file mode 100644 index 00000000000..562418647c8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * tests creating invalid names + * + * @author bratseth + */ +public class NameTestCase extends junit.framework.TestCase { + + public void testNames() { + assertLegalName("aB"); + assertIllegalName("a."); + assertLegalName("_a_b"); + assertLegalName("a_b"); + assertLegalName("a/b"); + assertLegalName("/a/b"); + assertLegalName("/a/b/"); + assertIllegalName(""); + } + + public void testFieldNames() { + assertLegalFieldName("aB"); + try { + QueryProfile profile=new QueryProfile("test"); + profile.set("a.","anyValue", (QueryProfileRegistry)null); + fail("Should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("'a.' is not a legal compound name. Names can not end with a dot.", e.getMessage()); + } + assertLegalFieldName("_a_b"); + assertLegalFieldName("a_b"); + assertLegalFieldName("a/b"); + assertLegalFieldName("/a/b"); + assertLegalFieldName("/a/b/"); + assertIllegalFieldName(""); + assertIllegalFieldName("aBc.dooEee.ce_d.-some-other.moreHere", + "Could not set 'aBc.dooEee.ce_d.-some-other.moreHere' to 'anyValue'", + "Illegal name '-some-other'"); + } + + private void assertLegalName(String name) { + new QueryProfile(name); + new QueryProfileType(name); + } + + private void assertLegalFieldName(String name) { + new QueryProfile(name).set(name, "value", (QueryProfileRegistry)null); + new FieldDescription(name,FieldType.stringType); + } + + /** Checks that this is illegal both for profiles and types */ + private void assertIllegalName(String name) { + try { + new QueryProfile(name); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + if (!name.isEmpty()) + assertEquals("Illegal name '" + name + "'",e.getMessage()); + } + + try { + new QueryProfileType(name); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + if (!name.isEmpty()) + assertEquals("Illegal name '" + name + "'",e.getMessage()); + } + } + + private void assertIllegalFieldName(String name) { + assertIllegalFieldName(name,"Could not set '" + name + "' to 'anyValue'","Illegal name '" + name + "'"); + } + + /** Checks that this is illegal both for profiles and types */ + private void assertIllegalFieldName(String name, String expectedHighError, String expectedLowError) { + try { + QueryProfile profile=new QueryProfile("test"); + profile.set(name, "anyValue", (QueryProfileRegistry)null); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals(expectedHighError + ": " + expectedLowError, Exceptions.toMessageString(e)); + } + + try { + new FieldDescription(name, FieldType.stringType); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals(expectedLowError, e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java new file mode 100644 index 00000000000..77e733a740a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.yolean.Exceptions; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; + +/** + * Tests that properties can not be set even if they are native, if declared not settable in the query profile + * + * @author bratseth + */ +public class NativePropertiesTestCase extends junit.framework.TestCase { + + public void testNativeInStrict() { + QueryProfileType strictType=new QueryProfileType("strict"); + strictType.setStrict(true); + QueryProfile strict=new QueryProfile("profile"); + strict.setType(strictType); + + try { + new Query(HttpRequest.createTestRequest("?hits=10&tracelevel=5", Method.GET), strict.compile(null)); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + } + + try { + new Query(HttpRequest.createTestRequest("?notnative=5", Method.GET), strict.compile(null)); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString( + "Could not set 'notnative' to '5':" + + " 'notnative' is not declared in query profile type 'strict', and the type is strict")); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java new file mode 100644 index 00000000000..77c3d26f9be --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * Tests overriding of field values + * + * @author bratseth + */ +public class OverrideTestCase extends junit.framework.TestCase { + + private QueryProfileTypeRegistry registry; + + private QueryProfileType type, user; + + protected @Override void setUp() { + type=new QueryProfileType(new ComponentId("testtype")); + user=new QueryProfileType(new ComponentId("user")); + registry=new QueryProfileTypeRegistry(); + registry.register(type); + registry.register(user); + + addTypeFields(type); + addUserFields(user); + } + + private void addTypeFields(QueryProfileType type) { + boolean overridable=true; + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry),false,!overridable)); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry))); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry))); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry))); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry))); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry))); + type.addField(new FieldDescription("myUserQueryProfile", FieldType.fromString("query-profile:user",registry),false,!overridable)); + } + + private void addUserFields(QueryProfileType user) { + boolean overridable=true; + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry),false,!overridable)); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry))); + } + + /** Check that a simple non-overridable string cannot be overridden */ + public void testSimpleUnoverridable() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile test=new QueryProfile("test"); + test.setType(type); + test.set("myString","finalString", (QueryProfileRegistry)null); + registry.register(test); + registry.freeze(); + + // Assert request assignment does not work + Query query = new Query(HttpRequest.createTestRequest("?myString=newValue", Method.GET), registry.compile().getComponent("test")); + assertEquals(0,query.errors().size()); + assertEquals("finalString",query.properties().get("myString")); + + // Assert direct assignment does not work + query.properties().set("myString","newValue"); + assertEquals("finalString",query.properties().get("myString")); + } + + /** Check that a query profile cannot be overridden */ + public void testUnoverridableQueryProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test = new QueryProfile("test"); + test.setType(type); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + myUser.set("myUserString","userValue", registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + QueryProfile otherUser = new QueryProfile("otherUser"); + otherUser.setType(user); + otherUser.set("myUserInteger", 2, registry); + registry.register(otherUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryprofile=otherUser", Method.GET), cRegistry.getComponent("test")); + assertEquals(0,query.errors().size()); + assertEquals(1,query.properties().get("myUserQueryProfile.myUserInteger")); + } + + /** Check that non-overridables are protected also in nested untyped references */ + public void testUntypedNestedUnoverridable() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile topMap = new QueryProfile("topMap"); + registry.register(topMap); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + registry.register(subMap); + + QueryProfile test = new QueryProfile("test"); + test.setType(type); + subMap.set("test",test, registry); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserString","finalValue", registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + registry.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?subMap.test.myUserQueryProfile.myUserString=newValue", Method.GET), registry.compile().getComponent("topMap")); + assertEquals(0,query.errors().size()); + assertEquals("finalValue",query.properties().get("subMap.test.myUserQueryProfile.myUserString")); + + query.properties().set("subMap.test.myUserQueryProfile.myUserString","newValue"); + assertEquals("finalValue",query.properties().get("subMap.test.myUserQueryProfile.myUserString")); + } + + /** Tests overridability in an inherited field */ + public void testInheritedNonOverridableInType() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + test.set("myString","finalString", (QueryProfileRegistry)null); + registry.register(test); + + QueryProfile profile=new QueryProfile("profile"); + profile.addInherited(test); + registry.register(profile); + + registry.freeze(); + + Query query = new Query(HttpRequest.createTestRequest("?myString=newString", Method.GET), registry.compile().getComponent("test")); + + assertEquals(0,query.errors().size()); + assertEquals("finalString",query.properties().get("myString")); + + query.properties().set("myString","newString"); + assertEquals("finalString",query.properties().get("myString")); + } + + /** Tests overridability in an inherited field */ + public void testInheritedNonOverridableInProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile test = new QueryProfile("test"); + test.setType(type); + test.set("myInteger", 1, registry); + test.setOverridable("myInteger", false, null); + registry.register(test); + + QueryProfile profile=new QueryProfile("profile"); + profile.addInherited(test); + registry.register(profile); + + registry.freeze(); + + Query query = new Query(HttpRequest.createTestRequest("?myInteger=32", Method.GET), registry.compile().getComponent("test")); + + assertEquals(0,query.errors().size()); + assertEquals(1,query.properties().get("myInteger")); + + query.properties().set("myInteger",32); + assertEquals(1,query.properties().get("myInteger")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java new file mode 100644 index 00000000000..65a552931ac --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * Tests that matching query profiles by path name works + * + * @author bratseth + */ +public class PatchMatchingTestCase extends junit.framework.TestCase { + + public void testPatchMatching() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(true); + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee=new QueryProfile("a/bee"); + abee.setType(type); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(type); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee/eee/").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee/cede").getId().getName()); + assertEquals("a",registry.findQueryProfile("a/foo/bee/cede").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + public void testNoPatchMatching() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(false); // Default, but set here for clarity + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee=new QueryProfile("a/bee"); + abee.setType(type); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(type); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertNull(registry.findQueryProfile("a/bee/ce/dee")); // Different from test above + assertNull(registry.findQueryProfile("a/bee/ce/dee/eee/")); // Different from test above + assertNull(registry.findQueryProfile("a/bee/cede")); // Different from test above + assertNull(registry.findQueryProfile("a/foo/bee/cede")); // Different from test above + assertNull(registry.findQueryProfile("abee")); + } + + /** Check that the path matching property is inherited to subtypes */ + public void testPatchMatchingInheritance() { + QueryProfileType type=new QueryProfileType("type"); + QueryProfileType subType=new QueryProfileType("subType"); + subType.inherited().add(type); + + type.setMatchAsPath(true); // Supertype only + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee=new QueryProfile("a/bee"); + abee.setType(subType); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(subType); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee/eee/").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee/cede").getId().getName()); + assertEquals("a",registry.findQueryProfile("a/foo/bee/cede").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + /** Check that the path matching works with versioned profiles */ + public void testPatchMatchingVersions() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(true); + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee11=new QueryProfile("a/bee:1.1"); + abee11.setType(type); + abee11.addInherited(a); + QueryProfile abee13=new QueryProfile("a/bee:1.3"); + abee13.setType(type); + abee13.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(type); + abeece.addInherited(abee13); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee11); + registry.register(abee13); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee:1.1",registry.findQueryProfile("a/bee:1.1").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee:1").getId().toString()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee/eee/").getId().getName()); + assertEquals("a/bee:1.1",registry.findQueryProfile("a/bee/cede:1.1").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee/cede").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee/cede:1").getId().toString()); + assertEquals("a",registry.findQueryProfile("a/foo/bee/cede").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + public void testQuirkyNames() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(true); + + QueryProfile a=new QueryProfile("/a"); + a.setType(type); + QueryProfile abee=new QueryProfile("/a//bee"); + abee.setType(type); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("/a//bee/ce/"); + abeece.setType(type); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("/a",registry.findQueryProfile("/a").getId().getName()); + assertNull(registry.findQueryProfile("a")); + assertEquals("/a//bee",registry.findQueryProfile("/a//bee").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce/").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce/dee").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce/dee/eee/").getId().getName()); + assertEquals("/a//bee",registry.findQueryProfile("/a//bee/cede").getId().getName()); + assertEquals("/a",registry.findQueryProfile("/a/foo/bee/cede").getId().getName()); + assertEquals("/a",registry.findQueryProfile("/a/bee").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java new file mode 100644 index 00000000000..85333e1e95a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * @author bratseth + */ +public class QueryProfileTypeInheritanceTestCase extends junit.framework.TestCase { + + private QueryProfileTypeRegistry registry; + + private QueryProfileType type, typeStrict, user, userStrict; + + protected @Override void setUp() { + type=new QueryProfileType(new ComponentId("testtype")); + typeStrict=new QueryProfileType(new ComponentId("testtypeStrict")); + typeStrict.setStrict(true); + user=new QueryProfileType(new ComponentId("user")); + userStrict=new QueryProfileType(new ComponentId("userStrict")); + userStrict.setStrict(true); + registry=new QueryProfileTypeRegistry(); + registry.register(type); + registry.register(typeStrict); + registry.register(user); + registry.register(userStrict); + + addTypeFields(type); + type.addField(new FieldDescription("myUserQueryProfile", FieldType.fromString("query-profile:user",registry))); + addTypeFields(typeStrict); + typeStrict.addField(new FieldDescription("myUserQueryProfile",FieldType.fromString("query-profile:userStrict",registry))); + addUserFields(user); + addUserFields(userStrict); + } + + private void addTypeFields(QueryProfileType type) { + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry))); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry))); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry))); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry))); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry))); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry))); + } + + private void addUserFields(QueryProfileType user) { + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry),true,false)); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry))); + } + + public void testInheritance() { + type.inherited().add(user); + type.freeze(); + user.freeze(); + + assertFalse(type.isOverridable("myUserString")); + assertEquals("myUserInteger", type.getField("myUserInteger").getName()); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + + test.set("myUserInteger","37", (QueryProfileRegistry)null); + test.set("myUnknownInteger","38", (QueryProfileRegistry)null); + CompiledQueryProfile ctest = test.compile(null); + + assertEquals(37, ctest.get("myUserInteger")); + assertEquals("38", ctest.get("myUnknownInteger")); + } + + public void testInheritanceStrict() { + typeStrict.inherited().add(userStrict); + typeStrict.freeze(); + userStrict.freeze(); + + QueryProfile test=new QueryProfile("test"); + test.setType(typeStrict); + + test.set("myUserInteger","37", (QueryProfileRegistry)null); + try { + test.set("myUnknownInteger","38", (QueryProfileRegistry)null); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("'myUnknownInteger' is not declared in query profile type 'testtypeStrict', and the type is strict", + e.getCause().getMessage()); + } + + assertEquals(37,test.get("myUserInteger")); + assertNull(test.get("myUnknownInteger")); + } + + public void testStrictIsInherited() { + type.inherited().add(userStrict); + type.freeze(); + userStrict.freeze(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + + test.set("myUserInteger","37", (QueryProfileRegistry)null); + try { + test.set("myUnknownInteger","38", (QueryProfileRegistry)null); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("'myUnknownInteger' is not declared in query profile type 'testtype', and the type is strict", + e.getCause().getMessage()); + } + + CompiledQueryProfile ctest = test.compile(null); + assertEquals(37, ctest.get("myUserInteger")); + assertNull(ctest.get("myUnknownInteger")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java new file mode 100644 index 00000000000..d9dfc733f04 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -0,0 +1,595 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.tensor.MapTensor; +import com.yahoo.tensor.Tensor; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.Query; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; + +/** + * tests query profiles with/and types + * + * @author bratseth + */ +public class QueryProfileTypeTestCase extends junit.framework.TestCase { + + private QueryProfileRegistry registry; + + private QueryProfileType type, typeStrict, user, userStrict; + + @Override + protected void setUp() { + registry = new QueryProfileRegistry(); + + type = new QueryProfileType(new ComponentId("testtype")); + type.inherited().add(registry.getTypeRegistry().getComponent(new ComponentId("native"))); + typeStrict = new QueryProfileType(new ComponentId("testtypeStrict")); + typeStrict.setStrict(true); + user = new QueryProfileType(new ComponentId("user")); + userStrict = new QueryProfileType(new ComponentId("userStrict")); + userStrict.setStrict(true); + + registry.getTypeRegistry().register(type); + registry.getTypeRegistry().register(typeStrict); + registry.getTypeRegistry().register(user); + registry.getTypeRegistry().register(userStrict); + + addTypeFields(type, registry.getTypeRegistry()); + type.addField(new FieldDescription("myUserQueryProfile",FieldType.fromString("query-profile:user",registry.getTypeRegistry()))); + addTypeFields(typeStrict, registry.getTypeRegistry()); + typeStrict.addField(new FieldDescription("myUserQueryProfile",FieldType.fromString("query-profile:userStrict",registry.getTypeRegistry()))); + addUserFields(user, registry.getTypeRegistry()); + addUserFields(userStrict, registry.getTypeRegistry()); + + } + + private void addTypeFields(QueryProfileType type, QueryProfileTypeRegistry registry) { + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry)), registry); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry),"int"), registry); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry)), registry); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry)), registry); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry)), registry); + type.addField(new FieldDescription("myBoolean",FieldType.fromString("boolean",registry)), registry); + type.addField(new FieldDescription("myBoolean",FieldType.fromString("boolean",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor1)",FieldType.fromString("tensor",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor2)",FieldType.fromString("tensor(x[2],y[2])",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor3)",FieldType.fromString("tensor(x{})",registry)), registry); + type.addField(new FieldDescription("myQuery",FieldType.fromString("query",registry)), registry); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry),"qp"), registry); + } + + private void addUserFields(QueryProfileType user, QueryProfileTypeRegistry registry) { + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry)), registry); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry),"uint"), registry); + } + + public void testTypedOfPrimitivesAssignmentNonStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + registry.register(profile); + + profile.set("myString","anyValue", registry); + profile.set("nontypedString", "anyValueToo", registry); // legal because this is not strict + assertWrongType(profile,"integer","myInteger","notInteger"); + assertWrongType(profile, "integer", "myInteger", "1.5"); + profile.set("myInteger", 3, registry); + assertWrongType(profile,"long","myLong","notLong"); + assertWrongType(profile, "long", "myLong", "1.5"); + profile.set("myLong", 4000000000000l, registry); + assertWrongType(profile, "float", "myFloat", "notFloat"); + profile.set("myFloat", 3.14f, registry); + assertWrongType(profile, "double", "myDouble", "notDouble"); + profile.set("myDouble",2.18, registry); + profile.set("myBoolean",true, registry); + + String tensorString1 = "{{a:a1, b:b1}:1.0, {a:a2}:2.0}}"; + profile.set("ranking.features.query(myTensor1)", tensorString1, registry); + String tensorString2 = "{{x:0, y:0}:1.0, {x:0, y:1}:2.0}}"; + profile.set("ranking.features.query(myTensor2)", tensorString2, registry); + String tensorString3 = "{{x:x1}:1.0, {x:x2}:2.0}}"; + profile.set("ranking.features.query(myTensor3)", tensorString3, registry); + + profile.set("myQuery", "...", registry); // TODO + profile.set("myQueryProfile.anyString","value1", registry); + profile.set("myQueryProfile.anyDouble",8.76, registry); + profile.set("myUserQueryProfile.myUserString","value2", registry); + profile.set("myUserQueryProfile.anyString", "value3", registry); // Legal because user is not strict + assertWrongType(profile, "integer", "myUserQueryProfile.myUserInteger", "notInteger"); + profile.set("myUserQueryProfile.uint",1337, registry); // Set using alias + profile.set("myUserQueryProfile.anyDouble", 9.13, registry); // Legal because user is not strict + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + QueryProfileProperties properties = new QueryProfileProperties(cRegistry.findQueryProfile("test")); + + assertEquals("anyValue", properties.get("myString")); + assertEquals("anyValueToo", properties.get("nontypedString")); + assertEquals(3, properties.get("myInteger")); + assertEquals(3, properties.get("Int")); + assertEquals(4000000000000l, properties.get("myLong")); + assertEquals(3.14f, properties.get("myFloat")); + assertEquals(2.18, properties.get("myDouble")); + assertEquals(true, properties.get("myBoolean")); + assertEquals(Tensor.from(tensorString1), properties.get("ranking.features.query(myTensor1)")); + assertEquals(Tensor.from("tensor(x[2],y[2])", tensorString2), properties.get("ranking.features.query(myTensor2)")); + assertEquals(Tensor.from("tensor(x{})", tensorString3), properties.get("ranking.features.query(myTensor3)")); + // TODO: assertEquals(..., cprofile.get("myQuery")); + assertEquals("value1", properties.get("myQueryProfile.anyString")); + assertEquals("value1", properties.get("QP.anyString")); + assertEquals(8.76, properties.get("myQueryProfile.anyDouble")); + assertEquals(8.76, properties.get("qp.anyDouble")); + assertEquals("value2", properties.get("myUserQueryProfile.myUserString")); + assertEquals("value3", properties.get("myUserQueryProfile.anyString")); + assertEquals(1337, properties.get("myUserQueryProfile.myUserInteger")); + assertEquals(1337, properties.get("myUserQueryProfile.uint")); + assertEquals(9.13, properties.get("myUserQueryProfile.anyDouble")); + assertNull(properties.get("nonExisting")); + + properties.set("INt", 51); + assertEquals(51, properties.get("InT")); + assertEquals(51, properties.get("myInteger")); + } + + public void testTypedOfPrimitivesAssignmentStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + profile.set("myString", "anyValue", registry); + assertNotPermitted(profile, "nontypedString", "anyValueToo"); // Illegal because this is strict + assertWrongType(profile,"integer","myInteger","notInteger"); + assertWrongType(profile, "integer", "myInteger", "1.5"); + profile.set("myInteger", 3, registry); + assertWrongType(profile,"long","myLong","notLong"); + assertWrongType(profile, "long", "myLong", "1.5"); + profile.set("myLong", 4000000000000l, registry); + assertWrongType(profile, "float", "myFloat", "notFloat"); + profile.set("myFloat", 3.14f, registry); + assertWrongType(profile, "double", "myDouble", "notDouble"); + profile.set("myDouble",2.18, registry); + profile.set("myQueryProfile.anyString","value1", registry); + profile.set("myQueryProfile.anyDouble",8.76, registry); + profile.set("myUserQueryProfile.myUserString", "value2", registry); + assertNotPermitted(profile, "myUserQueryProfile.anyString", "value3"); // Illegal because this is strict + assertWrongType(profile, "integer", "myUserQueryProfile.myUserInteger", "notInteger"); + profile.set("myUserQueryProfile.myUserInteger", 1337, registry); + assertNotPermitted(profile, "myUserQueryProfile.anyDouble", 9.13); // Illegal because this is strict + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("anyValue", cprofile.get("myString")); + assertNull(cprofile.get("nontypedString")); + assertEquals(3, cprofile.get("myInteger")); + assertEquals(4000000000000l, cprofile.get("myLong")); + assertEquals(3.14f, cprofile.get("myFloat")); + assertEquals(2.18, cprofile.get("myDouble")); + assertEquals("value1", cprofile.get("myQueryProfile.anyString")); + assertEquals(8.76, cprofile.get("myQueryProfile.anyDouble")); + assertEquals("value2", cprofile.get("myUserQueryProfile.myUserString")); + assertNull(cprofile.get("myUserQueryProfile.anyString")); + assertEquals(1337, cprofile.get("myUserQueryProfile.myUserInteger")); + assertNull(cprofile.get("myUserQueryProfile.anyDouble")); + } + + /** Tests assigning a subprofile directly */ + public void testTypedAssignmentOfQueryProfilesNonStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + + QueryProfile map1=new QueryProfile("myMap1"); + map1.set("key1","value1", registry); + + QueryProfile map2=new QueryProfile("myMap2"); + map2.set("key2","value2", registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + assertWrongType(profile,"reference to a query profile","myQueryProfile","aString"); + profile.set("myQueryProfile",map1, registry); + profile.set("someMap",map2, registry); // Legal because this is not strict + assertWrongType(profile,"reference to a query profile of type 'user'","myUserQueryProfile",map1); + profile.set("myUserQueryProfile",myUser, registry); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("value1", cprofile.get("myQueryProfile.key1")); + assertEquals("value2", cprofile.get("someMap.key2")); + assertEquals("userValue1", cprofile.get("myUserQueryProfile.myUserString")); + assertEquals(442, cprofile.get("myUserQueryProfile.myUserInteger")); + } + + /** Tests assigning a subprofile directly */ + public void testTypedAssignmentOfQueryProfilesStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + QueryProfile map1=new QueryProfile("myMap1"); + map1.set("key1","value1", registry); + + QueryProfile map2=new QueryProfile("myMap2"); + map2.set("key2","value2", registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(userStrict); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + assertWrongType(profile,"reference to a query profile","myQueryProfile","aString"); + profile.set("myQueryProfile",map1, registry); + assertNotPermitted(profile,"someMap",map2); + assertWrongType(profile,"reference to a query profile of type 'userStrict'","myUserQueryProfile",map1); + profile.set("myUserQueryProfile",myUser, registry); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("value1", cprofile.get("myQueryProfile.key1")); + assertNull(cprofile.get("someMap.key2")); + assertEquals("userValue1", cprofile.get("myUserQueryProfile.myUserString")); + assertEquals(442, cprofile.get("myUserQueryProfile.myUserInteger")); + } + + /** Tests assigning a subprofile as an id string */ + public void testTypedAssignmentOfQueryProfileReferencesNonStrict() { + QueryProfile profile = new QueryProfile("test"); + profile.setType(type); + + QueryProfile map1 = new QueryProfile("myMap1"); + map1.set("key1","value1", registry); + + QueryProfile map2 = new QueryProfile("myMap2"); + map2.set("key2","value2", registry); + + QueryProfile myUser = new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + registry.register(profile); + registry.register(map1); + registry.register(map2); + registry.register(myUser); + + assertWrongType(profile,"reference to a query profile", "myQueryProfile", "aString"); + registry.register(map1); + profile.set("myQueryProfile", "myMap1", registry); + registry.register(map2); + profile.set("someMap", "myMap2", registry); // NOTICE: Will set as a string because we cannot know this is a reference + assertWrongType(profile, "reference to a query profile of type 'user'", "myUserQueryProfile", "myMap1"); + registry.register(myUser); + profile.set("myUserQueryProfile","myUser", registry); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cprofile = cRegistry.getComponent("test"); + + assertEquals("value1", cprofile.get("myQueryProfile.key1")); + assertEquals("myMap2", cprofile.get("someMap")); + assertNull("Asking for an value which cannot be completely resolved returns null", cprofile.get("someMap.key2")); + assertEquals("userValue1", cprofile.get("myUserQueryProfile.myUserString")); + assertEquals(442, cprofile.get("myUserQueryProfile.myUserInteger")); + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here there exists a user profile already, and then a new one is overwritten + */ + public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQuery() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + QueryProfileRegistry registry = new QueryProfileRegistry(); + registry.register(profile); + registry.register(myUser); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cprofile = cRegistry.getComponent("test"); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cprofile); + + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("myUserQueryProfile.myUserInteger")); + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here no user profile is set before it is assigned in the query + */ + public void testTypedAssignmentOfQueryProfileReferencesNonStrictThroughQuery() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(profile); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cprofile = cRegistry.getComponent("test"); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cprofile); + + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("myUserQueryProfile.myUserInteger")); + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here no user profile is set before it is assigned in the query + */ + public void testTypedAssignmentOfQueryProfileReferencesStrictThroughQuery() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(userStrict); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(profile); + registry.register(newUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1",query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845,query.properties().get("myUserQueryProfile.myUserInteger")); + + try { + query.properties().set("myUserQueryProfile.someKey","value"); + fail("Should not be allowed to set this"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'myUserQueryProfile.someKey' to 'value': 'someKey' is not declared in query profile type 'userStrict', and the type is strict", + Exceptions.toMessageString(e)); + } + + } + + public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + String tensorString = "{{a:a1, b:b1}:1.0, {a:a2}:2.0}}"; + Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + + "=" + encode(tensorString), + com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); + assertEquals(0, query.errors().size()); + assertEquals(MapTensor.from(tensorString), query.properties().get("ranking.features.query(myTensor1)")); + assertEquals(MapTensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); + } + + private String encode(String s) throws UnsupportedEncodingException { + return URLEncoder.encode(s, "utf8"); + } + + public void testIllegalStrictAssignmentFromRequest() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(userStrict); + + profile.set("myUserQueryProfile", newUser, registry); + + try { + new Query( + HttpRequest.createTestRequest( + "?myUserQueryProfile.nondeclared=someValue", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + profile.compile(null)); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString("Could not set 'myUserQueryProfile.nondeclared' to 'someValue': 'nondeclared' is not declared in query profile type 'userStrict', and the type is strict")); + } + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here there exists a user profile already, and then a new one is overwritten. + * The whole thing is accessed through a two levels of nontyped top-level profiles + */ + public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("typeProfile",test, registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + test.set("myUserQueryProfile",myUser, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + registry.register(myUser); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserInteger")); + } + + /** + * Same as previous test but using the untyped myQueryProfile reference instead of the typed myUserQueryProfile + */ + public void testAnonTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("typeProfile",test, registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + test.set("myQueryProfile",myUser, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + registry.register(myUser); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1",query.properties().get("subMap.typeProfile.myQueryProfile.myUserString")); + assertEquals(845,query.properties().get("subMap.typeProfile.myQueryProfile.myUserInteger")); + } + + /** + * Tests setting a illegal value in a strict profile nested under untyped maps + */ + public void testSettingValueInStrictTypeNestedUnderUntypedMaps() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(typeStrict); + subMap.set("typeProfile",test, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + try { + new Query( + HttpRequest.createTestRequest( + "?subMap.typeProfile.someValue=value", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + cRegistry.getComponent("topMap")); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString("Could not set 'subMap.typeProfile.someValue' to 'value': 'someValue' is not declared in query profile type 'testtypeStrict', and the type is strict")); + } + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here, no user profile is set before it is assigned in the query + * The whole thing is accessed through a two levels of nontyped top-level profiles + */ + public void testTypedSettingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("typeProfile",test, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserInteger")); + } + + private void assertWrongType(QueryProfile profile,String typeName,String name,Object value) { + try { + profile.set(name,value, registry); + fail("Should fail setting " + name + " to " + value); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set '" + name + "' to '" + value + "': '" + value + "' is not a " + typeName, + Exceptions.toMessageString(e)); + } + } + + private void assertNotPermitted(QueryProfile profile,String name,Object value) { + String localName = new CompoundName(name).last(); + try { + profile.set(name, value, registry); + fail("Should fail setting " + name + " to " + value); + } + catch (IllegalArgumentException e) { + assertTrue(Exceptions.toMessageString(e).startsWith("Could not set '" + name + "' to '" + value + "': '" + localName + "' is not declared")); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java new file mode 100644 index 00000000000..d68745b0d57 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.properties.test; + +import com.yahoo.processing.request.properties.PropertyMap; + +import java.util.Collections; +import java.util.List; + +/** + * @author bratseth + */ +public class PropertyMapTestCase extends junit.framework.TestCase { + + public void testCloning() { + PropertyMap map=new PropertyMap(); + map.set("clonable",new ClonableObject()); + map.set("nonclonable",new NonClonableObject()); + map.set("clonableArray",new ClonableObject[] {new ClonableObject()}); + map.set("nonclonableArray",new NonClonableObject[] {new NonClonableObject()}); + map.set("clonableList", Collections.singletonList(new ClonableObject())); + map.set("nonclonableList", Collections.singletonList(new NonClonableObject())); + assertNotNull(map.get("clonable")); + assertNotNull(map.get("nonclonable")); + + PropertyMap mapClone=map.clone(); + assertTrue(map.get("clonable") != mapClone.get("clonable")); + assertTrue(map.get("nonclonable") == mapClone.get("nonclonable")); + + assertTrue(map.get("clonableArray") != mapClone.get("clonableArray")); + assertTrue(first(map.get("clonableArray")) != first(mapClone.get("clonableArray"))); + assertTrue(first(map.get("nonclonableArray")) == first(mapClone.get("nonclonableArray"))); + } + + private Object first(Object object) { + if (object instanceof Object[]) + return ((Object[])object)[0]; + if (object instanceof List) + return ((List)object).get(0); + throw new IllegalArgumentException(); + } + + public static class ClonableObject implements Cloneable { + + @Override + public ClonableObject clone() { + try { + return (ClonableObject)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + } + + private static class NonClonableObject { + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java new file mode 100644 index 00000000000..ff924bb59ea --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.properties.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.test.QueryTestCase; + +/** + * Tests that dimension arguments in queries are transferred correctly to dimension values + * + * @author bratseth + */ +public class RequestContextPropertiesTestCase extends junit.framework.TestCase { + + public void testIt() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a","a-default", (QueryProfileRegistry)null); + p.set("a","a-x1",new String[] {"x1"}, null); + p.set("a","a-+x1",new String[] {"+x1"}, null); + Query q1 = new Query(QueryTestCase.httpEncode("?query=foo"), p.compile(null)); + assertEquals("a-default",q1.properties().get("a")); + Query q2 = new Query(QueryTestCase.httpEncode("?query=foo&x=x1"),p.compile(null)); + assertEquals("a-x1",q2.properties().get("a")); + Query q3 = new Query(QueryTestCase.httpEncode("?query=foo&x=+x1"),p.compile(null)); + assertEquals("a-+x1",q3.properties().get("a")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java new file mode 100644 index 00000000000..35185b8d8f6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.properties.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.Arrays; +import java.util.HashSet; + +import com.yahoo.processing.request.properties.PropertyMap; +import org.junit.Test; + +import com.yahoo.search.query.properties.SubProperties; + +/** + * @author Arne Bergene Fossaa + */ +public class SubPropertiesTestCase { + + @Test + public void testSubProperties() { + PropertyMap map = new PropertyMap() {{ + set("a.e","1"); + set("a.f",2); + set("b.e","3"); + set("f",3); + set("e","2"); + set("d","a"); + }}; + + SubProperties sub = new SubProperties("a", map); + assertEquals("1",sub.get("e")); + assertEquals(2,sub.get("f")); + assertNull(sub.get("d")); + assertEquals(new HashSet<>(Arrays.asList("e", "f")), sub.listProperties("").keySet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java new file mode 100644 index 00000000000..8f3ac661d0f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.Execution.Context; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; + +/** + * Fine grained testing of RewriterFeatures for easier testing of innards. + */ +public class RewriterFeaturesTestCase { + + private static final String ASCII_ELLIPSIS = "..."; + + @Test + public final void testConvertStringToQTree() { + Execution placeholder = new Execution(Context.createContextStub()); + SpecialTokenRegistry tokenRegistry = new SpecialTokenRegistry( + new SpecialtokensConfig( + new SpecialtokensConfig.Builder() + .tokenlist(new Tokenlist.Builder().name( + "default").tokens( + new Tokens.Builder().token(ASCII_ELLIPSIS))))); + placeholder.context().setTokenRegistry(tokenRegistry); + Query query = new Query(); + query.getModel().setExecution(placeholder); + Item parsed = RewriterFeatures.convertStringToQTree(query, "a b c " + + ASCII_ELLIPSIS); + assertSame(AndItem.class, parsed.getClass()); + assertEquals(4, ((CompositeItem) parsed).getItemCount()); + assertEquals(ASCII_ELLIPSIS, ((CompositeItem) parsed).getItem(3).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java new file mode 100644 index 00000000000..c89ca16a265 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.util.*; +import java.io.File; + +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; +import com.yahoo.search.query.rewrite.RewritesConfig; + +/** + * Test Cases for GenericExpansionRewriter + * + * @author karenlee@yahoo-inc.com + */ +public class GenericExpansionRewriterTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_generic_expansion_rewriter.cfg"; + private final String GENERIC_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "generic_expansion.fsa"; + private final String REWRITER_NAME = GenericExpansionRewriter.REWRITER_NAME; + + /** + * Load the GenericExpansionRewriterSearcher and prepare the + * execution object + */ + protected void setUp() { + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(CONFIG_PATH); + HashMap fileList = new HashMap<>(); + fileList.put(GenericExpansionRewriter.GENERIC_EXPAND_DICT, new File(GENERIC_EXPAND_DICT_PATH)); + GenericExpansionRewriter searcher = new GenericExpansionRewriter(config, fileList); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searcher); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public GenericExpansionRewriterTestCase(String name) { + super(name); + } + + /** + * MaxRewrites=3, PartialPhraseMatch is on, type=adv case + */ + public void testPartialPhraseMaxRewriteAdvType() { + utils.assertRewrittenQuery("?query=(modern new york city travel phone number) OR (travel agency) OR travel&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true&" + + REWRITER_NAME + "." + RewriterConstants.MAX_REWRITES + "=3", + "query 'OR (AND modern (OR (AND rewrite11 rewrite12) rewrite2 rewrite3 " + + "(AND new york city travel)) (OR pn (AND phone number))) (OR ta (AND travel agency)) " + + "(OR tr travel)'"); + } + + /** + * PartialPhraseMatch is off, type=adv case + */ + public void testPartialPhraseNoMaxRewriteAdvType() { + utils.assertRewrittenQuery("?query=(modern new york city travel phone number) OR (travel agency) OR travel&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'OR (AND modern new york city travel phone number) " + + "(OR ta (AND travel agency)) (OR tr travel)'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, type=adv, added filter case + */ + public void testFullPhraseNoMaxRewriteAdvTypeFilter() { + utils.assertRewrittenQuery("?query=ca OR (modern new york city travel phone number) OR (travel agency) OR travel&" + + "type=adv&filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'RANK (OR (OR california ca) (AND modern new york city travel phone number) " + + "(OR ta (AND travel agency)) (OR tr travel)) |citystate:santa |clara |ca'"); + } + + /** + * MaxRewrites=0 (i.e No MaxRewrites), PartialPhraseMatch is on, type=adv, added filter case + */ + public void testPartialPhraseNoMaxRewriteAdvTypeFilter() { + utils.assertRewrittenQuery("?query=ca OR (modern new york city travel phone number) OR (travel agency) OR travel&" + + "type=adv&filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.MAX_REWRITES + "=0", + "query 'RANK (OR (OR california ca) (AND modern (OR \"rewrite11 rewrite12\" " + + "rewrite2 rewrite3 rewrite4 rewrite5 (AND new york city travel)) " + + "(OR pn (AND phone number))) (OR ta (AND travel agency)) (OR tr travel)) " + + "|citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, single word, added filter case + */ + public void testFullPhraseNoMaxRewriteSingleWordFilter() { + utils.assertRewrittenQuery("?query=ca&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'RANK (OR california ca) |citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, single word, added filter case + */ + public void testPartialPhraseNoMaxRewriteSingleWordFilter() { + utils.assertRewrittenQuery("?query=ca&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (OR california ca) |citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, multi word, added filter case + */ + public void testFullPhraseNoMaxRewriteMultiWordFilter() { + utils.assertRewrittenQuery("?query=travel agency&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'RANK (OR ta (AND travel agency)) |citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, multi word, added filter case + */ + public void testPartialPhraseNoMaxRewriteMultiWordFilter() { + utils.assertRewrittenQuery("?query=modern new york city travel phone number&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (AND modern (OR (AND rewrite11 rewrite12) rewrite2 rewrite3 " + + "rewrite4 rewrite5 (AND new york city travel)) (OR pn (AND phone number))) " + + "|citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, single word + */ + public void testFullPhraseNoMaxRewriteSingleWord() { + utils.assertRewrittenQuery("?query=ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'OR california ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, single word + */ + public void testPartialPhraseNoMaxRewriteSingleWord() { + utils.assertRewrittenQuery("?query=ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'OR california ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, multi word + */ + public void testFullPhraseNoMaxRewriteMultiWord() { + utils.assertRewrittenQuery("?query=travel agency&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'OR ta (AND travel agency)'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, multi word, no full match + */ + public void testFullPhraseNoMaxRewriteMultiWordNoMatch() { + utils.assertRewrittenQuery("?query=nyc travel agency&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'AND nyc travel agency'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, multi word + */ + public void testPartialPhraseNoMaxRewriteMultiWord() { + utils.assertRewrittenQuery("?query=modern new york city travel phone number&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'AND modern (OR (AND rewrite11 rewrite12) rewrite2 rewrite3 rewrite4 rewrite5 "+ + "(AND new york city travel)) (OR pn (AND phone number))'"); + } + + /** + * Matching multiple word in RANK subtree + * Dictionary contain the word "travel agency", the word "agency" and the word "travel" + * Should rewrite travel but not travel agency in this case + */ + public void testPartialPhraseMultiWordRankTree() { + utils.assertRewrittenQuery("?query=travel RANK agency&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (OR tr travel) agency'"); + } + + /** + * Matching multiple word in RANK subtree + * Dictionary contain the word "travel agency", the word "agency" and the word "travel" + * Should rewrite travel but not travel agency in this case + */ + public void testFullPhraseMultiWordRankTree() { + utils.assertRewrittenQuery("?query=travel RANK agency&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (OR tr travel) agency'"); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java new file mode 100644 index 00000000000..b5b4acff459 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.intent.model.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; + +/** + * Test Cases for MisspellRewriter + * + * @author karenlee@yahoo-inc.com + */ +public class MisspellRewriterTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + public final String REWRITER_NAME = MisspellRewriter.REWRITER_NAME; + + /** + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + protected void setUp() { + MisspellRewriter searcher = new MisspellRewriter(); + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searcher); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public MisspellRewriterTestCase(String name) { + super(name); + } + + /** + * QSSRewrite and QSSSuggest are on + * QLAS returns spell correction: qss_rw=0.9 qss_sugg=1.0 + */ + public void testQSSRewriteQSSSuggestWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw", 0.9, + true, false), + utils.createInterpretation("will smith sugg", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true", + "query 'OR (AND willl smith) (AND will smith sugg)'", + intentModel); + } + + /** + * QSSRewrite is on + * QLAS returns spell correction: qss_rw=0.9 qss_rw=0.9 qss_sugg=1.0 + */ + public void testQSSRewriteWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw1", 0.9, + true, false), + utils.createInterpretation("will smith rw2", 0.9, + true, false), + utils.createInterpretation("will smith sugg", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true", + "query 'OR (AND willl smith) (AND will smith rw1)'", + intentModel); + } + + /** + * QSSSuggest is on + * QLAS returns spell correction: qss_rw=1.0 qss_sugg=0.9 qss_sugg=0.8 + */ + public void testQSSSuggWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw", 1.0, + true, false), + utils.createInterpretation("will smith sugg1", 0.9, + false, true), + utils.createInterpretation("will smith sugg2", 0.8, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true", + "query 'OR (AND willl smith) (AND will smith sugg1)'", + intentModel); + } + + /** + * QSSRewrite and QSSSuggest are off + * QLAS returns spell correction: qss_rw=1.0 qss_sugg=1.0 + */ + public void testFeautureOffWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw", 1.0, + true, false), + utils.createInterpretation("will smith sugg", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith", + "query 'AND willl smith'", + intentModel); + } + + /** + * QSSRewrite and QSSSuggest are on + * QLAS returns no spell correction + */ + public void testQSSRewriteQSSSuggWithoutRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("use diff query for testing", 1.0, + false, false), + utils.createInterpretation("use diff query for testing", 1.0, + false, false)); + + utils.assertRewrittenQuery("?query=will+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true", + "query 'AND will smith'", + intentModel); + } + + /** + * IntentModel is null + * It should throw exception + */ + public void testNullIntentModelException() { + try { + RewriterUtils.getSpellCorrected(new Query("willl smith"), true, true); + fail(); + } catch (RuntimeException e) { + } + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java new file mode 100644 index 00000000000..ecd798caacd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.util.*; +import java.io.File; + +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; +import com.yahoo.search.query.rewrite.RewritesConfig; + +/** + * Test Cases for NameRewriter + * + * @author karenlee@yahoo-inc.com + */ +public class NameRewriterTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String NAME_ENTITY_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "name_rewriter_entity.fsa"; + private final String REWRITER_NAME = NameRewriter.REWRITER_NAME; + + /** + * Load the NameRewriterSearcher and prepare the + * execution object + */ + protected void setUp() { + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(CONFIG_PATH); + HashMap fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter searcher = new NameRewriter(config, fileList); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searcher); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public NameRewriterTestCase(String name) { + super(name); + } + + /** + * RewritesAsEquiv and OriginalAsUnit are on + */ + public void testRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query 'OR \"will smith\" (AND will smith movies) " + + "(AND will smith news) (AND will smith imdb) " + + "(AND will smith lyrics) (AND will smith dead) " + + "(AND will smith nfl) (AND will smith new movie hancock) " + + "(AND will smith biography)'"); + } + + /** + * RewritesAsEquiv is on + */ + public void testRewritesAsEquiv() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&", + "query 'OR (AND will smith) (AND will smith movies) " + + "(AND will smith news) (AND will smith imdb) " + + "(AND will smith lyrics) (AND will smith dead) " + + "(AND will smith nfl) (AND will smith new movie hancock) " + + "(AND will smith biography)'"); + } + + /** + * Complex query with more than two levels for RewritesAsEquiv is on case + * Should not rewrite + */ + public void testComplextQueryRewritesAsEquiv() { + utils.assertRewrittenQuery("?query=((will smith) OR (willl smith)) AND (tom cruise)&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&", + "query 'AND (OR (AND will smith) (AND willl smith)) (AND tom cruise)'"); + } + + /** + * Single word query for RewritesAsEquiv and OriginalAsUnit on case + */ + public void testSingleWordForRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=obama&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query 'OR obama (AND obama \"nobel peace prize\") " + + "(AND obama wiki) (AND obama nobel prize) " + + "(AND obama nobel peace prize) (AND obama wears mom jeans) " + + "(AND obama sucks) (AND obama news) (AND malia obama) " + + "(AND obama speech) (AND obama nobel) (AND obama wikipedia) " + + "(AND barack obama biography) (AND obama snl) " + + "(AND obama peace prize) (AND michelle obama) (AND barack obama)'"); + } + + /** + * RewritesAsUnitEquiv and OriginalAsUnitEquiv are on + */ + public void testRewritesAsUnitEquivAndOriginalAsUnitEquiv() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + + "=true", + "query 'OR (AND will smith) \"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'"); + } + + /** + * Single word query for RewritesAsUnitEquiv and OriginalAsUnitEquiv on case + */ + public void testSingleWordForRewritesAsUnitEquivAndOriginalAsUnitEquiv() { + utils.assertRewrittenQuery("?query=obama&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + + "=true", + "query 'OR obama \"obama nobel peace prize\" " + + "\"obama wiki\" \"obama nobel prize\" " + + "\"obama wears mom jeans\" " + + "\"obama sucks\" \"obama news\" \"malia obama\" " + + "\"obama speech\" \"obama nobel\" \"obama wikipedia\" " + + "\"barack obama biography\" \"obama snl\" " + + "\"obama peace prize\" \"michelle obama\" \"barack obama\"'"); + } + + /** + * Boosting only query (n/a as rewrite in FSA) + * for RewritesAsEquiv and OriginalAsUnit on case + */ + public void testBoostingQueryForRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=angelina jolie&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query '\"angelina jolie\"'"); + } + + /** + * No match in FSA for the query + * RewritesAsEquiv and OriginalAsUnit on case + */ + public void testFSANoMatchForRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=tom cruise&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query 'AND tom cruise'"); + } + + /** + * RewritesAsUnitEquiv is on + */ + public void testRewritesAsUnitEquiv() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true", + "query 'OR (AND will smith) \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'"); + } + + /** + * RewritesAsUnitEquiv is on and MaxRewrites is set to 2 + */ + public void testRewritesAsUnitEquivAndMaxRewrites() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + REWRITER_NAME + "." + RewriterConstants.MAX_REWRITES + "=2", + "query 'OR (AND will smith) \"will smith movies\" " + + "\"will smith news\"'"); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java new file mode 100644 index 00000000000..f3eaf6ae582 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.io.File; +import java.util.*; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.intent.model.*; +import com.yahoo.search.query.rewrite.RewritesConfig; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; + +/** + * Generic Test Cases for QueryRewriteSearcher + * + * @author karenlee@yahoo-inc.com + */ +public class QueryRewriteSearcherTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String NAME_REWRITER_CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String FAKE_FSA_CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_rewriter_fake_fsa.cfg"; + private final String NAME_ENTITY_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "name_rewriter_entity.fsa"; + private final String FAKE_FSA_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String NAME_REWRITER_NAME = NameRewriter.REWRITER_NAME; + private final String MISSPELL_REWRITER_NAME = MisspellRewriter.REWRITER_NAME; + + /** + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + protected void setUp() { + // Instantiate Name Rewriter + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(NAME_REWRITER_CONFIG_PATH); + HashMap fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter nameRewriter = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create a chain of two rewriters + ArrayList searchers = new ArrayList<>(); + searchers.add(misspellRewriter); + searchers.add(nameRewriter); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searchers); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public QueryRewriteSearcherTestCase(String name) { + super(name); + } + + /** + * Invalid FSA config path + * Query will be passed to next rewriter + */ + public void testInvalidFSAConfigPath() { + // Instantiate Name Rewriter with fake FSA path + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(FAKE_FSA_CONFIG_PATH); + HashMap fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(FAKE_FSA_PATH)); + NameRewriter nameRewriterWithFakePath = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create a chain of two rewriters + ArrayList searchers = new ArrayList<>(); + searchers.add(misspellRewriter); + searchers.add(nameRewriterWithFakePath); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searchers); + QueryRewriteSearcherTestUtils utilsWithFakePath = new QueryRewriteSearcherTestUtils(execution); + + utilsWithFakePath.assertRewrittenQuery("?query=will smith&" + + NAME_REWRITER_NAME + "." + + RewriterConstants.REWRITES_AS_UNIT_EQUIV + "=true", + "query 'AND will smith'"); + } + + /** + * IntentModel is null and rewriter throws exception + * It should skip to the next rewriter + */ + public void testExceptionInRewriter() { + utils.assertRewrittenQuery("?query=will smith&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'"); + } + + /** + * Two rewrites in chain + * Query will be rewritten twice + */ + public void testTwoRewritersInChain() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("wills smith", 0.9, + true, false), + utils.createInterpretation("will smith", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND willl smith) (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'", + intentModel); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java new file mode 100644 index 00000000000..74322a4d980 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import com.yahoo.search.test.QueryTestCase; +import junit.framework.Assert; +import java.util.*; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.RewritesConfig; +import com.yahoo.search.intent.model.*; +import com.yahoo.text.interpretation.Modification; +import com.yahoo.text.interpretation.Interpretation; +import com.yahoo.text.interpretation.Annotations; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.component.chain.Chain; + +/** + * Test utilities for QueryRewriteSearcher + * + * @author karenlee@yahoo-inc.com + */ +public class QueryRewriteSearcherTestUtils { + + private Execution execution; + + /** + * Constructor for this class + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + public QueryRewriteSearcherTestUtils(Execution execution) { + this.execution = execution; + } + + + /** + * Create config object based on config path + * + * @param configPath path for the searcher config + */ + public static RewritesConfig createConfigObj(String configPath) { + ConfigGetter getter = new ConfigGetter<>(RewritesConfig.class); + RewritesConfig config = getter.getConfig(configPath); + return config; + } + + /** + * Create execution object based on searcher + * + * @param searcher searcher to be added to the search chain + */ + public static Execution createExecutionObj(Searcher searcher) { + @SuppressWarnings("deprecation") + Chain searchChain = new Chain<>(searcher); + Execution myExecution = new Execution(searchChain, Execution.Context.createContextStub()); + return myExecution; + } + + /** + * Create execution object based on searchers + * + * @param searchers list of searchers to be added to the search chain + */ + public static Execution createExecutionObj(List searchers) { + @SuppressWarnings("deprecation") + Chain searchChain = new Chain<>(searchers); + Execution myExecution = new Execution(searchChain, Execution.Context.createContextStub()); + return myExecution; + } + + /** + * Compare the rewritten query returned after executing + * the origQuery against the provided finalQuery + * @param origQuery query to be passed to Query object + * e.g. "?query=will%20smith" + * @param finalQuery expected final query from result.getQuery() + * e.g. "query 'AND will smith'" + */ + public void assertRewrittenQuery(String origQuery, String finalQuery) { + Query query = new Query(QueryTestCase.httpEncode(origQuery)); + Result result = execution.search(query); + Assert.assertEquals(finalQuery, result.getQuery().toString()); + } + + /** + * Set the provided intent model + * Compare the rewritten query returned after executing + * the origQuery against the provided finalQuery + * @param origQuery query to be passed to Query object + * e.g. "?query=will%20smith" + * @param finalQuery expected final query from result.getQuery() + * e.g. "query 'AND will smith'" + * @param intentModel IntentModel to be added to the Query + */ + public void assertRewrittenQuery(String origQuery, String finalQuery, IntentModel intentModel) { + Query query = new Query(origQuery); + intentModel.setTo(query); + Result result = execution.search(query); + Assert.assertEquals(finalQuery, result.getQuery().toString()); + } + + /** + * Create a new interpretation with modification that + * contains the passed in query and score + * @param spellRewrite query to be used as modification + * @param score score to be used as modification score + * @param isQSSRW whether the modification is qss_rw + * @param isQSSSugg whether the modification is qss_sugg + * @return newly created interpretation with modification + */ + public Interpretation createInterpretation(String spellRewrite, double score, + boolean isQSSRW, boolean isQSSSugg) { + Modification modification = new Modification(spellRewrite); + Annotations annotation = modification.getAnnotation(); + annotation.put("score", score); + if(isQSSRW) + annotation.put("qss_rw", true); + if(isQSSSugg) + annotation.put("qss_sugg", true); + Interpretation interpretation = new Interpretation(modification); + return interpretation; + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java new file mode 100644 index 00000000000..608706605f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.util.*; +import java.io.File; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.query.rewrite.RewritesConfig; +import com.yahoo.search.intent.model.*; +import com.yahoo.component.chain.Chain; + +/** + * Test Cases for SearchChainDispatcherSearcher + * + * @author karenlee@yahoo-inc.com + */ +public class SearchChainDispatcherSearcherTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String NAME_REWRITER_CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String NAME_ENTITY_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "name_rewriter_entity.fsa"; + private final String NAME_REWRITER_NAME = NameRewriter.REWRITER_NAME; + private final String MISSPELL_REWRITER_NAME = MisspellRewriter.REWRITER_NAME; + private final String US_MARKET_SEARCH_CHAIN = "us_qrw"; + + + /** + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + @SuppressWarnings("deprecation") + protected void setUp() { + // Instantiate Name Rewriter + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(NAME_REWRITER_CONFIG_PATH); + HashMap fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter nameRewriter = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create market search chain of two rewriters + ArrayList searchers = new ArrayList<>(); + searchers.add(misspellRewriter); + searchers.add(nameRewriter); + Chain marketSearchChain = new Chain<>(US_MARKET_SEARCH_CHAIN, searchers); + + // Add market search chain to the registry + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(marketSearchChain); + + // Instantiate Search Chain Dispatcher Searcher + SearchChainDispatcherSearcher searchChainDispatcher = new SearchChainDispatcherSearcher(); + + // Create a chain containing only the dispatcher + Chain mainSearchChain = new Chain<>(searchChainDispatcher); + Execution execution = new Execution(mainSearchChain, Execution.Context.createContextStub(registry, null)); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public SearchChainDispatcherSearcherTestCase(String name) { + super(name); + } + + /** + * Execute the market chain + * Query will be rewritten twice + */ + public void testMarketChain() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("wills smith", 0.9, + true, false), + utils.createInterpretation("will smith", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&QRWChain=" + US_MARKET_SEARCH_CHAIN + "&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND willl smith) (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'", + intentModel); + } + + /** + * Market chain is not valid + * Query will be passed to next rewriter + */ + public void testInvalidMarketChain() { + utils.assertRewrittenQuery("?query=will smith&QRWChain=abc&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true", + "query 'AND will smith'"); + } + + /** + * Empty market chain value + * Query will be passed to next rewriter + */ + public void testEmptyMarketChain() { + utils.assertRewrittenQuery("?query=will smith&QRWChain=&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true", + "query 'AND will smith'"); + } + + /** + * Searchers down the chain after SearchChainDispatcher + * should be executed + */ + @SuppressWarnings("deprecation") + public void testChainContinuation() { + // Instantiate Name Rewriter + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(NAME_REWRITER_CONFIG_PATH); + HashMap fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter nameRewriter = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create market search chain of only misspell rewriter + Chain marketSearchChain = new Chain<>(US_MARKET_SEARCH_CHAIN, misspellRewriter); + + // Add market search chain to the registry + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(marketSearchChain); + + // Instantiate Search Chain Dispatcher Searcher + SearchChainDispatcherSearcher searchChainDispatcher = new SearchChainDispatcherSearcher(); + + // Create a chain containing the dispatcher and the name rewriter + ArrayList searchers = new ArrayList<>(); + searchers.add(searchChainDispatcher); + searchers.add(nameRewriter); + + // Create a chain containing only the dispatcher + Chain mainSearchChain = new Chain<>(searchers); + Execution execution = new Execution(mainSearchChain, Execution.Context.createContextStub(registry, null)); + new QueryRewriteSearcherTestUtils(execution); + + IntentModel intentModel = new IntentModel( + utils.createInterpretation("wills smith", 0.9, + true, false), + utils.createInterpretation("will smith", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&QRWChain=" + US_MARKET_SEARCH_CHAIN + "&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND willl smith) (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'", + intentModel); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa new file mode 100644 index 00000000000..2fb1db0cde2 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa differ diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa new file mode 100644 index 00000000000..0507632d2d9 Binary files /dev/null and b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa differ diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg new file mode 100644 index 00000000000..7b86c1385d7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg @@ -0,0 +1,3 @@ +fsaDict[1] +fsaDict[0].name GenericExpansion +fsaDict[0].path dictionaries/GenericExpansionRewriter.fsa diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg new file mode 100644 index 00000000000..ef0b82eb64d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg @@ -0,0 +1,3 @@ +fsaDict[1] +fsaDict[0].name NameEntityExpansion +fsaDict[0].path dictionaries/NameRewriter.fsa diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg new file mode 100644 index 00000000000..75cba5f8c90 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg @@ -0,0 +1,3 @@ +fsaDict[1] +fsaDict[0].name NameEntityExpansion +fsaDict[0].path dummyFSAPath diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java new file mode 100644 index 00000000000..301438a5e41 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.prelude.query.Item; +import com.yahoo.search.Query; +import com.yahoo.search.query.Model; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Arrays; +import java.util.LinkedHashSet; + + +/** + * @author Arne Bergene Fossaa + */ +public class ModelTestCase extends junit.framework.TestCase { + + String oldConfigId; + + public void setUp() { + oldConfigId = System.getProperty("config.id"); + System.setProperty("config.id", "file:src/test/java/com/yahoo/prelude/test/fieldtypes/field-info.cfg"); + } + + + public void tearDown() { + if (oldConfigId == null) + System.getProperties().remove("config.id"); + else + System.setProperty("config.id", oldConfigId); + } + + public void testCopyParameters() { + Query q1 = new Query("?query=test1&filter=test2&defidx=content&default-index=lala&encoding=iso8859-1"); + Query q2 = q1.clone(); + Model r1 = q1.getModel(); + Model r2 = q2.getModel(); + assertTrue(r1 != r2); + assertEquals(r1,r2); + assertEquals("test1",r2.getQueryString()); + } + + public void testSetQuery() { + Query q1 = new Query("?query=test1"); + Item r1 = q1.getModel().getQueryTree(); + q1.properties().set("query","test2"); + q1.getModel().setQueryString(q1.getModel().getQueryString()); // Force reparse + assertNotSame(r1,q1.getModel().getQueryTree()); + q1.properties().set("query","test1"); + q1.getModel().setQueryString(q1.getModel().getQueryString()); // Force reparse + assertEquals(r1,q1.getModel().getQueryTree()); + } + + + public void testSetofSetters() { + Query q1 = new Query("?query=test1&encoding=iso-8859-1&language=en&default-index=subject&filter=" + enc("\u00C5")); + Model r1 = q1.getModel(); + assertEquals(r1.getQueryString(), "test1"); + assertEquals("iso-8859-1", r1.getEncoding()); + assertEquals("\u00C5", r1.getFilter()); + assertEquals("subject", r1.getDefaultIndex()); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + public void testSearchPath() { + assertEquals("c6/r8",new Query("?query=test1&model.searchPath=c6/r8").getModel().getSearchPath()); + assertEquals("c6/r8",new Query("?query=test1&searchpath=c6/r8").getModel().getSearchPath()); + } + + public void testClone() { + Query q= new Query(); + Model sr = new Model(q); + sr.setRestrict("music, cheese,other"); + sr.setSources("cluster1"); + assertEquals(sr.getSources(), new LinkedHashSet<>(Arrays.asList(new String[]{"cluster1"}))); + assertEquals(sr.getRestrict(),new LinkedHashSet<>(Arrays.asList(new String[]{"cheese","music","other"}))); + } + + public void testEquals() { + Query q = new Query(); + Model sra = new Model(q); + sra.setRestrict("music,cheese"); + sra.setSources("cluster1,cluster2"); + + Model srb = new Model(q); + srb.setRestrict(" cheese , music"); + srb.setSources("cluster1,cluster2"); + assertEquals(sra,srb); + srb.setRestrict("music,cheese"); + assertNotSame(sra,srb); + } + + public void testSearchRestrictQueryParameters() { + Query query=new Query("?query=test&search=news,archive&restrict=fish,bird"); + assertTrue(query.getModel().getSources().contains("news")); + assertTrue(query.getModel().getSources().contains("archive")); + assertEquals(2,query.getModel().getSources().size()); + assertTrue(query.getModel().getRestrict().contains("fish")); + assertTrue(query.getModel().getRestrict().contains("bird")); + assertEquals(2,query.getModel().getRestrict().size()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java new file mode 100644 index 00000000000..2f304693d1c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +import static com.yahoo.jdisc.http.HttpRequest.Method; + +/** + * @author Jon Bratseth + */ +public class ParametersTestCase extends junit.framework.TestCase { + + public void testSettingRankProperty() { + Query query=new Query("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)"); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankPropertyAsAlias() { + Query query=new Query("?query=test&rankproperty.dotProduct.X=(a:1,b:2)"); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankFeature() { + Query query=new Query("?query=test&ranking.features.matches=3"); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public void testSettingRankFeatureAsAlias() { + Query query=new Query("?query=test&rankfeature.matches=3"); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public void testSettingRankPropertyWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankPropertyAsAliasWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&rankproperty.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankFeatureWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&ranking.features.matches=3", Method.GET), createProfile()); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public void testSettingRankFeatureAsAliasWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&rankfeature.matches=3", Method.GET), createProfile()); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public CompiledQueryProfile createProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("test"); + profile.set("model.filter", "+year:2001", registry); + profile.set("model.language", "en", registry); + return registry.compile().findQueryProfile("test"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java new file mode 100644 index 00000000000..3222bd99cd9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.prelude.query.Highlight; +import com.yahoo.search.Query; +import com.yahoo.search.query.Presentation; + +/** + * @author Arne Bergene Fossaa + */ +public class PresentationTestCase extends junit.framework.TestCase { + + + public void testClone() { + Query q= new Query(""); + Presentation p = new Presentation(q); + p.setBolding(true); + Highlight h = new Highlight(); + h.addHighlightTerm("date","today"); + p.setHighlight(h); + Presentation pc = (Presentation)p.clone(); + h.addHighlightTerm("title","Hello"); + assertTrue(pc.getBolding()); + pc.getHighlight().getHighlightItems(); + assertTrue(pc.getHighlight().getHighlightItems().containsKey("date")); + assertFalse(pc.getHighlight().getHighlightItems().containsKey("title")); + + assertEquals(p,pc); + + } + + + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java new file mode 100644 index 00000000000..7a9f5ce59b6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.search.Query; + +/** + * @author Jon Bratseth + */ +public class QueryCloneMicroBenchmark { + + public void benchmark() { + int runs = 10000; + + Query query = createQuery(); + for (int i = 0; i<100000; i++) // yes, this much is needed + query.clone(); + long startTime = System.currentTimeMillis(); + for (int i = 0; iArne Bergene Fossaa + */ +public class RankingTestCase extends junit.framework.TestCase { + + /** tests setting rank feature values */ + public void testRankFeatures() { + // Check initializing from query + Query query = new Query("?query=test&ranking.features.query(name)=0.1&ranking.features.fieldMatch(foo)=0.2"); + assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + assertEquals("0.2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals("{\"query(name)\":\"0.1\",\"fieldMatch(foo)\":\"0.2\"}", query.getRanking().getFeatures().toString()); + + // Test cloning + Query clone = query.clone(); + assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + assertEquals("0.2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + + // Check programmatic setting + that the clone really has a separate object + assertFalse(clone.getRanking().getFeatures() == query.getRanking().getFeatures()); + clone.properties().set("ranking.features.query(name)","0.3"); + assertEquals("0.3", clone.getRanking().getFeatures().get("query(name)")); + assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + + // Check getting + assertEquals("0.3",clone.properties().get("ranking.features.query(name)")); + + // Check map access + assertEquals(2, query.getRanking().getFeatures().asMap().size()); + assertEquals("0.2", query.getRanking().getFeatures().asMap().get("fieldMatch(foo)")); + query.getRanking().getFeatures().asMap().put("fieldMatch(foo)", "0.3"); + assertEquals("0.3", query.getRanking().getFeatures().get("fieldMatch(foo)")); + } + + //This test is order dependent. Fix this!! + public void test_setting_rank_feature_values() { + // Check initializing from query + Query query = new Query("?query=test&ranking.properties.foo=bar1&ranking.properties.foo2=bar2&ranking.properties.other=10"); + assertEquals("bar1", query.getRanking().getProperties().get("foo").get(0)); + assertEquals("bar2", query.getRanking().getProperties().get("foo2").get(0)); + assertEquals("10", query.getRanking().getProperties().get("other").get(0)); + assertEquals("{\"other\":[10],\"foo\":[bar1],\"foo2\":[bar2]}", query.getRanking().getProperties().toString()); + + // Test cloning + Query clone = query.clone(); + assertFalse(clone.getRanking().getProperties() == query.getRanking().getProperties()); + assertEquals("bar1", clone.getRanking().getProperties().get("foo").get(0)); + assertEquals("bar2", clone.getRanking().getProperties().get("foo2").get(0)); + assertEquals("10", clone.getRanking().getProperties().get("other").get(0)); + + // Check programmatic setting mean addition + clone.properties().set("ranking.properties.other","12"); + assertEquals("[10, 12]", clone.getRanking().getProperties().get("other").toString()); + assertEquals("[10]", query.getRanking().getProperties().get("other").toString()); + + // Check map access + assertEquals(3, query.getRanking().getProperties().asMap().size()); + assertEquals("bar1", query.getRanking().getProperties().asMap().get("foo").get(0)); + } + + /** Test setting sorting to null does not cause an exception. */ + public void testResetSorting() { + Query q=new Query(); + q.getRanking().setSorting((Sorting)null); + q.getRanking().setSorting((String)null); + } + + /** Tests deprecated naming */ + @Test + public void testFeatureOverride() { + Query query = new Query("?query=abc&featureoverride.something=2"); + assertEquals("2", query.getRanking().getFeatures().get("something")); + } + + @Test + public void testStructuredRankProperty() { + Query query = new Query("?query=abc&rankproperty.distanceToPath(gps_position).path=(0,0,10,0,10,5,20,5)"); + assertEquals("(0,0,10,0,10,5,20,5)", query.getRanking().getProperties().get("distanceToPath(gps_position).path").get(0).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java b/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java new file mode 100644 index 00000000000..d59fd23e567 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.textserialize.item.test; + +import com.yahoo.prelude.query.*; +import com.yahoo.search.query.textserialize.item.ItemContext; +import com.yahoo.search.query.textserialize.item.ItemFormHandler; +import com.yahoo.search.query.textserialize.parser.ParseException; +import com.yahoo.search.query.textserialize.parser.Parser; +import org.junit.Test; + +import java.io.StringReader; + +import static junit.framework.Assert.assertNull; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class ParseItemTestCase { + public static Object parse(String input) throws ParseException { + ItemContext context = new ItemContext(); + Object result = new Parser(new StringReader(input.replace("'", "\"")), new ItemFormHandler(), context).start(); + context.connectItems(); + return result; + } + + @Test + public void parse_and() throws ParseException { + assertThat(parse("(AND)"), instanceOf(AndItem.class)); + } + + @Test + public void parse_and_with_children() throws ParseException { + AndItem andItem = (AndItem) parse("(AND (WORD 'first') (WORD 'second'))"); + + assertThat(andItem.getItemCount(), is(2)); + assertThat(getWord(andItem.getItem(0)), is("first")); + } + + @Test + public void parse_or() throws ParseException { + assertThat(parse("(OR)"), instanceOf(OrItem.class)); + } + + @Test + public void parse_and_not_rest() throws ParseException { + assertThat(parse("(AND-NOT-REST)"), instanceOf(NotItem.class)); + } + + @Test + public void parse_and_not_rest_with_children() throws ParseException { + NotItem notItem = (NotItem) parse("(AND-NOT-REST (WORD 'positive') (WORD 'negative'))"); + assertThat(getWord(notItem.getPositiveItem()), is("positive")); + assertThat(getWord(notItem.getItem(1)), is("negative")); + } + + @Test + public void parse_and_not_rest_with_only_negated_children() throws ParseException { + NotItem notItem = (NotItem) parse("(AND-NOT-REST null (WORD 'negated-item'))"); + assertNull(notItem.getPositiveItem()); + assertThat(notItem.getItem(1), instanceOf(WordItem.class)); + } + + @Test + public void parse_rank() throws ParseException { + assertThat(parse("(RANK (WORD 'first'))"), instanceOf(RankItem.class)); + } + + @Test + public void parse_word() throws ParseException { + WordItem wordItem = (WordItem) parse("(WORD 'text')"); + assertThat(wordItem.getWord(), is("text")); + } + + @Test(expected = IllegalArgumentException.class) + public void fail_when_word_given_multiple_strings() throws ParseException { + parse("(WORD 'one' 'invalid')"); + } + + @Test(expected = IllegalArgumentException.class) + public void fail_when_word_given_no_string() throws ParseException { + parse("(WORD)"); + } + + @Test + public void parse_int() throws ParseException { + IntItem intItem = (IntItem) parse("(INT '[42;]')"); + assertThat(intItem.getNumber(), is("[42;]")); + } + + @Test + public void parse_range() throws ParseException { + IntItem intItem = (IntItem) parse("(INT '[42;73]')"); + assertThat(intItem.getNumber(), is("[42;73]")); + } + + @Test + public void parse_range_withlimit() throws ParseException { + IntItem intItem = (IntItem) parse("(INT '[42;73;32]')"); + assertThat(intItem.getNumber(), is("[42;73;32]")); + } + + @Test + public void parse_prefix() throws ParseException { + PrefixItem prefixItem = (PrefixItem) parse("(PREFIX 'word')"); + assertThat(prefixItem.getWord(), is("word")); + } + + @Test + public void parse_subString() throws ParseException { + SubstringItem subStringItem = (SubstringItem) parse("(SUBSTRING 'word')"); + assertThat(subStringItem.getWord(), is("word")); + } + + @Test + public void parse_exactString() throws ParseException { + ExactstringItem subStringItem = (ExactstringItem) parse("(EXACT 'word')"); + assertThat(subStringItem.getWord(), is("word")); + } + + @Test + public void parse_suffix() throws ParseException { + SuffixItem suffixItem = (SuffixItem) parse("(SUFFIX 'word')"); + assertThat(suffixItem.getWord(), is("word")); + } + + @Test + public void parse_phrase() throws ParseException { + PhraseItem phraseItem = (PhraseItem) parse("(PHRASE (WORD 'word'))"); + assertThat(phraseItem.getItem(0), instanceOf(WordItem.class)); + } + + @Test + public void parse_near() throws ParseException { + assertThat(parse("(NEAR)"), instanceOf(NearItem.class)); + } + + @Test + public void parse_onear() throws ParseException { + assertThat(parse("(ONEAR)"), instanceOf(ONearItem.class)); + } + + @Test + public void parse_near_with_distance() throws ParseException { + NearItem nearItem = (NearItem) parse("(NEAR {'distance' 42} (WORD 'first'))"); + assertThat(nearItem.getDistance(), is(42)); + } + + @Test + public void parse_items_with_connectivity() throws ParseException { + AndItem andItem = (AndItem) parse("(AND (WORD {'id' '1'} 'first') (WORD {'connectivity' ['1' 23.5]} 'second'))"); + WordItem secondItem = (WordItem) andItem.getItem(1); + + assertThat(secondItem.getConnectedItem(), is(andItem.getItem(0))); + assertThat(secondItem.getConnectivity(), is(23.5)); + } + + @Test + public void parse_word_with_index() throws ParseException { + WordItem wordItem = (WordItem) parse("(WORD {'index' 'someIndex'} 'text')"); + assertThat(wordItem.getIndexName(), is("someIndex")); + } + + @Test + public void parse_unicode_word() throws ParseException { + WordItem wordItem = (WordItem) parse("(WORD 'trăm')"); + assertThat(wordItem.getWord(), is("trăm")); + } + + public static String getWord(Object item) { + return ((WordItem)item).getWord(); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java b/container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java new file mode 100644 index 00000000000..47bf4072f60 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.textserialize.serializer.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.query.textserialize.parser.ParseException; +import com.yahoo.search.query.textserialize.serializer.QueryTreeSerializer; +import org.junit.Test; + +import static com.yahoo.search.query.textserialize.item.test.ParseItemTestCase.parse; +import static com.yahoo.search.query.textserialize.item.test.ParseItemTestCase.getWord; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class SerializeItemTestCase { + @Test + public void serialize_word_item() { + WordItem item = new WordItem("test that \" and \\ works"); + item.setIndexName("index\"Name"); + + WordItem deSerialized = serializeThenParse(item); + assertThat(deSerialized.getWord(), is(item.getWord())); + assertThat(deSerialized.getIndexName(), is(item.getIndexName())); + } + + @Test + public void serialize_and_item() throws ParseException { + AndItem andItem = new AndItem(); + andItem.addItem(new WordItem("first")); + andItem.addItem(new WordItem("second")); + + AndItem deSerialized = serializeThenParse(andItem); + assertThat(getWord(deSerialized.getItem(0)), is("first")); + assertThat(getWord(deSerialized.getItem(1)), is("second")); + assertThat(deSerialized.getItemCount(), is(2)); + } + + @Test + public void serialize_or_item() throws ParseException { + assertThat(serializeThenParse(new OrItem()), + instanceOf(OrItem.class)); + } + + @Test + public void serialize_not_item() throws ParseException { + NotItem notItem = new NotItem(); + { + notItem.addItem(new WordItem("first")); + notItem.addItem(new WordItem("second")); + } + + serializeThenParse(notItem); + } + + @Test + public void serialize_near_item() throws ParseException { + int distance = 23; + NearItem nearItem = new NearItem(distance); + { + nearItem.addItem(new WordItem("first")); + nearItem.addItem(new WordItem("second")); + } + + NearItem deSerialized = serializeThenParse(nearItem); + + assertThat(deSerialized.getDistance(), is(distance)); + assertThat(deSerialized.getItemCount(), is(2)); + } + + @Test + public void serialize_phrase_item() throws ParseException { + PhraseItem phraseItem = new PhraseItem(new String[] {"first", "second"}); + phraseItem.setIndexName("indexName"); + + PhraseItem deSerialized = serializeThenParse(phraseItem); + assertThat(deSerialized.getItem(0), is(phraseItem.getItem(0))); + assertThat(deSerialized.getItem(1), is(phraseItem.getItem(1))); + assertThat(deSerialized.getIndexName(), is(phraseItem.getIndexName())); + } + + @Test + public void serialize_equiv_item() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.addItem(new WordItem("first")); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getItemCount(), is(1)); + } + + @Test + public void serialize_connectivity() throws ParseException { + OrItem orItem = new OrItem(); + { + WordItem first = new WordItem("first"); + WordItem second = new WordItem("second"); + first.setConnectivity(second, 3.14); + + orItem.addItem(first); + orItem.addItem(second); + } + + OrItem deSerialized = serializeThenParse(orItem); + WordItem first = (WordItem) deSerialized.getItem(0); + Item second = deSerialized.getItem(1); + + assertThat(first.getConnectedItem(), is(second)); + assertThat(first.getConnectivity(), is(3.14)); + } + + @Test + public void serialize_significance() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.setSignificance(24.2); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getSignificance(), is(24.2)); + } + + @Test + public void serialize_unique_id() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.setUniqueID(42); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getUniqueID(), is(42)); + } + + @Test + public void serialize_weight() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.setWeight(42); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getWeight(), is(42)); + } + + private static String serialize(Item item) { + return new QueryTreeSerializer().serialize(item); + } + + @SuppressWarnings("unchecked") + private static T serializeThenParse(T oldItem) { + try { + return (T) parse(serialize(oldItem)); + } catch (ParseException e) { + throw new RuntimeException(e); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java new file mode 100644 index 00000000000..764a44a1bd6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.prelude.query.PredicateQueryItem; +import org.junit.Test; + +import java.math.BigInteger; +import java.util.Iterator; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.fail; + +/** + * Created with IntelliJ IDEA. + * User: magnarn + * Date: 2/5/13 + * Time: 3:52 PM + */ +public class BooleanAttributeParserTest { + + @Test + public void requireThatParseHandlesAllFormats() throws Exception { + assertParse(null, 0); + assertParse("{}", 0); + assertParse("{foo:bar}", 1); + assertParse("{foo:[bar]}", 1); + assertParse("{foo:bar, baz:qux}", 2); + + assertParse("{foo:bar, foo:baz}", 2); + assertParse("{foo:[bar, baz, qux]}", 3); + assertParse("{foo:[bar, baz, qux], quux:corge}", 4); + assertParse("{foo:[bar, baz, qux], quux:[corge, grault]}", 5); + assertParse("{foo:bar, foo:bar, foo:bar}", 3); + + assertParse("{foo:bar:0x1, foo:baz:0xf}", 2); + assertParse("{foo:[bar:0xbabe, baz:0xbeef, qux:0xfee1], quux:corge:0x1234}", 4); + assertParse("{foo:bar:[1], foo:baz:[0,1,2,3]}", 2); + assertParse("{foo:bar:[ 1 ], foo:baz:[ 0 , 1 , 2 , 3 ]}", 2); + assertParse("{foo:[bar:[4,7],baz:[8,5],qux:[3,2]], quux:corge:[2, 5, 7, 58]}", 4); + } + + @Test + public void requireThatIllegalStringsFail() throws Exception { + assertException("{foo:[bar:[baz]}"); + assertException("{foo:[bar:baz}"); + assertException("{foo:bar:[0,1,2}"); + assertException("{foo:[bar:[0,1,2],baz:[0,,2]]}"); + assertException("{foo:[bar:[0,1,2],baz:[0,1,2]}"); + assertException("{foo:bar:[64]}"); + assertException("{foo:bar:[-1]}"); + assertException("{foo:bar:[a]}"); + assertException("{foo:bar:[0,1,[2]]}"); + assertException("{foo:bar}extrachars"); + } + + private void assertException(String s) { + try { + PredicateQueryItem item = new PredicateQueryItem(); + new BooleanSearcher.PredicateValueAttributeParser(item).parse(s); + fail("Expected an exception"); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void requireThatTermsCanHaveBitmaps() throws Exception { + PredicateQueryItem q = assertParse("{foo:bar:0x1}", 1); + PredicateQueryItem.Entry[] features = new PredicateQueryItem.Entry[q.getFeatures().size()]; + q.getFeatures().toArray(features); + assertEquals(1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:0x1, baz:qux:0xf}", 2); + Iterator it = q.getFeatures().iterator(); + assertEquals(1l, it.next().getSubQueryBitmap()); + assertEquals(15l, it.next().getSubQueryBitmap()); + q = assertParse("{foo:bar:0xffffffffffffffff}", 1); + assertEquals(-1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[63]}", 1); + + assertEquals(new BigInteger("ffffffffffffffff", 16).shiftRight(1).add(BigInteger.ONE).longValue(), q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:0x7fffffffffffffff}", 1); + assertEquals(new BigInteger("ffffffffffffffff", 16).shiftRight(1).longValue(), q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[0]}", 1); + assertEquals(1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}", 1); + assertEquals(1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[0,2,6,8]}", 1); + assertEquals(0x145l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:[bar:[0,8,6,2],baz:[1,3,4,15]]}", 2); + it = q.getFeatures().iterator(); + assertEquals(0x145l, it.next().getSubQueryBitmap()); + assertEquals(0x801al, it.next().getSubQueryBitmap()); + } + + private PredicateQueryItem assertParse(String s, int numFeatures) { + PredicateQueryItem item = new PredicateQueryItem(); + BooleanAttributeParser parser = new BooleanSearcher.PredicateValueAttributeParser(item); + parser.parse(s); + assertEquals(numFeatures, item.getFeatures().size()); + return item; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java new file mode 100644 index 00000000000..5c3d9b4824f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collection; + +import static junit.framework.Assert.assertEquals; + +/** + * Test BooleanSearcher + */ +public class BooleanSearcherTestCase { + + private Execution exec; + + private Execution buildExec() { + return new Execution(new Chain(new BooleanSearcher()), + Execution.Context.createContextStub()); + } + + @Before + public void setUp() throws Exception { + exec = buildExec(); + } + + @Test + public void requireThatAttributeMapToSingleFeature() { + PredicateQueryItem item = buildPredicateQueryItem("{gender:female}", null); + assertEquals(1, item.getFeatures().size()); + assertEquals(0, item.getRangeFeatures().size()); + assertEquals("gender", item.getFeatures().iterator().next().getKey()); + assertEquals("female", item.getFeatures().iterator().next().getValue()); + assertEquals("PREDICATE_QUERY_ITEM gender=female", item.toString()); + } + + @Test + public void requireThatAttributeListMapToMultipleFeatures() { + PredicateQueryItem item = buildPredicateQueryItem("{gender:[female,male]}", null); + assertEquals(2, item.getFeatures().size()); + assertEquals(0, item.getRangeFeatures().size()); + assertEquals("PREDICATE_QUERY_ITEM gender=female, gender=male", item.toString()); + } + + @Test + public void requireThatRangeAttributesMapToRangeTerm() { + PredicateQueryItem item = buildPredicateQueryItem(null, "{age:25}"); + assertEquals(0, item.getFeatures().size()); + assertEquals(1, item.getRangeFeatures().size()); + assertEquals("PREDICATE_QUERY_ITEM age:25", item.toString()); + + item = buildPredicateQueryItem(null, "{age:25:0x43, height:170:[2,3,4]}"); + assertEquals(0, item.getFeatures().size()); + assertEquals(2, item.getRangeFeatures().size()); + } + + @Test + public void requireThatQueryWithoutBooleanPropertiesIsUnchanged() { + Query q = new Query(""); + q.getModel().getQueryTree().setRoot(new WordItem("foo", "otherfield")); + Result r = exec.search(q); + + WordItem root = (WordItem)r.getQuery().getModel().getQueryTree().getRoot(); + assertEquals("foo", root.getWord()); + } + + @Test + public void requireThatBooleanSearcherCanBuildPredicateQueryItem() { + PredicateQueryItem root = buildPredicateQueryItem("{gender:female}", "{age:23:[2, 3, 5]}"); + + Collection features = root.getFeatures(); + assertEquals(1, features.size()); + PredicateQueryItem.Entry entry = (PredicateQueryItem.Entry) features.toArray()[0]; + assertEquals("gender", entry.getKey()); + assertEquals("female", entry.getValue()); + assertEquals(-1L, entry.getSubQueryBitmap()); + + Collection rangeFeatures = root.getRangeFeatures(); + assertEquals(1, rangeFeatures.size()); + PredicateQueryItem.RangeEntry rangeEntry = (PredicateQueryItem.RangeEntry) rangeFeatures.toArray()[0]; + assertEquals("age", rangeEntry.getKey()); + assertEquals(23L, rangeEntry.getValue()); + assertEquals(44L, rangeEntry.getSubQueryBitmap()); + } + + @Test + public void requireThatKeysAndValuesCanContainSpaces() { + PredicateQueryItem item = buildPredicateQueryItem("{'My Key':'My Value'}", null); + assertEquals(1, item.getFeatures().size()); + assertEquals(0, item.getRangeFeatures().size()); + assertEquals("My Key", item.getFeatures().iterator().next().getKey()); + assertEquals("'My Value'", item.getFeatures().iterator().next().getValue()); + assertEquals("PREDICATE_QUERY_ITEM My Key='My Value'", item.toString()); + } + + private PredicateQueryItem buildPredicateQueryItem(String attributes, String rangeAttributes) { + Query q = buildQuery("predicate", attributes, rangeAttributes); + Result r = exec.search(q); + return (PredicateQueryItem)r.getQuery().getModel().getQueryTree().getRoot(); + } + + private Query buildQuery(String field, String attributes, String rangeAttributes) { + Query q = new Query(""); + q.properties().set("boolean.field", field); + if (attributes != null) { + q.properties().set("boolean.attributes", attributes); + } + if (rangeAttributes != null) { + q.properties().set("boolean.rangeAttributes", rangeAttributes); + } + return q; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java new file mode 100644 index 00000000000..77c9d1ddb97 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java @@ -0,0 +1,245 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import junit.framework.TestCase; + +import com.yahoo.container.protect.Error; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Unit testing of the searcher com.yahoo.search.querytransform.LegacyCombinator. + * + * @author Steinar Knutsen + */ +public class LegacyCombinatorTestCase extends TestCase { + Searcher searcher; + + protected void setUp() throws Exception { + searcher = new LegacyCombinator(); + } + + public void testStraightForwardSearch() { + Query q = new Query("?query=a&query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu="); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + q = new Query("?query=a+c&query.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a c b", q.getModel().getQueryTree().toString()); + } + + public void testNoBaseQuery() { + Query q = new Query("?query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("b", q.getModel().getQueryTree().toString()); + } + + public void testIncompatibleNewAndOldQuery() { + Query q = new Query("?query.juhu=b&defidx.juhu=a&query.juhu.defidx=c"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + assertTrue("No expected error found.", q.errors().size() > 0); + assertEquals("Did not get invalid query parameter error as expected.", + Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); + } + + public void testNotCombinatorWithoutRoot() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + assertTrue("No expected error found.", q.errors().size() > 0); + System.out.println(q.errors()); + assertEquals("Did not get invalid query parameter error as expected.", + Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); + } + + public void testRankCombinator() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("nalle:b", q.getModel().getQueryTree().toString()); + } + + public void testRankAndNot() { + Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+nalle:b -1:2", q.getModel().getQueryTree().toString()); + } + + public void testReqAndRankAndNot() { + Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+(RANK y:z nalle:b) -1:2", q.getModel().getQueryTree().toString()); + } + + public void testReqAndRank() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("RANK y:z nalle:b", q.getModel().getQueryTree().toString()); + } + + public void testReqAndNot() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+y:z -nalle:b", q.getModel().getQueryTree().toString()); + } + + public void testNewAndOld() { + Query q = new Query("?query.juhu=b&defidx.juhu=nalle&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + Set nastierItems = new HashSet<>(); + nastierItems.add(new StringPair("nalle", "b")); + nastierItems.add(new StringPair("y", "z")); + e.search(q); + AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); + Iterator iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); + if (nastierItems.contains(asPair)) { + nastierItems.remove(asPair); + } else { + assertFalse("Got unexpected item in query tree: (" + + word.getIndexName() + ", " + word.stringValue() + ")", + true); + } + } + assertEquals("Not all expected items found in query.", 0, nastierItems.size()); + } + + public void testReqAndNotWithQuerySyntaxAll() { + Query q = new Query("?query.juhu=b+c&query.juhu.defidx=nalle&query.juhu.operator=not&query.juhu.type=any&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+y:z -(OR nalle:b nalle:c)", q.getModel().getQueryTree().toString()); + } + + public void testDefaultIndexWithoutQuery() { + Query q = new Query("?defidx.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&defidx.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + } + + private static class StringPair { + public final String index; + public final String value; + + StringPair(String index, String value) { + super(); + this.index = index; + this.value = value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((index == null) ? 0 : index.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final StringPair other = (StringPair) obj; + if (index == null) { + if (other.index != null) + return false; + } else if (!index.equals(other.index)) + return false; + if (value == null) { + if (other.value != null) + return false; + } else if (!value.equals(other.value)) + return false; + return true; + } + + } + + public void testMultiPart() { + Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + Set items = new HashSet<>(); + items.add("a"); + items.add("b"); + items.add("c"); + e.search(q); + // OK, the problem here is we have no way of knowing whether nalle or + // juhu was added first, since we have passed through HashMap instances + // inside the implementation + + AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); + Iterator iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + if (items.contains(word.stringValue())) { + items.remove(word.stringValue()); + } else { + assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); + } + } + assertEquals("Not all expected items found in query.", 0, items.size()); + + Set nastierItems = new HashSet<>(); + nastierItems.add(new StringPair("", "a")); + nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); + nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + root = (AndItem) q.getModel().getQueryTree().getRoot(); + iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); + if (nastierItems.contains(asPair)) { + nastierItems.remove(asPair); + } else { + assertFalse("Got unexpected item in query tree: (" + + word.getIndexName() + ", " + word.stringValue() + ")", + true); + } + } + assertEquals("Not all expected items found in query.", 0, nastierItems.size()); + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java new file mode 100644 index 00000000000..6b8f8861af5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java @@ -0,0 +1,217 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordAlternativesItem.Alternative; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests term lowercasing in the search chain. + * + * @author Steinar Knutsen + */ +public class LowercasingTestCase { + + private static final String TEDDY = "teddy"; + private static final String BAMSE = "bamse"; + IndexFacts settings; + Execution execution; + + @Before + public void setUp() throws Exception { + IndexFacts f = new IndexFacts(); + Index bamse = new Index(BAMSE); + Index teddy = new Index(TEDDY); + Index defaultIndex = new Index("default"); + bamse.setLowercase(true); + teddy.setLowercase(false); + defaultIndex.setLowercase(true); + f.addIndex("nalle", bamse); + f.addIndex("nalle", teddy); + f.addIndex("nalle", defaultIndex); + f.freeze(); + settings = f; + execution = new Execution(new Chain( + new VespaLowercasingSearcher(new LowercasingConfig(new LowercasingConfig.Builder()))), + Execution.Context.createContextStub(settings)); + } + + @After + public void tearDown() throws Exception { + execution = null; + } + + @Test + public void smoke() { + Query q = new Query(); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("Gnuff", BAMSE, true); + root.addItem(tmp); + tmp = new WordItem("Blaff", TEDDY, true); + root.addItem(tmp); + tmp = new WordItem("Blyant", "", true); + root.addItem(tmp); + q.getModel().getQueryTree().setRoot(root); + + Result r = execution.search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + WordItem w1 = (WordItem) root.getItem(1); + WordItem w2 = (WordItem) root.getItem(2); + assertEquals("gnuff", w0.getWord()); + assertEquals("Blaff", w1.getWord()); + assertEquals("blyant", w2.getWord()); + } + + @Test + public void slightlyMoreComplexTree() { + Query q = new Query(); + AndItem a0 = new AndItem(); + OrItem o0 = new OrItem(); + PhraseItem p0 = new PhraseItem(); + p0.setIndexName(BAMSE); + PhraseSegmentItem p1 = new PhraseSegmentItem("Overbuljongterningpakkmesterassistent", true, false); + p1.setIndexName(BAMSE); + + WordItem tmp; + tmp = new WordItem("Nalle0", BAMSE, true); + a0.addItem(tmp); + + tmp = new WordItem("Nalle1", BAMSE, true); + o0.addItem(tmp); + tmp = new WordItem("Nalle2", BAMSE, true); + o0.addItem(tmp); + a0.addItem(o0); + + tmp = new WordItem("Nalle3", BAMSE, true); + p0.addItem(tmp); + + p1.addItem(new WordItem("Over", BAMSE, true)); + p1.addItem(new WordItem("buljong", BAMSE, true)); + p1.addItem(new WordItem("terning", BAMSE, true)); + p1.addItem(new WordItem("pakk", BAMSE, true)); + p1.addItem(new WordItem("Mester", BAMSE, true)); + p1.addItem(new WordItem("assistent", BAMSE, true)); + p1.lock(); + p0.addItem(p1); + a0.addItem(p0); + + q.getModel().getQueryTree().setRoot(a0); + + Result r = execution.search(q); + AndItem root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + tmp = (WordItem) root.getItem(0); + assertEquals("nalle0", tmp.getWord()); + OrItem orElement = (OrItem) root.getItem(1); + tmp = (WordItem) orElement.getItem(0); + assertEquals("nalle1", tmp.getWord()); + tmp = (WordItem) orElement.getItem(1); + assertEquals("nalle2", tmp.getWord()); + PhraseItem phrase = (PhraseItem) root.getItem(2); + tmp = (WordItem) phrase.getItem(0); + assertEquals("nalle3", tmp.getWord()); + PhraseSegmentItem locked = (PhraseSegmentItem) phrase.getItem(1); + assertEquals("over", ((WordItem) locked.getItem(0)).getWord()); + assertEquals("buljong", ((WordItem) locked.getItem(1)).getWord()); + assertEquals("terning", ((WordItem) locked.getItem(2)).getWord()); + assertEquals("pakk", ((WordItem) locked.getItem(3)).getWord()); + assertEquals("mester", ((WordItem) locked.getItem(4)).getWord()); + assertEquals("assistent", ((WordItem) locked.getItem(5)).getWord()); + } + + @Test + public void testWeightedSet() { + Query q = new Query(); + AndItem root = new AndItem(); + WeightedSetItem tmp; + tmp = new WeightedSetItem(BAMSE); + tmp.addToken("AbC", 3); + root.addItem(tmp); + tmp = new WeightedSetItem(TEDDY); + tmp.addToken("dEf", 5); + root.addItem(tmp); + q.getModel().getQueryTree().setRoot(root); + Result r = execution.search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WeightedSetItem w0 = (WeightedSetItem) root.getItem(0); + WeightedSetItem w1 = (WeightedSetItem) root.getItem(1); + assertEquals(1, w0.getNumTokens()); + assertEquals(1, w1.getNumTokens()); + assertEquals("abc", w0.getTokens().next().getKey()); + assertEquals("dEf", w1.getTokens().next().getKey()); + } + + @Test + public void testDisableLowercasingWeightedSet() { + execution = new Execution(new Chain( + new VespaLowercasingSearcher(new LowercasingConfig( + new LowercasingConfig.Builder() + .transform_weighted_sets(false)))), + Execution.Context.createContextStub(settings)); + + Query q = new Query(); + AndItem root = new AndItem(); + WeightedSetItem tmp; + tmp = new WeightedSetItem(BAMSE); + tmp.addToken("AbC", 3); + root.addItem(tmp); + tmp = new WeightedSetItem(TEDDY); + tmp.addToken("dEf", 5); + root.addItem(tmp); + q.getModel().getQueryTree().setRoot(root); + Result r = execution.search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WeightedSetItem w0 = (WeightedSetItem) root.getItem(0); + WeightedSetItem w1 = (WeightedSetItem) root.getItem(1); + assertEquals(1, w0.getNumTokens()); + assertEquals(1, w1.getNumTokens()); + assertEquals("AbC", w0.getTokens().next().getKey()); + assertEquals("dEf", w1.getTokens().next().getKey()); + } + + @Test + public void testLowercasingWordAlternatives() { + execution = new Execution(new Chain(new VespaLowercasingSearcher(new LowercasingConfig( + new LowercasingConfig.Builder().transform_weighted_sets(false)))), Execution.Context.createContextStub(settings)); + + Query q = new Query(); + WordAlternativesItem root; + List terms = new ArrayList<>(); + terms.add(new Alternative("ABC", 1.0)); + terms.add(new Alternative("def", 1.0)); + root = new WordAlternativesItem(BAMSE, true, null, terms); + q.getModel().getQueryTree().setRoot(root); + Result r = execution.search(q); + root = (WordAlternativesItem) r.getQuery().getModel().getQueryTree().getRoot(); + assertEquals(3, root.getAlternatives().size()); + assertEquals("ABC", root.getAlternatives().get(0).word); + assertEquals(1.0d, root.getAlternatives().get(0).exactness, 1e-15d); + assertEquals("abc", root.getAlternatives().get(1).word); + assertEquals(.7d, root.getAlternatives().get(1).exactness, 1e-15d); + assertEquals("def", root.getAlternatives().get(2).word); + assertEquals(1.0d, root.getAlternatives().get(2).exactness, 1e-15d); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java new file mode 100644 index 00000000000..512c28b28b1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.prelude.query.Item; + +import com.yahoo.search.Result; + +public class TestUtils { + public static Item getQueryTreeRoot(Result result) { + return result.getQuery().getModel().getQueryTree().getRoot(); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java new file mode 100644 index 00000000000..cbd168225d4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java @@ -0,0 +1,232 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.*; +import com.yahoo.processing.request.ErrorMessage; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import org.junit.Before; +import org.junit.Test; + +import java.util.ListIterator; + +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; + +/** + * Testing of WandSearcher. + */ +public class WandSearcherTestCase { + + private static final String VESPA_FIELD = "vespa-field"; + + private Execution exec; + + private IndexFacts buildIndexFacts() { + IndexFacts retval = new IndexFacts(); + retval.addIndex("test", new Index(VESPA_FIELD)); + retval.freeze(); + return retval; + } + + private Execution buildExec() { + return new Execution(new Chain(new WandSearcher()), + Execution.Context.createContextStub(buildIndexFacts())); + } + + private Query buildQuery(String wandFieldName, String wandTokens, String wandHeapSize, String wandType, String wandScoreThreshold, String wandThresholdBoostFactor) { + Query q = new Query(""); + q.properties().set("wand.field", wandFieldName); + q.properties().set("wand.tokens", wandTokens); + if (wandHeapSize != null) { + q.properties().set("wand.heapSize", wandHeapSize); + } + if (wandType != null) { + q.properties().set("wand.type", wandType); + } + if (wandScoreThreshold != null) { + q.properties().set("wand.scoreThreshold", wandScoreThreshold); + } + if (wandThresholdBoostFactor != null) { + q.properties().set("wand.thresholdBoostFactor", wandThresholdBoostFactor); + } + q.setHits(9); + return q; + } + + private Query buildDefaultQuery(String wandFieldName, String wandHeapSize) { + return buildQuery(wandFieldName, "{a:1,b:2,c:3}", wandHeapSize, null, null, null); + } + + private Query buildDefaultQuery() { + return buildQuery(VESPA_FIELD, "{a:1,\"b\":2,c:3}", null, null, null, null); + } + + private void assertWordItem(String expToken, String expField, int expWeight, Item item) { + WordItem wordItem = (WordItem) item; + assertEquals(expToken, wordItem.getWord()); + assertEquals(expField, wordItem.getIndexName()); + assertEquals(expWeight, wordItem.getWeight()); + } + + @Before + public void setUp() throws Exception { + exec = buildExec(); + } + + @Test + public void requireThatVespaWandCanBeSpecified() { + Query q = buildDefaultQuery(); + Result r = exec.search(q); + + WeakAndItem root = (WeakAndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(100, root.getN()); + assertEquals(3, root.getItemCount()); + ListIterator itr = root.getItemIterator(); + assertWordItem("a", VESPA_FIELD, 1, itr.next()); + assertWordItem("b", VESPA_FIELD, 2, itr.next()); + assertWordItem("c", VESPA_FIELD, 3, itr.next()); + assertFalse(itr.hasNext()); + } + + @Test + public void requireThatVespaWandHeapSizeCanBeSpecified() { + Query q = buildDefaultQuery(VESPA_FIELD, "50"); + Result r = exec.search(q); + + WeakAndItem root = (WeakAndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(50, root.getN()); + } + + @Test + public void requireThatWandCanBeSpecifiedTogetherWithNonAndQueryRoot() { + Query q = buildDefaultQuery(); + q.getModel().getQueryTree().setRoot(new WordItem("foo", "otherfield")); + Result r = exec.search(q); + + AndItem root = (AndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(2, root.getItemCount()); + ListIterator itr = root.getItemIterator(); + assertTrue(itr.next() instanceof WordItem); + assertTrue(itr.next() instanceof WeakAndItem); + assertFalse(itr.hasNext()); + } + + @Test + public void requireThatWandCanBeSpecifiedTogetherWithAndQueryRoot() { + Query q = buildDefaultQuery(); + { + AndItem root = new AndItem(); + root.addItem(new WordItem("foo", "otherfield")); + root.addItem(new WordItem("bar", "otherfield")); + q.getModel().getQueryTree().setRoot(root); + } + Result r = exec.search(q); + + AndItem root = (AndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(3, root.getItemCount()); + ListIterator itr = root.getItemIterator(); + assertTrue(itr.next() instanceof WordItem); + assertTrue(itr.next() instanceof WordItem); + assertTrue(itr.next() instanceof WeakAndItem); + assertFalse(itr.hasNext()); + } + + + @Test + public void requireThatNothingIsAddedWithoutWandPropertiesSet() { + Query foo = new Query(""); + foo.getModel().getQueryTree().setRoot(new WordItem("foo", "otherfield")); + Result r = exec.search(foo); + + WordItem root = (WordItem)TestUtils.getQueryTreeRoot(r); + assertEquals("foo", root.getWord()); + } + + @Test + public void requireThatErrorIsReturnedOnInvalidTokenList() { + Query q = buildQuery(VESPA_FIELD, "{a1,b:1}", null, null, null, null); + Result r = exec.search(q); + + ErrorMessage msg = r.hits().getError(); + assertNotNull(msg); + assertEquals(INVALID_QUERY_PARAMETER.code, msg.getCode()); + assertEquals("'{a1,b:1}' is not a legal sparse vector string: Expected ':' starting at position 3 but was ','",msg.getDetailedMessage()); + } + + @Test + public void requireThatErrorIsReturnedOnUnknownField() { + Query q = buildDefaultQuery("unknown", "50"); + Result r = exec.search(q); + ErrorMessage msg = r.hits().getError(); + assertNotNull(msg); + assertEquals(INVALID_QUERY_PARAMETER.code, msg.getCode()); + assertEquals("Field 'unknown' was not found in index facts for search definitions [test]",msg.getDetailedMessage()); + } + + @Test + public void requireThatVespaOrCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", null, "or", null, null); + Result r = exec.search(q); + + OrItem root = (OrItem)TestUtils.getQueryTreeRoot(r); + assertEquals(3, root.getItemCount()); + ListIterator itr = root.getItemIterator(); + assertWordItem("a", VESPA_FIELD, 1, itr.next()); + assertWordItem("b", VESPA_FIELD, 2, itr.next()); + assertWordItem("c", VESPA_FIELD, 3, itr.next()); + assertFalse(itr.hasNext()); + } + + private void assertWeightedSetItem(WeightedSetItem item) { + assertEquals(3, item.getNumTokens()); + assertEquals(new Integer(1), item.getTokenWeight("a")); + assertEquals(new Integer(2), item.getTokenWeight("b")); + assertEquals(new Integer(3), item.getTokenWeight("c")); + } + + @Test + public void requireThatDefaultParallelWandCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", null, "parallel", null, null); + Result r = exec.search(q); + + WandItem root = (WandItem)TestUtils.getQueryTreeRoot(r); + assertEquals(VESPA_FIELD, root.getIndexName()); + assertEquals(100, root.getTargetNumHits()); + assertEquals(0.0, root.getScoreThreshold()); + assertEquals(1.0, root.getThresholdBoostFactor()); + assertWeightedSetItem(root); + } + + @Test + public void requireThatParallelWandCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", "50", "parallel", "70.5", "2.3"); + Result r = exec.search(q); + + WandItem root = (WandItem)TestUtils.getQueryTreeRoot(r); + assertEquals(VESPA_FIELD, root.getIndexName()); + assertEquals(50, root.getTargetNumHits()); + assertEquals(70.5, root.getScoreThreshold()); + assertEquals(2.3, root.getThresholdBoostFactor()); + assertWeightedSetItem(root); + } + + @Test + public void requireThatDotProductCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", null, "dotProduct", null, null); + Result r = exec.search(q); + + DotProductItem root = (DotProductItem)TestUtils.getQueryTreeRoot(r); + assertEquals(VESPA_FIELD, root.getIndexName()); + assertWeightedSetItem(root); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java new file mode 100644 index 00000000000..4479650cd49 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -0,0 +1,369 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.CJKSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.querytransform.NGramSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +import static com.yahoo.search.searchchain.Execution.Context.createContextStub; + +/** + * @author Jon Bratseth + */ +public class NGramSearcherTestCase extends junit.framework.TestCase { + + private Searcher searcher; + private IndexFacts indexFacts; + + @Override + public void setUp() { + searcher=new NGramSearcher(new SimpleLinguistics()); + indexFacts=new IndexFacts(); + + Index defaultIndex=new Index("default"); + defaultIndex.setNGram(true,3); + defaultIndex.setDynamicSummary(true); + indexFacts.addIndex("default",defaultIndex); + + Index test=new Index("test"); + test.setHighlightSummary(true); + indexFacts.addIndex("default",test); + + Index gram2=new Index("gram2"); + gram2.setNGram(true,2); + gram2.setDynamicSummary(true); + indexFacts.addIndex("default",gram2); + + Index gram3=new Index("gram3"); + gram3.setNGram(true,3); + gram3.setHighlightSummary(true); + indexFacts.addIndex("default",gram3); + + Index gram14=new Index("gram14"); + gram14.setNGram(true,14); + gram14.setDynamicSummary(true); + indexFacts.addIndex("default",gram14); + } + + private IndexFacts getMixedSetup() { + IndexFacts indexFacts = new IndexFacts(); + String musicDoctype = "music"; + String songDoctype = "song"; + Index musicDefault = new Index("default"); + musicDefault.setNGram(true, 1); + indexFacts.addIndex(musicDoctype, musicDefault); + Index songDefault = new Index("default"); + indexFacts.addIndex(songDoctype, songDefault); + Map> clusters = new HashMap<>(); + clusters.put("musicOnly", Arrays.asList(new String[] { musicDoctype })); + clusters.put("songOnly", Arrays.asList(new String[] { songDoctype })); + clusters.put("musicAndSong", Arrays.asList(new String[] { musicDoctype, songDoctype })); + indexFacts.setClusters(clusters); + return indexFacts; + } + + public void testMixedDocTypes() { + final IndexFacts mixedSetup = getMixedSetup(); + { + Query q = new Query("?query=abc&restrict=song"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&restrict=music"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=song"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=music"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + } + + public void testMixedClusters() { + final IndexFacts mixedSetup = getMixedSetup(); + { + Query q = new Query("?query=abc&search=songOnly"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=musicOnly"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=musicAndSong&restrict=music"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=musicAndSong&restrict=song"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + } + + public void testClusterMappingWithMixedDoctypes() { + final IndexFacts mixedSetup = getMixedSetup(); + + } + + public void testNGramRewritingMixedQuery() { + Query q=new Query("?query=foo+gram3:engul+test:bar"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND foo (AND gram3:eng gram3:ngu gram3:gul) test:bar",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingNGramOnly() { + Query q=new Query("?query=gram3:engul"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND gram3:eng gram3:ngu gram3:gul",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewriting2NGramsOnly() { + Query q=new Query("?query=gram3:engul+gram2:123"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND (AND gram3:eng gram3:ngu gram3:gul) (AND gram2:12 gram2:23)",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingShortOnly() { + Query q=new Query("?query=gram3:en"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("gram3:en",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingShortInMixes() { + Query q=new Query("?query=test:a+gram3:en"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND test:a gram3:en",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingPhrase() { + Query q=new Query("?query=gram3:%22engul+a+holi%22"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("gram3:\"eng ngu gul a hol oli\"",q.getModel().getQueryTree().toString()); + } + + /** + * Note that single-term phrases are simplified to just the term at parse time, + * so the ngram rewriter cannot know to keep the grams as a phrase in this case. + */ + public void testNGramRewritingPhraseSingleTerm() { + Query q=new Query("?query=gram3:%22engul%22"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND gram3:eng gram3:ngu gram3:gul",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingAdditionalTermInfo() { + Query q=new Query("?query=gram3:engul!50+foo+gram2:123!150"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + AndItem root=(AndItem)q.getModel().getQueryTree().getRoot(); + AndItem gram3And=(AndItem)root.getItem(0); + AndItem gram2And=(AndItem)root.getItem(2); + + assertExtraTermInfo(50,"engul",gram3And.getItem(0)); + assertExtraTermInfo(50,"engul",gram3And.getItem(1)); + assertExtraTermInfo(50,"engul",gram3And.getItem(2)); + assertExtraTermInfo(150,"123",gram2And.getItem(0)); + assertExtraTermInfo(150,"123",gram2And.getItem(1)); + } + + private void assertExtraTermInfo(int weight,String origin, Item g) { + WordItem gram=(WordItem)g; + assertEquals(weight,gram.getWeight()); + assertEquals(origin,gram.getOrigin().getValue()); + assertTrue(gram.isProtected()); + assertFalse(gram.isFromQuery()); + } + + public void testNGramRewritingExplicitDefault() { + Query q=new Query("?query=default:engul"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND default:eng default:ngu default:gul",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingImplicitDefault() { + Query q=new Query("?query=engul"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND eng ngu gul",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentation() { + assertGramsWithSegmentation(new Chain<>(searcher)); + assertGramsWithSegmentation(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentation(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentation(Chain chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("gram14:firstsecondthird", Query.Type.ANY); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND gram14:firstsecondthi gram14:irstsecondthir gram14:rstsecondthird",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationSingleSegment() { + assertGramsWithSegmentationSingleSegment(new Chain<>(searcher)); + assertGramsWithSegmentationSingleSegment(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationSingleSegment(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationSingleSegment(Chain chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("gram14:first", Query.Type.ANY); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("gram14:first",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationSubstringSegmented() { + assertGramsWithSegmentationSubstringSegmented(new Chain<>(searcher)); + assertGramsWithSegmentationSubstringSegmented(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationSubstringSegmented(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationSubstringSegmented(Chain chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("gram14:afirstsecondthirdo", Query.Type.ANY); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND gram14:afirstsecondth gram14:firstsecondthi gram14:irstsecondthir gram14:rstsecondthird gram14:stsecondthirdo",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationMixed() { + assertGramsWithSegmentationMixed(new Chain<>(searcher)); + assertGramsWithSegmentationMixed(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationMixed(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationMixed(Chain chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("a gram14:afirstsecondthird b gram14:hi", Query.Type.ALL); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND a (AND gram14:afirstsecondth gram14:firstsecondthi gram14:irstsecondthir gram14:rstsecondthird) b gram14:hi",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationMixedAndPhrases() { + assertGramsWithSegmentationMixedAndPhrases(new Chain<>(searcher)); + assertGramsWithSegmentationMixedAndPhrases(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationMixedAndPhrases(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationMixedAndPhrases(Chain chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("a gram14:\"afirstsecondthird b hi\"", Query.Type.ALL); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND a gram14:\"afirstsecondth firstsecondthi irstsecondthir rstsecondthird b hi\"",q.getModel().getQueryTree().toString()); + } + + public void testNGramRecombining() { + Query q=new Query("?query=ignored"); + Result r=new Execution(new Chain<>(searcher,new MockBackend1()),createContextStub(indexFacts)).search(q); + Hit h1=r.hits().get("hit1"); + assertEquals("Should be untouched,\u001feven if containing \u001f",h1.getField("test").toString()); + assertTrue(h1.getField("test") instanceof String); + + assertEquals("Blue red Ed A",h1.getField("gram2").toString()); + assertTrue(h1.getField("gram2") instanceof XMLString); + + assertEquals("Separators on borders work","Blue red ed a\u001f",h1.getField("gram3").toString()); + assertTrue(h1.getField("gram3") instanceof String); + + Hit h2=r.hits().get("hit2"); + assertEquals("katt i...morgen",h2.getField("gram3").toString()); + assertTrue(h2.getField("gram3") instanceof JSONString); + + Hit h3=r.hits().get("hit3"); + assertEquals("\u001ffin\u001f \u001fen\u001f \u001fa\u001f",h3.getField("gram2").toString()); + assertEquals("#Logging in #Java is like that \"Judean P\u001fopul\u001far Front\" scene from \"Life of Brian\".", + h3.getField("gram3").toString()); + } + + private Item parseQuery(String query, Query.Type type) { + Parser parser = ParserFactory.newInstance(type, new ParserEnvironment().setIndexFacts(indexFacts)); + return parser.parse(new Parsable().setQuery(query).setLanguage(Language.UNKNOWN)).getRoot(); + } + + private static class MockBackend1 extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result r=new Result(query); + HitGroup g=new HitGroup(); + r.hits().add(g); + + Hit h1=new Hit("hit1"); + h1.setField(Hit.SDDOCNAME_FIELD,"default"); + h1.setField("test","Should be untouched,\u001feven if containing \u001f"); + h1.setField("gram2",new XMLString("\uFFF9Bl\uFFFAbl\uFFFBluue reed \uFFF9Ed\uFFFAed\uFFFB \uFFF9A\uFFFAa\uFFFB")); + h1.setField("gram3","\uFFF9Blu\uFFFAblu\uFFFBlue red ed a\u001f"); // separator on borders should not trip anything + g.add(h1); + + Hit h2=new Hit("hit2"); + h2.setField(Hit.SDDOCNAME_FIELD,"default"); + h2.setField("gram3",new JSONString("katatt i...mororgrgegen")); + r.hits().add(h2); + + // Test bolding + Hit h3=new Hit("hit3"); + h3.setField(Hit.SDDOCNAME_FIELD,"default"); + + // the result of searching for "fin en a" + h3.setField("gram2","\u001ffi\u001f\u001fin\u001f \u001fen\u001f \u001fa\u001f"); + + // the result from Juniper from of bolding the substring "opul": + h3.setField("gram3","#Logoggggigining in #Javava is likike thahat \"Jududedeaean Pop\u001fopu\u001f\u001fpul\u001fulalar Froronont\" scecenene frorom \"Lifife of Bririaian\"."); + r.hits().add(h3); + return r; + } + + } + + + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java new file mode 100644 index 00000000000..975eec9ce5c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.querytransform.QueryCombinator; +import com.yahoo.search.searchchain.Execution; + +import junit.framework.TestCase; + +/** + * Unit testing of the searcher com.yahoo.search.querytransform.QueryCombinator. + * + * @author Steinar Knutsen + */ +public class QueryCombinatorTestCase extends TestCase { + Searcher searcher; + + protected void setUp() throws Exception { + super.setUp(); + searcher = new QueryCombinator(new ComponentId("combinationTest")); + } + + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testStraightForwardSearch() { + Query q = new Query("?query=a&query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu="); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + q = new Query("?query=a+c&query.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a c b", q.getModel().getQueryTree().toString()); + } + + public void testNoBaseQuery() { + Query q = new Query("?query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("b", q.getModel().getQueryTree().toString()); + } + + public void testDefaultIndexWithoutQuery() { + Query q = new Query("?defidx.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&defidx.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + } + + private static class StringPair { + public final String index; + public final String value; + + StringPair(String index, String value) { + super(); + this.index = index; + this.value = value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((index == null) ? 0 : index.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final StringPair other = (StringPair) obj; + if (index == null) { + if (other.index != null) + return false; + } else if (!index.equals(other.index)) + return false; + if (value == null) { + if (other.value != null) + return false; + } else if (!value.equals(other.value)) + return false; + return true; + } + + } + + public void testMultiPart() { + Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + Set items = new HashSet<>(); + items.add("a"); + items.add("b"); + items.add("c"); + e.search(q); + // OK, the problem here is we have no way of knowing whether nalle or + // juhu was added first, since we have passed through HashMap instances + // inside the implementation + + AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); + Iterator iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + if (items.contains(word.stringValue())) { + items.remove(word.stringValue()); + } else { + assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); + } + } + assertEquals("Not all expected items found in query.", 0, items.size()); + + Set nastierItems = new HashSet<>(); + nastierItems.add(new StringPair("", "a")); + nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); + nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + root = (AndItem) q.getModel().getQueryTree().getRoot(); + iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); + if (nastierItems.contains(asPair)) { + nastierItems.remove(asPair); + } else { + assertFalse("Got unexpected item in query tree: (" + + word.getIndexName() + ", " + word.stringValue() + ")", + true); + } + } + assertEquals("Not all expected items found in query.", 0, nastierItems.size()); + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java new file mode 100644 index 00000000000..9362f729766 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.querytransform.RangeQueryOptimizer; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class RangeQueryOptimizerTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + private static IndexFacts indexFacts = createIndexFacts(); + + @Test + public void testRangeOptimizing() { + assertOptimized("s:<15", "s:<15"); + assertOptimized("AND a s:[1999;2002]","a AND s:[1999;2002]"); + assertOptimized("AND s:<10;15>", "s:<15 AND s:>10"); + assertOptimized("AND s:give s:5 s:me", "s:give s:5 s:me"); + assertOptimized("AND s:[;15> b:<10;]", "s:<15 AND b:>10"); + assertOptimized("AND s:<10;15> b:[;20>", "s:<15 AND b:<20 AND s:>10"); + assertOptimized("AND c:foo s:<10;15> b:<35;40>", "s:<15 AND s:>10 b:>35 AND c:foo b:<40"); + assertOptimized("AND s:<12;15>", "s:<15 AND s:>10 AND s:>12"); + assertOptimized("Nonoverlapping ranges: Cannot match", "AND s:13 s:4 FALSE", "s:<15 AND s:>10 AND s:>100 AND s:13 AND s:<110 AND s:4"); + assertOptimized("Multivalue ranges are not optimized", "AND m:<15 m:>10", "m:<15 AND m:>10"); + assertOptimized("AND s:[13;15>", "s:<15 AND s:[13;17]"); + assertOptimized("AND s:[13;15>", "s:<15 AND s:[13;15]"); + assertOptimized("AND s:[13;15>", "s:[13;15] AND s:<15"); + assertOptimized("AND s:13 s:4 m:<100 s:[13;15> t:<101;109>", "s:<15 AND s:>10 AND t:>100 AND s:13 AND t:<110 AND s:4 AND t:>101 AND t:<111 AND t:<109 AND m:<100 AND s:[13;17]"); + assertOptimized("AND (AND s:<10;15>) (AND s:<22;27>)", "(s:<15 AND s:>10) AND (s:<27 AND s:>22 AND s:>20"); + assertOptimized("AND (AND s:<10;15.5>) (AND s:<22;27.37>)", "(s:<15.5 AND s:>10) AND (s:<27.37 AND s:>22 AND s:>20"); + assertOptimized("AND FALSE", "s:<2 AND s:>2"); + assertOptimized("AND FALSE", "s:>2 AND s:<2"); + assertOptimized("AND s:2", "s:[;2] AND s:[2;]"); + assertOptimized("AND s:2", "s:[2;] AND s:[;2]"); + } + + @Test + public void testRangeOptimizingCarriesOverItemAttributesWhenNotOptimized() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + Item intItem = new IntItem(">" + 15, "s"); + intItem.setWeight(500); + intItem.setFilter(true); + intItem.setRanked(false); + root.addItem(intItem); + assertOptimized("Not optimized", "AND |s:<15;]!500", query); + IntItem transformedIntItem = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue("Filter was carried over", transformedIntItem.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem.getWeight()); + } + + @Test + public void testRangeOptimizingCarriesOverItemAttributesWhenOptimized() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + + Item intItem1 = new IntItem(">" + 15, "s"); + intItem1.setFilter(true); + intItem1.setRanked(false); + intItem1.setWeight(500); + root.addItem(intItem1); + + Item intItem2 = new IntItem("<" + 30, "s"); + intItem2.setFilter(true); + intItem2.setRanked(false); + intItem2.setWeight(500); + root.addItem(intItem2); + + assertOptimized("Optimized", "AND |s:<15;30>!500", query); + IntItem transformedIntItem = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue("Filter was carried over", transformedIntItem.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem.getWeight()); + } + + @Test + public void testNoRangeOptimizingWhenAttributesAreIncompatible() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + + Item intItem1 = new IntItem(">" + 15, "s"); + intItem1.setFilter(true); + intItem1.setRanked(false); + intItem1.setWeight(500); + root.addItem(intItem1); + + Item intItem2 = new IntItem("<" + 30, "s"); + intItem2.setFilter(false); // Disagrees with item1 + intItem2.setRanked(false); + intItem2.setWeight(500); + root.addItem(intItem2); + + assertOptimized("Not optimized", "AND |s:<15;]!500 s:[;30>!500", query); + + IntItem transformedIntItem1 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue("Filter was carried over", transformedIntItem1.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem1.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem1.getWeight()); + + IntItem transformedIntItem2 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(1); + assertFalse("Filter was carried over", transformedIntItem2.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem2.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem2.getWeight()); + } + + @Test + public void testDifferentCompatibleRangesPerFieldAreOptimizedSeparately() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + + // Two internally compatible items + Item intItem1 = new IntItem(">" + 15, "s"); + intItem1.setRanked(false); + root.addItem(intItem1); + + Item intItem2 = new IntItem("<" + 30, "s"); + intItem2.setRanked(false); + root.addItem(intItem2); + + // Two other internally compatible items incompatible with the above + Item intItem3 = new IntItem(">" + 100, "s"); + root.addItem(intItem3); + + Item intItem4 = new IntItem("<" + 150, "s"); + root.addItem(intItem4); + + assertOptimized("Optimized", "AND s:<15;30> s:<100;150>", query); + + IntItem transformedIntItem1 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertFalse("Ranked was carried over", transformedIntItem1.isRanked()); + + IntItem transformedIntItem2 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(1); + assertTrue("Ranked was carried over", transformedIntItem2.isRanked()); + } + + @Test + public void assertOptmimizedYQLQuery() { + Query query = new Query("/?query=select%20%2A%20from%20sources%20%2A%20where%20%28range%28s%2C%20100000%2C%20100000%29%20OR%20range%28t%2C%20-20000000000L%2C%20-20000000000L%29%20OR%20range%28t%2C%2030%2C%2030%29%29%3B&type=yql"); + assertOptimized("YQL usage of the IntItem API works", "OR s:100000 t:-20000000000 t:30", query); + } + + @Test + public void testTracing() { + Query notOptimized = new Query("/?tracelevel=2"); + notOptimized.getModel().getQueryTree().setRoot(parseQuery("s:<15")); + assertOptimized("", "s:<15", notOptimized); + assertFalse(contains("Optimized query ranges", notOptimized.getContext(true).getTrace().traceNode().descendants(String.class))); + + Query optimized = new Query("/?tracelevel=2"); + optimized.getModel().getQueryTree().setRoot(parseQuery("s:<15 AND s:>10")); + assertOptimized("", "AND s:<10;15>", optimized); + assertTrue(contains("Optimized query ranges", optimized.getContext(true).getTrace().traceNode().descendants(String.class))); + } + + private boolean contains(String prefix, Iterable traceEntries) { + for (String traceEntry : traceEntries) + if (traceEntry.startsWith(prefix)) return true; + return false; + } + + private Query assertOptimized(String expected, String queryString) { + return assertOptimized(null, expected, queryString); + } + + private Query assertOptimized(String explanation, String expected, String queryString) { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(parseQuery(queryString)); + return assertOptimized(explanation, expected, query); + } + + private Query assertOptimized(String explanation, String expected, Query query) { + Chain chain = new Chain<>("test", new RangeQueryOptimizer()); + new Execution(chain, Execution.Context.createContextStub(indexFacts)).search(query); + assertEquals(explanation, expected, query.getModel().getQueryTree().getRoot().toString()); + return query; + } + + private Item parseQuery(String query) { + IndexFacts indexFacts = new IndexFacts(); + Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment() + .setIndexFacts(indexFacts) + .setLinguistics(linguistics)); + return parser.parse(new Parsable().setQuery(query)).getRoot(); + } + + private static IndexFacts createIndexFacts() { + IndexFacts indexFacts = new IndexFacts(); + Index singleValue1 = new Index("s"); + Index singleValue2 = new Index("t"); + Index multiValue = new Index("m"); + multiValue.setMultivalue(true); + indexFacts.addIndex("test", singleValue1); + indexFacts.addIndex("test", singleValue2); + indexFacts.addIndex("test", multiValue); + return indexFacts; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java new file mode 100644 index 00000000000..8e645f2781b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.GroupingQueryParser; +import com.yahoo.search.query.properties.DefaultProperties; +import com.yahoo.search.querytransform.SortingDegrader; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author Jon Bratseth + */ +public class SortingDegraderTestCase { + + @Test + public void testDegradingAscending() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(1400l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + assertEquals(1.0, query.getRanking().getMatchPhase().getMaxFilterCoverage().doubleValue(), 1e-16); + } + + @Test + public void testDegradingDescending() { + Query query = new Query("?ranking.sorting=-a1%20-a2"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertFalse(query.getRanking().getMatchPhase().getAscending()); + assertEquals(1400l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + @Test + public void testDegradingNonDefaultMaxHits() { + Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxHits=37"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertFalse(query.getRanking().getMatchPhase().getAscending()); + assertEquals(37l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + @Test + public void testDegradingNonDefaultMaxFilterCoverage() { + Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxFilterCoverage=0.37"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertFalse(query.getRanking().getMatchPhase().getAscending()); + assertEquals(0.37d, query.getRanking().getMatchPhase().getMaxFilterCoverage().doubleValue(), 1e-16); + } + + @Test + public void testDegradingNonDefaultIllegalMaxFilterCoverage() { + try { + Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxFilterCoverage=37"); + assertTrue(false); + } catch (QueryException qe) { + assertEquals("Invalid request parameter", qe.getMessage()); + Throwable setE = qe.getCause(); + assertTrue(setE instanceof IllegalArgumentException); + assertEquals("Could not set 'ranking.matchPhase.maxFilterCoverage' to '37'", setE.getMessage()); + Throwable rootE = setE.getCause(); + assertTrue(rootE instanceof IllegalArgumentException); + assertEquals("maxFilterCoverage must be in the range [0.0, 1.0]. It is 37.0", rootE.getMessage()); + } + + } + + @Test + public void testNoDegradingWhenGrouping() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2&select=all(group(a1)%20each(output(a1)))"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testNoDegradingWhenNonFastSearchAttribute() { + Query query = new Query("?ranking.sorting=%2bnonFastSearchAttribute%20-a2"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testNoDegradingWhenNonNumericalAttribute() { + Query query = new Query("?ranking.sorting=%2bstringAttribute%20-a2"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testNoDegradingWhenTurnedOff() { + Query query = new Query("?ranking.sorting=-a1%20-a2&sorting.degrading=false"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testAccessAllDegradingParametersInQuery() { + Query query = new Query("?ranking.matchPhase.maxHits=555&ranking.matchPhase.attribute=foo&ranking.matchPhase.ascending=true"); + execute(query); + + assertEquals("foo", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(555l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + + assertEquals("foo", query.properties().get("ranking.matchPhase.attribute")); + assertTrue(query.properties().getBoolean("ranking.matchPhase.ascending")); + assertEquals(555l, query.properties().getLong("ranking.matchPhase.maxHits").longValue()); + } + + @Test + public void testDegradingWithLargeMaxHits() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2"); + query.properties().set(DefaultProperties.MAX_HITS, 13 * 1000); + query.properties().set(DefaultProperties.MAX_OFFSET, 8 * 1000); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(21000l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + @Test + public void testDegradingWithoutPaginationSupport() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2&hits=7&offset=1"); + query.properties().set(DefaultProperties.MAX_HITS, 13 * 1000); + query.properties().set(DefaultProperties.MAX_OFFSET, 8 * 1000); + query.properties().set(SortingDegrader.PAGINATION, "false"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(8l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + private Result execute(Query query) { + // Add the grouping parser to transfer the select parameter to a grouping expression + Chain chain = new Chain(new GroupingQueryParser(), new SortingDegrader()); + return new Execution(chain, Execution.Context.createContextStub(createIndexFacts())).search(query); + } + + private IndexFacts createIndexFacts() { + IndexFacts indexFacts = new IndexFacts(); + + Index fastSearchAttribute1 = new Index("a1"); + fastSearchAttribute1.setFastSearch(true); + fastSearchAttribute1.setNumerical(true); + + Index fastSearchAttribute2 = new Index("a2"); + fastSearchAttribute2.setFastSearch(true); + fastSearchAttribute2.setNumerical(true); + + Index nonFastSearchAttribute = new Index("nonFastSearchAttribute"); + nonFastSearchAttribute.setNumerical(true); + + Index stringAttribute = new Index("stringAttribute"); + stringAttribute.setFastSearch(true); + + indexFacts.addIndex("test", fastSearchAttribute1); + indexFacts.addIndex("test", fastSearchAttribute2); + indexFacts.addIndex("test", nonFastSearchAttribute); + indexFacts.addIndex("stringAttribute", stringAttribute); + return indexFacts; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java new file mode 100644 index 00000000000..fa690138e88 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.concurrent.Receiver; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; +import com.yahoo.processing.response.DefaultIncomingData; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.Utf8; + +/** + * Test adding hits to a hit group during rendering. + * + * @author Steinar Knutsen + */ +public class AsyncGroupPopulationTestCase { + private static class WrappedFuture implements ListenableFuture { + Receiver isListening = new Receiver<>(); + + private ListenableFuture wrapped; + + WrappedFuture(ListenableFuture wrapped) { + this.wrapped = wrapped; + } + + public void addListener(Runnable listener, Executor executor) { + wrapped.addListener(listener, executor); + isListening.put(Boolean.TRUE); + } + + public boolean cancel(boolean mayInterruptIfRunning) { + return wrapped.cancel(mayInterruptIfRunning); + } + + public boolean isCancelled() { + return wrapped.isCancelled(); + } + + public boolean isDone() { + return wrapped.isDone(); + } + + public F get() throws InterruptedException, ExecutionException { + return wrapped.get(); + } + + public F get(long timeout, TimeUnit unit) throws InterruptedException, + ExecutionException, TimeoutException { + return wrapped.get(timeout, unit); + } + } + + private static class ObservableIncoming extends DefaultIncomingData { + WrappedFuture> waitForIt = null; + private final Object lock = new Object(); + + @Override + public ListenableFuture> completed() { + synchronized (lock) { + if (waitForIt == null) { + waitForIt = new WrappedFuture<>(super.completed()); + } + } + return waitForIt; + } + } + + private static class InstrumentedGroup extends HitGroup { + private static final long serialVersionUID = 4585896586414935558L; + + InstrumentedGroup(String id) { + super(id, new Relevance(1), new ObservableIncoming()); + ((ObservableIncoming) incoming()).assignOwner(this); + } + + } + + @Test + public final void test() throws InterruptedException, ExecutionException, + JsonParseException, JsonMappingException, IOException { + String rawExpected = "{" + + " \"root\": {" + + " \"children\": [" + + " {" + + " \"id\": \"yahoo1\"," + + " \"relevance\": 1.0" + + " }," + + " {" + + " \"id\": \"yahoo2\"," + + " \"relevance\": 1.0" + + " }" + + " ]," + + " \"fields\": {" + + " \"totalCount\": 0" + + " }," + + " \"id\": \"yahoo\"," + + " \"relevance\": 1.0" + + " }" + + "}"; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HitGroup h = new InstrumentedGroup("yahoo"); + h.incoming().add(new Hit("yahoo1")); + JsonRenderer renderer = new JsonRenderer(); + Result result = new Result(new Query(), h); + renderer.init(); + ListenableFuture f = renderer.render(out, result, + new Execution(Execution.Context.createContextStub()), + result.getQuery()); + WrappedFuture> x = (WrappedFuture>) h.incoming().completed(); + x.isListening.get(86_400_000); + h.incoming().add(new Hit("yahoo2")); + h.incoming().markComplete(); + Boolean b = f.get(); + assertTrue(b); + String rawGot = Utf8.toString(out.toByteArray()); + ObjectMapper m = new ObjectMapper(); + Map expected = m.readValue(rawExpected, Map.class); + Map got = m.readValue(rawGot, Map.class); + assertEquals(expected, got); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java new file mode 100644 index 00000000000..4b26187c9d3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -0,0 +1,1111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.times; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.predicate.Predicate; + +import com.yahoo.tensor.MapTensor; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.chain.Chain; +import com.yahoo.data.access.slime.SlimeAdapter; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentId; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.result.DoubleBucketId; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.grouping.result.StringId; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.NanNumber; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.result.StructuredData; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.ElapsedTimeTestCase; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; +import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.trace.TraceNode; + +import org.mockito.Mockito; + +/** + * Functional testing of {@link JsonRenderer}. + * + * @author Steinar Knutsen + */ +public class JsonRendererTestCase { + + JsonRenderer originalRenderer; + JsonRenderer renderer; + + public JsonRendererTestCase() { + originalRenderer = new JsonRenderer(); + } + + @Before + public void setUp() throws Exception { + // Do the same dance as in production + renderer = (JsonRenderer) originalRenderer.clone(); + renderer.init(); + } + + @After + public void tearDown() throws Exception { + renderer = null; + } + + private static final class Thingie { + @Override + public String toString() { + return "thingie"; + } + } + + @Test + public final void testDocumentId() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"documentid\": \"id:unittest:smoke::whee\"\n" + + " },\n" + + " \"id\": \"id:unittest:smoke::whee\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("docIdTest"); + h.setField("documentid", new DocumentId("id:unittest:smoke::whee")); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + private Result newEmptyResult() { + Query q = new Query("/?query=a"); + Result r = new Result(q); + return r; + } + + @Test + public final void testDataTypes() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"double\": 0.00390625,\n" + + " \"float\": 14.29,\n" + + " \"integer\": 1,\n" + + " \"long\": 4398046511104,\n" + + " \"object\": \"thingie\",\n" + + " \"string\": \"stuff\",\n" + + " \"predicate\": \"a in [b]\",\n" + + " \"tensor\": { \"dimensions\": [\"x\"], \n" + + " \"cells\": [ { \"address\": {\"x\": \"a\"}, \"value\":2.0 } ] }\n" + + " },\n" + + " \"id\": \"datatypestuff\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("datatypestuff"); + // the floating point values are chosen to get a deterministic string representation + h.setField("double", Double.valueOf(0.00390625d)); + h.setField("float", Float.valueOf(14.29f)); + h.setField("integer", Integer.valueOf(1)); + h.setField("long", Long.valueOf(4398046511104L)); + h.setField("string", "stuff"); + h.setField("predicate", Predicate.fromString("a in [b]")); + h.setField("tensor", new TensorFieldValue(MapTensor.from("{ {x:a}: 2.0}"))); + h.setField("object", new Thingie()); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + + @Test + public final void testTracing() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + // which clearly shows a trace child is created once too often... + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"something\"\n" + + " },\n" + + " {\n" + + " \"message\": \"something else\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"yellow\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"message\": \"marker\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + q.trace("something", 1); + q.trace("something else", 1); + Execution e2 = new Execution(new Chain(), execution.context()); + Query subQuery = new Query("/?query=b&tracelevel=1"); + e2.search(subQuery); + subQuery.trace("yellow", 1); + q.trace("marker", 1); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + @Test + public final void testEmptyTracing() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=0"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + Execution e2 = new Execution(new Chain(), execution.context()); + Query subQuery = new Query("/?query=b&tracelevel=0"); + e2.search(subQuery); + subQuery.trace("yellow", 1); + q.trace("marker", 1); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + assertEqualJson(expected, summary); + } + + @SuppressWarnings("unchecked") + @Test + public final void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"message\": \"Resolved properties:\\ntracelevel=10 (value from request)\\nquery=a (value from request)\\n\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"timestamp\": 42\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + Query q = new Query("/?query=a&tracelevel=10"); + Execution execution = new Execution(Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + new Execution(new Chain(), execution.context()); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + ObjectMapper m = new ObjectMapper(); + + Map exp = m.readValue(expected, Map.class); + Map gen = m.readValue(summary, Map.class); + { + // nuke timestamp and check it's there + Map trace = (Map) gen.get("trace"); + List children1 = (List) trace.get("children"); + Map subtrace = (Map) children1.get(2); + List children2 = (List) subtrace.get("children"); + Map traceElement = (Map) children2.get(0); + traceElement.put("timestamp", Integer.valueOf(42)); + } + assertEquals(exp, gen); + } + + + @Test + public final void testHalfEmptyTracing() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"green\"" + + " }" + + " ]" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=0"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + subExecution(execution, "red", 0); + subExecution(execution, "green", 1); + subExecution(execution, "blue", 0); + q.trace("marker", 1); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + assertEqualJson(expected, summary); + } + + private void subExecution(Execution execution, String color, int traceLevel) { + Execution e2 = new Execution(new Chain(), execution.context()); + Query subQuery = new Query("/?query=b&tracelevel=" + traceLevel); + e2.search(subQuery); + subQuery.trace(color, 1); + } + + @Test + public final void testTracingOfNodesWithBothChildrenAndData() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"string payload\",\n" + + " \"children\": [" + + " {\n" + + " \"message\": \"leafnode\"" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"message\": \"something\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + execution.search(q); + final TraceNode child = new TraceNode("string payload", 0L); + child.add(new TraceNode("leafnode", 0L)); + execution.trace().traceNode().add(child); + q.trace("something", 1); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + + @Test + public final void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"string payload\"\n" + + " },\n" + + " {\n" + + " \"message\": \"something\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + execution.search(q); + final TraceNode child = new TraceNode("string payload", 0L); + child.add(new TraceNode(null, 0L)); + execution.trace().traceNode().add(child); + q.trace("something", 1); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + @Test + public final void testTracingOfNestedNodesWithDataAndSubnodes() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"string payload\",\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"in OO languages, nesting is for birds\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + execution.search(q); + final TraceNode child = new TraceNode("string payload", 0L); + final TraceNode childOfChild = new TraceNode(null, 0L); + child.add(childOfChild); + childOfChild.add(new TraceNode("in OO languages, nesting is for birds", 0L)); + execution.trace().traceNode().add(child); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + + @Test + public final void test() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"c\": \"d\",\n" + + " \"uri\": \"http://localhost/1\"\n" + + " },\n" + + " \"id\": \"http://localhost/1\",\n" + + " \"relevance\": 0.9,\n" + + " \"types\": [\n" + + " \"summary\"\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"id\": \"usual\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " {\n" + + " \"fields\": {\n" + + " \"e\": \"f\"\n" + + " },\n" + + " \"id\": \"type grouphit\",\n" + + " \"relevance\": 1.0,\n" + + " \"types\": [\n" + + " \"grouphit\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"fields\": {\n" + + " \"b\": \"foo\",\n" + + " \"uri\": \"http://localhost/\"\n" + + " },\n" + + " \"id\": \"http://localhost/\",\n" + + " \"relevance\": 0.95,\n" + + " \"types\": [\n" + + " \"summary\"\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"coverage\": {\n" + + " \"coverage\": 100,\n" + + " \"documents\": 500,\n" + + " \"full\": true,\n" + + " \"nodes\": 1,\n" + + " \"results\": 1,\n" + + " \"resultsFull\": 1\n" + + " },\n" + + " \"errors\": [\n" + + " {\n" + + " \"code\": 18,\n" + + " \"message\": \"boom\",\n" + + " \"summary\": \"Internal server error.\"\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}"; + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + r.setCoverage(new Coverage(500, 1, true)); + + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + r.hits().add(h); + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + r.hits().add(g); + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + r.hits().add(gg); + r.hits().addError(ErrorMessage.createInternalServerError("boom")); + String summary = render(execution, r); + // System.out.println(summary); + assertEqualJson(expected, summary); + } + + @Test + public void testMoreTypes() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"bigDecimal\": 3.402823669209385e+38,\n" + + " \"bigInteger\": 340282366920938463463374607431768211455,\n" + + " \"byte\": 8,\n" + + " \"short\": 16\n" + + " },\n" + + " \"id\": \"moredatatypestuff\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("moredatatypestuff"); + h.setField("byte", Byte.valueOf((byte) 8)); + h.setField("short", Short.valueOf((short) 16)); + h.setField("bigInteger", new BigInteger( + "340282366920938463463374607431768211455")); + h.setField("bigDecimal", new BigDecimal( + "340282366920938463463374607431768211456.5")); + h.setField("nanNumber", NanNumber.NaN); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testNullField() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"null\": null\n" + + " },\n" + + " \"id\": \"nullstuff\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("nullstuff"); + h.setField("null", null); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testLazyDecoding() throws IOException { + FastHit f = new FastHit("http://a.b/c", 0.5); + String checkWeCanDecode = "bamse"; + String dontCare = "don't care"; + final String fieldName = "checkWeCanDecode"; + f.setLazyStringField(fieldName, Utf8.toBytes(checkWeCanDecode)); + final String fieldName2 = "dontCare"; + f.setLazyStringField(fieldName2, Utf8.toBytes(dontCare)); + assertEquals(checkWeCanDecode, f.getField(fieldName)); + + JsonGenerator mock = Mockito.mock(JsonGenerator.class); + + renderer.setGenerator(mock); + assertTrue(renderer.tryDirectRendering(fieldName2, f)); + + byte[] expectedBytes = Utf8.toBytes(dontCare); + Mockito.verify(mock, times(1)).writeUTF8String(expectedBytes, 0, expectedBytes.length); + } + + @Test + public void testHitWithSource() throws JsonParseException, JsonMappingException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"datatypestuff\",\n" + + " \"relevance\": 1.0,\n" + + " \"source\": \"unit test\"\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("datatypestuff"); + h.setSource("unit test"); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testErrorWithStackTrace() throws InterruptedException, + ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"errors\": [\n" + + " {\n" + + " \"code\": 1234,\n" + + " \"message\": \"top of the day\",\n" + + " \"stackTrace\": \"java.lang.Throwable\\n\\tat com.yahoo.search.rendering.JsonRendererTestCase.testErrorWithStackTrace(JsonRendererTestCase.java:732)\\n\",\n" + + " \"summary\": \"hello\"\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Result r = new Result(q); + Throwable t = new Throwable(); + StackTraceElement[] stack = new StackTraceElement[1]; + stack[0] = new StackTraceElement( + "com.yahoo.search.rendering.JsonRendererTestCase", + "testErrorWithStackTrace", "JsonRendererTestCase.java", 732); + t.setStackTrace(stack); + ErrorMessage e = new ErrorMessage(1234, "hello", "top of the day", t); + r.hits().addError(e); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testContentHeader() { + assertEquals("utf-8", renderer.getEncoding()); + assertEquals("application/json", renderer.getMimeType()); + } + + @Test + public void testGrouping() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"count()\": 7\n" + + " },\n" + + " \"value\": \"Jones\",\n" + + " \"id\": \"group:string:Jones\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"continuation\": {\n" + + " \"next\": \"CCCC\",\n" + + " \"prev\": \"BBBB\"\n" + + " },\n" + + " \"id\": \"grouplist:customer\",\n" + + " \"label\": \"customer\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"continuation\": {\n" + + " \"this\": \"AAAA\"\n" + + " },\n" + + " \"id\": \"group:root:0\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + RootGroup rg = new RootGroup(0, new Continuation() { + @Override + public String toString() { + return "AAAA"; + } + }); + GroupList gl = new GroupList("customer"); + gl.continuations().put("prev", new Continuation() { + @Override + public String toString() { + return "BBBB"; + } + }); + gl.continuations().put("next", new Continuation() { + @Override + public String toString() { + return "CCCC"; + } + }); + Group g = new Group(new StringId("Jones"), new Relevance(1.0)); + g.setField("count()", Integer.valueOf(7)); + gl.add(g); + rg.add(gl); + r.hits().add(rg); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testGroupingWithBucket() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"something()\": 7\n" + + " },\n" + + " \"limits\": {\n" + + " \"from\": \"1.0\",\n" + + " \"to\": \"2.0\"\n" + + " },\n" + + " \"id\": \"group:double_bucket:1.0:2.0\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"id\": \"grouplist:customer\",\n" + + " \"label\": \"customer\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"continuation\": {\n" + + " \"this\": \"AAAA\"\n" + + " },\n" + + " \"id\": \"group:root:0\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + RootGroup rg = new RootGroup(0, new Continuation() { + @Override + public String toString() { + return "AAAA"; + } + }); + GroupList gl = new GroupList("customer"); + Group g = new Group(new DoubleBucketId(1.0, 2.0), new Relevance(1.0)); + g.setField("something()", Integer.valueOf(7)); + gl.add(g); + rg.add(gl); + r.hits().add(rg); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testJsonObjects() throws JsonParseException, JsonMappingException, InterruptedException, ExecutionException, IOException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"inspectable\": {\n" + + " \"a\": \"b\"\n" + + " },\n" + + " \"jackson\": {\n" + + " \"Nineteen-eighty-four\": 1984\n" + + " },\n" + + " \"json producer\": {\n" + + " \"long in structured\": 7809531904\n" + + " },\n" + + " \"org.json array\": [\n" + + " true,\n" + + " true,\n" + + " false\n" + + " ],\n" + + " \"org.json object\": {\n" + + " \"forty-two\": 42\n" + + " }\n" + + " },\n" + + " \"id\": \"json objects\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("json objects"); + JSONObject o = new JSONObject(); + JSONArray a = new JSONArray(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode j = mapper.createObjectNode(); + JSONString s = new JSONString("{\"a\": \"b\"}"); + Slime slime = new Slime(); + Cursor c = slime.setObject(); + c.setLong("long in structured", 7809531904L); + SlimeAdapter slimeInit = new SlimeAdapter(slime.get()); + StructuredData struct = new StructuredData(slimeInit); + ((ObjectNode) j).put("Nineteen-eighty-four", 1984); + o.put("forty-two", 42); + a.put(true); + a.put(true); + a.put(false); + h.setField("inspectable", s); + h.setField("jackson", j); + h.setField("json producer", struct); + h.setField("org.json array", a); + h.setField("org.json object", o); + r.hits().add(h); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public final void testFieldValueInHit() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"fromDocumentApi\":{\"integerField\":123, \"stringField\":\"abc\"}" + + " },\n" + + " \"id\": \"fieldValueTest\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("fieldValueTest"); + StructDataType structType = new StructDataType("jsonRenderer"); + structType.addField(new Field("stringField", DataType.STRING)); + structType.addField(new Field("integerField", DataType.INT)); + Struct struct = structType.createFieldValue(); + struct.setFieldValue("stringField", "abc"); + struct.setFieldValue("integerField", 123); + h.setField("fromDocumentApi", struct); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public final void testHiddenFields() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"hiddenFields\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = createHitWithOnlyHiddenFields(); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + private Hit createHitWithOnlyHiddenFields() { + Hit h = new Hit("hiddenFields"); + h.setField("NaN", NanNumber.NaN); + h.setField("emptyString", ""); + h.setField("emptyStringFieldValue", new StringFieldValue("")); + h.setField("$vespaImplementationDetail", "Hello, World!"); + return h; + } + + @Test + public final void testDebugRendering() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"NaN\": \"NaN\",\n" + + " \"emptyString\": \"\",\n" + + " \"emptyStringFieldValue\": \"\",\n" + + " \"$vespaImplementationDetail\": \"Hello, World!\"\n" + + " },\n" + + " \"id\": \"hiddenFields\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = new Result(new Query("/?renderer.json.debug=true")); + Hit h = createHitWithOnlyHiddenFields(); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public final void testTimingRendering() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{" + + " \"root\": {" + + " \"fields\": {" + + " \"totalCount\": 0" + + " }," + + " \"id\": \"toplevel\"," + + " \"relevance\": 1.0" + + " }," + + " \"timing\": {" + + " \"querytime\": 0.006," + + " \"searchtime\": 0.007," + + " \"summaryfetchtime\": 0.0" + + " }" + + "}"; + Result r = new Result(new Query("/?renderer.json.debug=true&presentation.timing=true")); + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( + new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + r.getElapsedTime().add(t); + renderer.setTimeSource(() -> 8L); + String summary = render(r); + System.out.println(summary); + assertEqualJson(expected, summary); + } + + private String render(Result r) throws InterruptedException, + ExecutionException { + Execution execution = new Execution( + Execution.Context.createContextStub()); + return render(execution, r); + } + + private String render(Execution execution, Result r) + throws InterruptedException, ExecutionException { + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + return summary; + } + + @SuppressWarnings("unchecked") + private void assertEqualJson(String expected, String generated) throws JsonParseException, JsonMappingException, IOException { + ObjectMapper m = new ObjectMapper(); + Map exp = m.readValue(expected, Map.class); + Map gen = m.readValue(generated, Map.class); + assertEquals(exp, gen); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java new file mode 100644 index 00000000000..dc0bc42d410 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.statistics.ElapsedTimeTestCase; +import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; +import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.text.Utf8; + +/** + * Check the legacy sync default renderer doesn't spontaneously combust. + * + * @author Steinar Knutsen + */ +public class SyncDefaultRendererTestCase { + + SyncDefaultRenderer d; + + @Before + public void setUp() throws Exception { + d = new SyncDefaultRenderer(); + d.init(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetEncoding() { + assertEquals("utf-8", d.getEncoding()); + } + + @Test + public final void testGetMimeType() { + assertEquals("text/xml", d.getMimeType()); + } + + @SuppressWarnings("deprecation") + @Test + public final void testRenderWriterResult() throws IOException, InterruptedException, ExecutionException { + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + q.getPresentation().setTiming(true); + Result r = new Result(q); + r.setCoverage(new Coverage(500, 1, true)); + + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( + new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + r.getElapsedTime().add(t); + r.getTemplating().setRenderer(d); + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + r.hits().add(h); + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + r.hits().add(g); + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + r.hits().add(gg); + r.hits().addError(ErrorMessage.createInternalServerError("boom")); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture f = d.render(bs, r, null, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + // TODO figure out a reasonably strict and reasonably flexible way to test + assertTrue(summary.length() > 1000); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java new file mode 100644 index 00000000000..a51dfc1b12f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +import com.yahoo.search.handler.SearchHandler; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.statistics.ElapsedTimeTestCase; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; +import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; +import com.yahoo.text.Utf8; + +/** + * Test the XML renderer + * + * @author Steinar Knutsen + */ +public class XMLRendererTestCase { + + DefaultRenderer d; + + @Before + public void setUp() throws Exception { + d = new DefaultRenderer(); + d.init(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetEncoding() { + assertEquals("utf-8", d.getEncoding()); + } + + @Test + public final void testGetMimeType() { + assertEquals("text/xml", d.getMimeType()); + } + + @Test + public final void testImplicitDefaultRender() throws Exception { + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + q.getPresentation().setTiming(true); + Result r = new Result(q); + r.setCoverage(new Coverage(500, 1, true)); + + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( + new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + r.getElapsedTime().add(t); + r.getTemplating().setRenderer(d); + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + r.hits().add(h); + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + r.hits().add(g); + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + r.hits().add(gg); + r.hits().addError(ErrorMessage.createInternalServerError("boom")); + + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture f = d.render(bs, r, null, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + + assertEquals("\n" + + "")); + assertTrue(summary.contains("Internal server error.")); + assertTrue(summary.contains("")); + assertTrue(summary.contains("")); + assertEquals(2, occurrences(" 1000); + } + + private int occurrences(String fragment, String string) { + int occurrences = 0; + int cursor = 0; + while ( -1 != (cursor = string.indexOf(fragment, cursor))) { + occurrences++; + cursor += fragment.length(); + } + return occurrences; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java b/container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java new file mode 100644 index 00000000000..582b8be1170 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import static org.junit.Assert.*; + +import java.util.Iterator; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * @author steinar + * @author bratseth + */ +public class DefaultErrorHitTestCase { + + private static final String SOURCE = "nalle"; + DefaultErrorHit de; + + @Before + public void setUp() throws Exception { + de = new DefaultErrorHit(SOURCE, ErrorMessage.createUnspecifiedError("DefaultErrorHitTestCase")); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testSetSourceTakeTwo() { + assertEquals(SOURCE, de.getSource()); + de.setSource(null); + assertNull(de.getSource()); + de.setSource("bamse"); + assertEquals("bamse", de.getSource()); + de.addError(ErrorMessage.createBackendCommunicationError("blblbl")); + final Iterator errorIterator = de.errorIterator(); + assertEquals(SOURCE, errorIterator.next().getSource()); + assertEquals("bamse", errorIterator.next().getSource()); + } + + @Test + public final void testToString() { + assertEquals("Error: Source 'nalle': 5: Unspecified error: DefaultErrorHitTestCase", de.toString()); + } + + @Test + public final void testSetMainError() { + ErrorMessage e = ErrorMessage.createBackendCommunicationError("abc"); + assertNull(e.getSource()); + de.addError(e); + assertEquals(SOURCE, e.getSource()); + boolean caught = false; + try { + new DefaultErrorHit(SOURCE, null); + } catch (NullPointerException ex) { + caught = true; + } + assertTrue(caught); + + caught = false; + try { + de.addError(null); + } catch (NullPointerException ex) { + caught = true; + } + assertTrue(caught); + } + + @Test + public final void testAddError() { + ErrorMessage e = ErrorMessage + .createBackendCommunicationError("ljkhlkjh"); + assertNull(e.getSource()); + de.addError(e); + assertEquals(SOURCE, e.getSource()); + e = ErrorMessage.createBadRequest("kdjfhsdkfhj"); + de.addError(e); + int i = 0; + for (Iterator errors = de.errorIterator(); errors + .hasNext(); errors.next()) { + ++i; + } + assertEquals(3, i); + } + + @Test + public final void testAddErrors() { + DefaultErrorHit other = new DefaultErrorHit("abc", + ErrorMessage.createBadRequest("sdasd")); + de.addErrors(other); + int i = 0; + for (Iterator errors = de.errorIterator(); errors + .hasNext(); errors.next()) { + ++i; + } + assertEquals(2, i); + other = new DefaultErrorHit("abd", + ErrorMessage.createEmptyDocsums("uiyoiuy")); + other.addError(ErrorMessage.createNoAnswerWhenPingingNode("xzvczx")); + de.addErrors(other); + i = 0; + for (Iterator errors = de.errorIterator(); errors + .hasNext(); errors.next()) { + ++i; + } + assertEquals(4, i); + } + + @Test + public final void testHasOnlyErrorCode() { + assertTrue(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.UNSPECIFIED.code)); + assertFalse(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR.code)); + + de.addError(ErrorMessage.createUnspecifiedError("dsfsdfs")); + assertTrue(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.UNSPECIFIED.code)); + assertEquals(com.yahoo.container.protect.Error.UNSPECIFIED.code, de.errors().iterator().next().getCode()); + + de.addError(ErrorMessage.createBackendCommunicationError("dsfsdfsd")); + assertFalse(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.UNSPECIFIED.code)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java b/container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java new file mode 100644 index 00000000000..f6b2472cfb5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Integrity test for representation of undefined numeric field values. + * + * @author Steinar Knutsen + */ +public class NanNumberTestCase { + + + @Test + public final void testIntValue() { + assertEquals(0, NanNumber.NaN.intValue()); + } + + @Test + public final void testLongValue() { + assertEquals(0L, NanNumber.NaN.longValue()); + } + + @Test + public final void testFloatValue() { + assertTrue(Float.isNaN(NanNumber.NaN.floatValue())); + } + + @Test + public final void testDoubleValue() { + assertTrue(Double.isNaN(NanNumber.NaN.doubleValue())); + } + + @Test + public final void testToString() { + assertEquals("", NanNumber.NaN.toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java new file mode 100644 index 00000000000..0e382e454b1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java @@ -0,0 +1,174 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.Writer; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import com.yahoo.search.rendering.Renderer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.base.Splitter; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.templates.UserTemplate; +import com.yahoo.prelude.templates.test.BoomTemplate; +import com.yahoo.search.Query; +import com.yahoo.search.Result; + +/** + * Control helper method for result rendering/result templates. + * + * @author Steinar Knutsen + */ +public class TemplatingTestCase { + Result result; + + @Before + public void setUp() throws Exception { + Query q = new Query("/?query=a&presentation.format=nalle&offset=1&hits=5"); + result = new Result(q); + result.setTotalHitCount(1000L); + result.hits().add(new FastHit("http://localhost/1", .95)); + result.hits().add(new FastHit("http://localhost/2", .90)); + result.hits().add(new FastHit("http://localhost/3", .85)); + result.hits().add(new FastHit("http://localhost/4", .80)); + result.hits().add(new FastHit("http://localhost/5", .75)); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetFirstHitNo() { + assertEquals(2, result.getTemplating().getFirstHitNo()); + } + + @Test + public final void testGetNextFirstHitNo() { + assertEquals(7, result.getTemplating().getNextFirstHitNo()); + result.getQuery().setHits(6); + assertEquals(0, result.getTemplating().getNextFirstHitNo()); + } + + @Test + public final void testGetNextLastHitNo() { + assertEquals(11, result.getTemplating().getNextLastHitNo()); + result.getQuery().setHits(6); + assertEquals(0, result.getTemplating().getNextLastHitNo()); + } + + @Test + public final void testGetLastHitNo() { + assertEquals(6, result.getTemplating().getLastHitNo()); + } + + @Test + public final void testGetPrevFirstHitNo() { + assertEquals(1, result.getTemplating().getPrevFirstHitNo()); + } + + @Test + public final void testGetPrevLastHitNo() { + assertEquals(1, result.getTemplating().getPrevLastHitNo()); + } + + @Test + public final void testGetNextResultURL() { + String next = result.getTemplating().getNextResultURL(); + Set expectedParameters = new HashSet<>(Arrays.asList(new String[] { + "hits=5", + "query=a", + "presentation.format=nalle", + "offset=6" + })); + Set actualParameters = new HashSet<>(); + Splitter s = Splitter.on("&"); + for (String parameter : s.split(next.substring(next.indexOf('?') + 1))) { + actualParameters.add(parameter); + } + assertEquals(expectedParameters, actualParameters); + } + + @Test + public final void testGetPreviousResultURL() { + String previous = result.getTemplating().getPreviousResultURL(); + Set expectedParameters = new HashSet<>(Arrays.asList(new String[] { + "hits=5", + "query=a", + "presentation.format=nalle", + "offset=0" + })); + Set actualParameters = new HashSet<>(); + Splitter s = Splitter.on("&"); + for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { + actualParameters.add(parameter); + } + assertEquals(expectedParameters, actualParameters); + } + + @Test + public final void testGetCurrentResultURL() { + String previous = result.getTemplating().getCurrentResultURL(); + Set expectedParameters = new HashSet<>(Arrays.asList(new String[] { + "hits=5", + "query=a", + "presentation.format=nalle", + "offset=1" + })); + Set actualParameters = new HashSet<>(); + Splitter s = Splitter.on("&"); + for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { + actualParameters.add(parameter); + } + assertEquals(expectedParameters, actualParameters); + } + + @Test + public final void testGetTemplates() { + @SuppressWarnings({ "unchecked", "deprecation" }) + UserTemplate t = result.getTemplating().getTemplates(); + assertEquals("default", t.getName()); + } + + @SuppressWarnings("deprecation") + @Test + public final void testSetTemplates() { + result.getTemplating().setTemplates(new BoomTemplate("gnuff", "text/plain", "ISO-8859-15")); + @SuppressWarnings("unchecked") + UserTemplate t = result.getTemplating().getTemplates(); + assertEquals("gnuff", t.getName()); + } + + private static class TestRenderer extends Renderer { + + @Override + public void render(Writer writer, Result result) throws IOException { + } + + @Override + public String getEncoding() { + return null; + } + + @Override + public String getMimeType() { + return null; + } + } + + @SuppressWarnings("deprecation") + @Test + public final void testUsesDefaultTemplate() { + assertTrue(result.getTemplating().usesDefaultTemplate()); + result.getTemplating().setRenderer(new TestRenderer()); + assertFalse(result.getTemplating().usesDefaultTemplate()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java new file mode 100644 index 00000000000..35841a72428 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import java.io.IOException; + +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.prelude.templates.test.TilingTestCase; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class ArrayOutputTestCase extends junit.framework.TestCase { + + public void testArrayOutput() throws IOException { + Result r=new Result(new Query("?query=ignored")); + Hit hit=new Hit("test"); + hit.setField("phone",new XMLString("\n 408-555-1234" + "\n 408-555-5678\n ")); + r.hits().add(hit); + + String rendered = TilingTestCase.getRendered(r); + String[] lines= rendered.split("\n"); + assertEquals(" ",lines[4]); + assertEquals(" 408-555-1234",lines[5]); + assertEquals(" 408-555-5678",lines[6]); + assertEquals(" ",lines[7]); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java new file mode 100644 index 00000000000..efa01cc7c53 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Coverage; + +/** + * @author Steinar Knutsen + */ +public class CoverageTestCase extends junit.framework.TestCase { + + public void testZeroCoverage() { + Coverage c = new Coverage(0L, 0, false, 0); + assertEquals(0, c.getResultPercentage()); + assertEquals(0, c.getResultSets()); + } + + public void testActiveCoverage() { + Coverage c = new Coverage(6, 5); + assertEquals(5, c.getActive()); + assertEquals(6, c.getDocs()); + + Coverage d = new Coverage(7, 6); + c.merge(d); + assertEquals(11, c.getActive()); + assertEquals(13, c.getDocs()); + } + + public void testDefaultCoverage() { + boolean create=true; + + Result r1=new Result(new Query()); + assertEquals(0,r1.getCoverage(create).getResultSets()); + Result r2=new Result(new Query()); + + r1.mergeWith(r2); + assertEquals(0,r1.getCoverage(create).getResultSets()); + } + + public void testDefaultSearchScenario() { + boolean create=true; + + Result federationSearcherResult=new Result(new Query()); + Result singleSourceResult=new Result(new Query()); + federationSearcherResult.mergeWith(singleSourceResult); + assertNull(federationSearcherResult.getCoverage(!create)); + assertEquals(0,federationSearcherResult.getCoverage(create).getResultSets()); + } + + public void testRequestingCoverageSearchScenario() { + boolean create=true; + + Result federationSearcherResult=new Result(new Query()); + Result singleSourceResult=new Result(new Query()); + singleSourceResult.setCoverage(new Coverage(10,1,true)); + federationSearcherResult.mergeWith(singleSourceResult); + assertEquals(1,federationSearcherResult.getCoverage(create).getResultSets()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java new file mode 100644 index 00000000000..386e04ba943 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import com.yahoo.search.result.DeepHitIterator; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * Ensure that the {@link DeepHitIterator} works as intended. + * + * @author havardpe + */ +public class DeepHitIteratorTestCase extends junit.framework.TestCase { + + public void testEmpty() { + HitGroup hits = new HitGroup(); + Iterator it = hits.deepIterator(); + assertFalse(it.hasNext()); + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + // regular iterator behavior + } + } + + public void testRemove() { + HitGroup hits = new HitGroup(); + hits.add(new Hit("foo")); + hits.add(new Hit("bar")); + + Iterator it = hits.deepIterator(); + try { + it.remove(); + fail(); + } catch (IllegalStateException e) { + // need to call next() first + } + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + try { + it.remove(); + fail(); + } catch (IllegalStateException e) { + // prefetch done + } + assertEquals("bar", it.next().getId().toString()); + it.remove(); // no prefetch done + assertFalse(it.hasNext()); + } + + public void testShallow() { + HitGroup hits = new HitGroup(); + hits.add(new Hit("foo")); + hits.add(new Hit("bar")); + hits.add(new Hit("baz")); + + Iterator it = hits.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("bar", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("baz", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testDeep() { + HitGroup grandParent = new HitGroup(); + grandParent.add(new Hit("a")); + HitGroup parent = new HitGroup(); + parent.add(new Hit("b")); + HitGroup child = new HitGroup(); + child.add(new Hit("c")); + HitGroup grandChild = new HitGroup(); + grandChild.add(new Hit("d")); + child.add(grandChild); + child.add(new Hit("e")); + parent.add(child); + parent.add(new Hit("f")); + grandParent.add(parent); + grandParent.add(new Hit("g")); + + Iterator it = grandParent.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("a", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("b", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("c", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("d", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("e", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("f", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("g", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testFirstHitIsGroup() { + HitGroup root = new HitGroup(); + HitGroup group = new HitGroup(); + group.add(new Hit("foo")); + root.add(group); + root.add(new Hit("bar")); + + Iterator it = root.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("bar", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testSecondHitIsGroup() { + HitGroup root = new HitGroup(); + root.add(new Hit("foo")); + HitGroup group = new HitGroup(); + group.add(new Hit("bar")); + root.add(group); + + Iterator it = root.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("bar", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testOrder() { + HitGroup root = new HitGroup(); + MyHitGroup group = new MyHitGroup(); + group.add(new Hit("foo")); + root.add(group); + + Iterator it = root.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertEquals(Boolean.TRUE, group.ordered); + assertFalse(it.hasNext()); + + it = root.unorderedDeepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertEquals(Boolean.FALSE, group.ordered); + assertFalse(it.hasNext()); + } + + @SuppressWarnings("serial") + private static class MyHitGroup extends HitGroup { + + Boolean ordered = null; + + @Override + public Iterator iterator() { + ordered = Boolean.TRUE; + return super.iterator(); + } + + @Override + public Iterator unorderedIterator() { + ordered = Boolean.FALSE; + return super.unorderedIterator(); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java new file mode 100644 index 00000000000..2935c826539 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.search.result.DefaultErrorHit; +import com.yahoo.search.result.ErrorMessage; + +import java.io.IOException; +import java.io.StringWriter; + +/** + * @author bratseth + */ +public class DefaultErrorHitTestCase extends junit.framework.TestCase { + + @SuppressWarnings("null") + public void testErrorHitRenderingWithException() throws IOException { + NullPointerException cause=null; + try { + Object a=null; + a.toString(); + } + catch (NullPointerException e) { + cause=e; + } + StringWriter w=new StringWriter(); + SearchRendererAdaptor.simpleRenderDefaultErrorHit(w, new DefaultErrorHit("test", new ErrorMessage(79, "Myerror", "Mydetail", cause))); + String sep = System.getProperty("line.separator"); + assertEquals( + "\n" + + " Mydetail\n" + + " \n" + + "java.lang.NullPointerException" + sep + + "\tat " + ,w.toString().substring(0, 119+sep.length())); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java new file mode 100644 index 00000000000..9e16b7312eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * @author bratseth + */ +public class FillingTestCase extends junit.framework.TestCase { + + public void testFillingAPIConsistency() { + HitGroup group=new HitGroup(); + group.add(new Hit("hit:1")); + group.add(new Hit("hit:2")); + assertTrue(group.isFilled("summary")); + } + + public void testFillingAPIConsistencyTwoPhase() { + HitGroup group=new HitGroup(); + group.add(createNonFilled("hit:1")); + group.add(createNonFilled("hit:2")); + assertFalse(group.isFilled("summary")); + fillHitsIn(group, "summary"); + group.analyze(); + assertTrue(group.isFilled("summary")); // consistent again + } + + public void testFillingAPIConsistencyThreePhase() { + HitGroup group=new HitGroup(); + group.add(createNonFilled("hit:1")); + group.add(createNonFilled("hit:2")); + assertFalse(group.isFilled("summary")); + assertFalse(group.isFilled("otherSummary")); + fillHitsIn(group, "otherSummary"); + group.analyze(); + assertFalse(group.isFilled("summary")); + assertTrue(group.isFilled("otherSummary")); + fillHitsIn(group, "summary"); + assertTrue(group.isFilled("otherSummary")); + group.analyze(); + assertTrue(group.isFilled("summary")); // consistent again + assertTrue(group.isFilled("otherSummary")); + } + + private Hit createNonFilled(String id) { + Hit hit=new Hit(id); + hit.setFillable(); + return hit; + } + + private void fillHitsIn(HitGroup group,String summary) { + for (Hit hit : group.asList()) { + if (hit.isMeta()) continue; + hit.setFilled(summary); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java new file mode 100644 index 00000000000..c2d5e73fb97 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java @@ -0,0 +1,189 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.Arrays; + +/** + * @author Jon Bratseth + */ +public class HitGroupTestCase extends junit.framework.TestCase { + + public void testStringStripping() { + assertEquals("avabarne", Hit.stripCharacter('j', "javabjarne")); + assertEquals("", Hit.stripCharacter('j', "")); + assertEquals("", Hit.stripCharacter('j', "j")); + assertEquals("frank", Hit.stripCharacter('j', "frank")); + assertEquals("foo", Hit.stripCharacter('j', "fooj")); + assertEquals("", Hit.stripCharacter('j', "jjjjj")); + } + + public void testRecursiveGet() { + // Level 1 + HitGroup g1=new HitGroup(); + g1.add(new Hit("1")); + + // Level 2 + HitGroup g1_1=new HitGroup(); + g1_1.add(new Hit("1.1")); + g1.add(g1_1); + + HitGroup g1_2=new HitGroup(); + g1_2.add(new Hit("1.2")); + g1.add(g1_2); + + // Level 3 + HitGroup g1_1_1=new HitGroup(); + g1_1_1.add(new Hit("1.1.1")); + g1_1.add(g1_1_1); + + HitGroup g1_1_2=new HitGroup(); + g1_1_2.add(new Hit("1.1.2")); + g1_1.add(g1_1_2); + + HitGroup g1_2_1=new HitGroup(); + g1_2_1.add(new Hit("1.2.1")); + g1_2.add(g1_2_1); + + HitGroup g1_2_2=new HitGroup(); + g1_2_2.add(new Hit("1.2.2")); + g1_2.add(g1_2_2); + + // Level 4 + HitGroup g1_1_1_1=new HitGroup(); + g1_1_1_1.add(new Hit("1.1.1.1")); + g1_1_1.add(g1_1_1_1); + + assertNotNull(g1.get("1")); + assertNotNull(g1.get("1.1")); + assertNotNull(g1.get("1.2")); + assertNotNull(g1.get("1.1.1")); + assertNotNull(g1.get("1.1.2")); + assertNotNull(g1.get("1.2.1")); + assertNotNull(g1.get("1.2.2")); + assertNotNull(g1.get("1.1.1.1")); + + assertNotNull(g1.get("1",-1)); + assertNotNull(g1.get("1.1",-1)); + assertNotNull(g1.get("1.2",-1)); + assertNotNull(g1.get("1.1.1",-1)); + assertNotNull(g1.get("1.1.2",-1)); + assertNotNull(g1.get("1.2.1",-1)); + assertNotNull(g1.get("1.2.2",-1)); + assertNotNull(g1.get("1.1.1.1",-1)); + + assertNotNull(g1.get("1",0)); + assertNull(g1.get("1.1",0)); + assertNull(g1.get("1.2",0)); + assertNull(g1.get("1.1.1",0)); + assertNull(g1.get("1.1.2",0)); + assertNull(g1.get("1.2.1",0)); + assertNull(g1.get("1.2.2",0)); + assertNull(g1.get("1.1.1.1",0)); + + assertNotNull(g1.get("1",1)); + assertNotNull(g1.get("1.1",1)); + assertNotNull(g1.get("1.2",1)); + assertNull(g1.get("1.1.1",1)); + assertNull(g1.get("1.1.2",1)); + assertNull(g1.get("1.2.1",1)); + assertNull(g1.get("1.2.2",1)); + assertNull(g1.get("1.1.1.1",1)); + + assertNotNull(g1.get("1",2)); + assertNotNull(g1.get("1.1",2)); + assertNotNull(g1.get("1.2",2)); + assertNotNull(g1.get("1.1.1",2)); + assertNotNull(g1.get("1.1.2",2)); + assertNotNull(g1.get("1.2.1",2)); + assertNotNull(g1.get("1.2.2",2)); + assertNull(g1.get("1.1.1.1",2)); + + assertNotNull(g1.get("1.1.1.1",3)); + + assertNull(g1.get("3",2)); + } + + public void testThatHitGroupIsUnFillable() { + HitGroup hg = new HitGroup("test"); + { + Hit hit = new Hit("http://nalle.balle/1.html", 832); + hit.setField("url", "http://nalle.balle/1.html"); + hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + { + Hit hit = new Hit("http://nalle.balle/2.html", 442); + hit.setField("url", "http://nalle.balle/2.html"); + hit.setField("clickurl", ""); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + assertFalse(hg.isFillable()); + assertTrue(hg.isFilled("anyclass")); + assertNull(hg.getFilled()); + } + + public void testThatHitGroupIsFillable() { + HitGroup hg = new HitGroup("test"); + { + Hit hit = new Hit("http://nalle.balle/1.html", 832); + hit.setField("url", "http://nalle.balle/1.html"); + hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); + hit.setField("attributes", Arrays.asList("typevideo")); + hit.setFillable(); + hg.add(hit); + } + { + Hit hit = new Hit("http://nalle.balle/2.html", 442); + hit.setField("url", "http://nalle.balle/2.html"); + hit.setField("clickurl", ""); + hit.setField("attributes", Arrays.asList("typevideo")); + hit.setFillable(); + hg.add(hit); + } + assertTrue(hg.isFillable()); + assertFalse(hg.isFilled("anyclass")); + assertTrue(hg.getFilled().isEmpty()); + } + + public void testThatHitGroupIsFillableAfterFillableChangeunderTheHood() { + HitGroup hg = new HitGroup("test"); + { + Hit hit = new Hit("http://nalle.balle/1.html", 832); + hit.setField("url", "http://nalle.balle/1.html"); + hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + { + Hit hit = new Hit("http://nalle.balle/2.html", 442); + hit.setField("url", "http://nalle.balle/2.html"); + hit.setField("clickurl", ""); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + assertFalse(hg.isFillable()); + assertTrue(hg.isFilled("anyclass")); + + for (Hit h : hg.asList()) { + h.setFillable(); + } + + HitGroup toplevel = new HitGroup("toplevel"); + toplevel.add(hg); + + assertTrue(toplevel.isFillable()); + assertNotNull(toplevel.getFilled()); + assertFalse(toplevel.isFilled("anyclass")); + + assertTrue(hg.isFillable()); + assertNotNull(hg.getFilled()); + assertFalse(hg.isFilled("anyclass")); + assertTrue(hg.getFilled().isEmpty()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java new file mode 100644 index 00000000000..9066df45309 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChainRegistry; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * @author tonytv + */ +public class DependencyConfigTestCase { + + private static HandlersConfigurerTestWrapper configurer; + + private static SearchChainRegistry registry; + + public static final String root = "src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig"; + + @BeforeClass + public static void createComponentsConfig() throws IOException { + SearchChainConfigurerTestCase. + createComponentsConfig(root + "/chains.cfg", root + "/handlers.cfg", root + "/components.cfg"); + setUp(); + } + + @AfterClass + public static void removeComponentsConfig() throws IOException { + new File(root + "/components.cfg").delete(); + tearDown(); + } + + public static void setUp() { + String configId = "dir:" + root; + configurer = new HandlersConfigurerTestWrapper(configId); + registry=((SearchHandler) configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.SearchHandler")).getSearchChainRegistry(); + } + + public static void tearDown() { + configurer.shutdown(); + } + + @Provides("P") + @Before("B") + @After("A") + public static class Searcher1 extends Searcher { + + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + @Test + public void test() { + Dependencies dependencies = registry.getSearcherRegistry().getComponent(Searcher1.class.getName()).getDependencies(); + + assertTrue(dependencies.provides().containsAll(Arrays.asList("P", "P1", "P2", Searcher1.class.getSimpleName()))); + assertTrue(dependencies.before().containsAll(Arrays.asList("B", "B1", "B2"))); + assertTrue(dependencies.after().containsAll(Arrays.asList("A", "A1", "A2"))); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java new file mode 100644 index 00000000000..18073a1cedd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java @@ -0,0 +1,302 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test; + +import com.yahoo.config.search.IntConfig; +import com.yahoo.config.search.StringConfig; +import com.yahoo.container.config.testutil.TestUtil; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.SearcherRegistry; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.*; +import java.util.*; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; + +/** + * @author bratseth + * @author gjoranv + */ +public class SearchChainConfigurerTestCase { + + private static Random random = new Random(1); + private static String topCfgDir = System.getProperty("java.io.tmpdir") + File.separator + + "SearchChainConfigurerTestCase" + File.separator; + + private static final String testDir = "src/test/java/com/yahoo/search/searchchain/config/test/"; + + + public void cleanup(File cfgDir) { + if (cfgDir.exists()) { + for (File f : cfgDir.listFiles()) { + f.delete(); + } + cfgDir.delete(); + } + } + + @BeforeClass + public static void createDefaultComponentsConfigs() throws IOException { + createComponentsConfig(testDir + "chains.cfg", testDir + "handlers.cfg", testDir + "components.cfg"); + } + + @AfterClass + public static void removeDefaultComponentsConfigs() throws IOException { + new File(testDir + "components.cfg").delete(); + } + + private SearchChainRegistry getSearchChainRegistryFrom(HandlersConfigurerTestWrapper configurer) { + return ((SearchHandler)configurer.getRequestHandlerRegistry(). + getComponent("com.yahoo.search.handler.SearchHandler")).getSearchChainRegistry(); + } + + @Test + public synchronized void testConfiguration() throws Exception { + HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper("dir:" + testDir); + + SearchChain simple=getSearchChainRegistryFrom(configurer).getComponent("simple"); + assertNotNull(simple); + assertThat(getSearcherNumbers(simple), is(Arrays.asList(1, 2, 3))); + + SearchChain child1=getSearchChainRegistryFrom(configurer).getComponent("child:1"); + assertThat(getSearcherNumbers(child1), is(Arrays.asList(1, 2, 4, 5, 7, 8))); + + SearchChain child2=getSearchChainRegistryFrom(configurer).getComponent("child"); + assertThat(getSearcherNumbers(child2), is(Arrays.asList(3, 6, 7, 9))); + + // Verify successful loading of an explicitly declared searcher that takes no user-defined configs. + //assertNotNull(SearchChainRegistry.get().getSearcherRegistry().getComponent + // ("com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher")); + configurer.shutdown(); + } + + private List getSearcherNumbers(SearchChain chain) { + List numbers = new ArrayList<>(); + for (int i=0; i 0) { + dst.write(buf, 0, len); + } + src.close(); + dst.close(); + } + + public static File getCfgDir() { + String token = Long.toHexString(random.nextLong()); + File cfgDir = new File(topCfgDir + File.separator + token + File.separator); + cfgDir.mkdirs(); + return cfgDir; + } + + /** + * Copies the ids from the 'search' array in chains to a 'components' array in a new components file. + * Also adds the default SearchHandler. + */ + public static void createComponentsConfig(String chainsFile, String handlersFile, String componentsFile) throws IOException { + TestUtil.createComponentsConfig(handlersFile, componentsFile, "handler"); + TestUtil.createComponentsConfig(chainsFile, componentsFile, "components", true); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg new file mode 100644 index 00000000000..4c2b8b0ea27 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg @@ -0,0 +1,56 @@ +chains[8] +chains[0].id simple +chains[0].components[3] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher2 +chains[0].components[2] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher3 +chains[1].id mother:1.1 +chains[1].components[2] +chains[1].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[1].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher2 +chains[2].id mother:1.2 +chains[2].components[2] +chains[2].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[2].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher3 +chains[3].id father:2 +chains[3].components[2] +chains[3].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +chains[3].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher6 +chains[4].id father:1 +chains[4].components[2] +chains[4].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +chains[4].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher5 +chains[5].id child:1 +chains[5].components[2] +chains[5].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher7 +chains[5].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher8 +chains[5].inherits[2] +chains[5].inherits[0] mother:1.1 +chains[5].inherits[1] father:1 +chains[6].id child:2 +chains[6].components[2] +chains[6].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher7 +chains[6].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher9 +chains[6].inherits[2] +chains[6].inherits[0] mother +chains[6].inherits[1] father +chains[6].excludes[2] +chains[6].excludes[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[6].excludes[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +chains[7].id configurable +chains[7].components[1] +chains[7].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher + +components[11] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher +components[2].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +components[3].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher2 +components[4].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher3 +components[5].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +components[6].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher5 +components[7].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher6 +components[8].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher7 +components[9].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher8 +components[10].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher9 + diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg new file mode 100755 index 00000000000..1c8c83cef10 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg @@ -0,0 +1,8 @@ +chains[1] +chains[0].id test-chains-config-update +chains[0].components[2] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher +components[2] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg new file mode 100755 index 00000000000..9717f7b4ee5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg @@ -0,0 +1,10 @@ +chains[1] +chains[0].id test-chains-config-update +chains[0].components[3] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher +chains[0].components[2] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher +components[3] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher +components[2].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg new file mode 100644 index 00000000000..bb4c2919bec --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg @@ -0,0 +1,20 @@ +chains[1] +chains[0].id "default" +chains[0].phases[2] +chains[0].phases[0].id "phase1" +chains[0].phases[1].id "phase2" +chains[0].phases[1].before[1] +chains[0].phases[1].before[0] "phase1" +chains[0].components[1] +chains[0].components[0] "com.yahoo.search.searchchain.config.test.DependencyConfigTestCase$Searcher1" +components[1] +components[0].id "com.yahoo.search.searchchain.config.test.DependencyConfigTestCase$Searcher1" +components[0].dependencies.provides[2] +components[0].dependencies.provides[0] "P1" +components[0].dependencies.provides[1] "P2" +components[0].dependencies.before[2] +components[0].dependencies.before[0] "B1" +components[0].dependencies.before[1] "B2" +components[0].dependencies.after[2] +components[0].dependencies.after[0] "A1" +components[0].dependencies.after[1] "A2" diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/index-info.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg new file mode 100644 index 00000000000..d9838a95665 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg @@ -0,0 +1,14 @@ +chains[1] +chains[0].id default +chains[0].components[2] +chains[0].components[0].id com.yahoo.search.searchchain.config.test.ImplicitDependenciesTestCase$First +chains[0].components[1].id com.yahoo.search.searchchain.config.test.ImplicitDependenciesTestCase$Second + +components[2] +components[0].id PoSearcher +components[0].classid com.yahoo.pageopt.system.PoSearcher +components[1].id ExampleSearcher +components[1].classid com.yahoo.pageopt.searcher.ExampleSearcher + +components[1].dependencies.after[1] +components[1].dependencies.after[0] PoSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/index-info.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg new file mode 100644 index 00000000000..379e768d6f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg @@ -0,0 +1 @@ +intVal 7 diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg new file mode 100644 index 00000000000..f514ae59a37 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg @@ -0,0 +1 @@ +speciallog[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF new file mode 100644 index 00000000000..09956bb2aef --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 2.1 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text new file mode 100644 index 00000000000..ecbedc85875 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text @@ -0,0 +1,22 @@ +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:2.1")); + return result; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF new file mode 100644 index 00000000000..719fe7a81ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 2.2 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text new file mode 100644 index 00000000000..413575aca39 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text @@ -0,0 +1,22 @@ +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:2.2")); + return result; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF new file mode 100644 index 00000000000..9e7835f6ee2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 2 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text new file mode 100644 index 00000000000..29e5fb7697a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text @@ -0,0 +1,22 @@ +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:2")); + return result; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF new file mode 100644 index 00000000000..70447069705 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 0 +Bundle-Vendor: Yahoo! +Import-Package: com.yahoo.prelude, + org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java new file mode 100644 index 00000000000..5c42629524b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override + Result search(Query query,Execution execution) { + Result result=execution.search(query); + ErrorMessage.createErrorInPluginSearcher("nop"); // Check that we may access legacy packages + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:0")); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF new file mode 100644 index 00000000000..972c3090b8d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: HelloWorld +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher2.Searcher2 +Bundle-Version: 1.0.0 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher2 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java new file mode 100644 index 00000000000..942bb2fe97a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.searcher2; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class Searcher2 extends Searcher { + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg new file mode 100644 index 00000000000..af532c6d565 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg @@ -0,0 +1 @@ +stringVal "com.yahoo.search.searchchain.config.test" diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg new file mode 100644 index 00000000000..609c4708306 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg @@ -0,0 +1,27 @@ +chains[3] +chains[0].id classInstances +chains[0].components[3] +chains[0].components[0] class1-instance1 +chains[0].components[1] class1-instance2 +chains[0].components[2] class2-instance2 +chains[1].id osgiInstances +chains[1].components[3] +chains[1].components[0] osgi1-instance1 +chains[1].components[1] osgi1-instance2 +chains[1].components[2] osgi2-instance2 +chains[2].id multiOsgiInstances +chains[2].components[4] +chains[2].components[0] osgim1-instance1 +chains[2].components[1] osgim1-instance2 +chains[2].components[2] osgi2-instance2 +chains[2].components[3] osgim2-instance2 +components[9] +components[0].id class1-instance1 +components[1].id class1-instance2 +components[2].id class2-instance2 +components[3].id osgi1-instance1 +components[4].id osgi1-instance2 +components[5].id osgi2-instance2 +components[6].id osgim1-instance1 +components[7].id osgim1-instance2 +components[8].id osgim2-instance2 diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg new file mode 100644 index 00000000000..8a985f92d10 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg @@ -0,0 +1,24 @@ +components[11] +components[0].id class1-instance1 +components[0].classId com.yahoo.search.searchchain.config.test.SearcherInstancesTestCase$Searcher1 +components[1].id class1-instance2 +components[1].classId com.yahoo.search.searchchain.config.test.SearcherInstancesTestCase$Searcher1 +components[2].id class2-instance2 +components[2].classId com.yahoo.search.searchchain.config.test.SearcherInstancesTestCase$Searcher2 +components[3].id osgi1-instance1 +components[3].classId com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +components[4].id osgi1-instance2 +components[4].classId com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +components[5].id osgi2-instance2 +components[5].classId com.yahoo.search.searchchain.config.test.searcher2.Searcher2 +components[6].id osgim1-instance1 +components[6].classId com.yahoo.search.searchchain.config.test.twosearchers.MultiSearcher1 +components[6].bundle twosearchers +components[7].id osgim1-instance2 +components[7].classId com.yahoo.search.searchchain.config.test.twosearchers.MultiSearcher1 +components[7].bundle twosearchers +components[8].id osgim2-instance2 +components[8].classId com.yahoo.search.searchchain.config.test.twosearchers.MultiSearcher2 +components[8].bundle twosearchers +components[9].id com.yahoo.search.handler.SearchHandler +components[10].id com.yahoo.container.handler.config.HandlersConfigurerDi$RegistriesHack diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg new file mode 100644 index 00000000000..712b7071447 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg @@ -0,0 +1,6 @@ +chains[1] +chains[0].id update-searcher +chains[0].components[1] +chains[0].components[0] update-searcher +components[1] +components[0].id update-searcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg new file mode 100644 index 00000000000..39b0237deb0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg @@ -0,0 +1,7 @@ +chains[1] +chains[0].id update-searcher +chains[0].components[1] +chains[0].components[0] update-searcher +components[2] +components[0].id update-searcher +components[1].id update-searcher2 diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg new file mode 100644 index 00000000000..13ec94e9aa5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg @@ -0,0 +1,10 @@ +chains[1] +chains[0].id three-searchers +chains[0].components[3] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher +chains[0].components[2] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher +components[3] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher +components[2].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF new file mode 100644 index 00000000000..20260ba2733 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF @@ -0,0 +1,11 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: HelloWorld +Bundle-SymbolicName: twosearchers +Bundle-Version: 1.0.0 +Bundle-Vendor: Yahoo! +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java new file mode 100644 index 00000000000..0e7e5fb54b7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.twosearchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class MultiSearcher1 extends Searcher { + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java new file mode 100644 index 00000000000..091380a524e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.twosearchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class MultiSearcher2 extends Searcher { + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF new file mode 100644 index 00000000000..7c98e11231e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.updatesearcher.UpdateSearcher +Bundle-Version: 0 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text new file mode 100644 index 00000000000..f62333761a2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text @@ -0,0 +1,24 @@ +package com.yahoo.search.searchchain.config.test.updatesearcher; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Result; +import com.yahoo.search.Query; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class UpdateSearcher extends com.yahoo.search.Searcher { + + public String test = "update2"; + + public @Override + Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:updatesearcher:2")); + return result; + } +} \ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF new file mode 100644 index 00000000000..7c98e11231e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.updatesearcher.UpdateSearcher +Bundle-Version: 0 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java new file mode 100644 index 00000000000..6c38df8fe08 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.updatesearcher; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class UpdateSearcher extends com.yahoo.search.Searcher { + + public String test = "update"; + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:updatesearcher:0")); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg new file mode 100644 index 00000000000..3c19c691e9b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg @@ -0,0 +1,33 @@ +chains[1] +chains[0].id "default_chain" +chains[0].configId searchchains/searchchain/default_chain +chains[0].components[3] +chains[0].components[0] InitSearcher +chains[0].components[1] PrepareSearcher +chains[0].components[2] RunSearcher +chains[0].phases[1] +chains[0].phases[0].id "phase_1" +chains[0].phases[0].before[1] +chains[0].phases[0].before[0] phase_2 +components[3] +components[0].id "InitSearcher" +components[0].configId searchchains/searcher/InitSearcher +components[0].dependencies.before[1] +components[0].dependencies.before[0] init +components[0].dependencies.after[1] +components[0].dependencies.after[0] prepare +components[1].id "PrepareSearcher" +components[1].classid "PrepareSearcherClass" +components[1].configId searchchains/searcher/PrepareSearcher +components[1].dependencies.provides[1] +components[1].dependencies.provides[0] init +components[1].dependencies.before[1] +components[1].dependencies.before[0] prepare +components[2].id "RunSearcher" +components[2].classid "RunSearcherClass" +components[2].bundle "RunSearcherBundle" +components[2].configId searchchains/searcher/RunSearcher +components[2].dependencies.provides[1] +components[2].dependencies.provides[0] prepare +components[2].dependencies.before[1] +components[2].dependencies.before[0] run diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java new file mode 100644 index 00000000000..482b3e08661 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; +import junit.framework.TestCase; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Jon Bratseth + */ +public class AsyncExecutionOfOneChainTestCase extends TestCase { + + /** Tests having a result with some slow source data which should pass directly to rendering */ + public void testParallelExecutionOfOneChain() { + // Setup + Chain mainChain=new Chain<>(new ParallelExecutor(),new ResultProcessor(),new RegularProvider()); + + // Execute + Result result=new Execution(mainChain, Execution.Context.createContextStub()).search(new Query()); + + // Verify + assertEquals("Received 2 hits from 3 threads",3*2,result.hits().size()); + assertEquals(1.0, result.hits().get("thread-0:hit-0").getRelevance().getScore()); + assertEquals(1.0, result.hits().get("thread-1:hit-0").getRelevance().getScore()); + assertEquals(1.0, result.hits().get("thread-2:hit-0").getRelevance().getScore()); + assertEquals(0.5, result.hits().get("thread-0:hit-1").getRelevance().getScore()); + assertEquals(0.5, result.hits().get("thread-1:hit-1").getRelevance().getScore()); + assertEquals(0.5, result.hits().get("thread-2:hit-1").getRelevance().getScore()); + } + + private class ParallelExecutor extends Searcher { + + /** The number of parallel executions */ + private static final int parallelism=2; + + @Override + public Result search(Query query, Execution execution) { + List futureResults=new ArrayList<>(parallelism); + for (int i=0; i hits=result.hits().deepIterator(); hits.hasNext(); ) + hits.next().setRelevance(1d/i++); + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java new file mode 100644 index 00000000000..a54367628a3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Test for aynchrounous execution + * @author Arne Bergene Fossaa + */ +public class AsyncExecutionTestCase extends junit.framework.TestCase { + + public class WaitingSearcher extends Searcher { + + int waittime; + private WaitingSearcher(String id,int waittime) { + super(new ComponentId(id)); + this.waittime = waittime; + } + + @Override + public Result search(Query query,Execution execution) { + Result result=execution.search(query); + if(waittime != 0) + try { + Thread.sleep(waittime); + } catch (InterruptedException e) { + } + return result; + } + } + + public class SimpleSearcher extends Searcher { + + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + //This should take ~50+ ms + public void testAsync() { + List searchList = new ArrayList<>(); + searchList.add(new WaitingSearcher("one",60000)); + searchList.add(new WaitingSearcher("two",0)); + Chain searchChain = new Chain<>(new ComponentId("chain"), searchList); + + AsyncExecution asyncExecution = new AsyncExecution(searchChain, Execution.Context.createContextStub()); + FutureResult future = asyncExecution.search(new Query("?hits=0")); + Result result = future.get(0, TimeUnit.MILLISECONDS); + + assertTrue(result.hits().getError() != null); + } + + public void testWaitForAll() { + Chain slowChain = new Chain<>( + new ComponentId("slow"), + Arrays.asList(new Searcher[]{new WaitingSearcher("slow",30000)} + ) + ); + + Chain fastChain = new Chain<>( + new ComponentId("fast"), + Arrays.asList(new Searcher[]{new SimpleSearcher()}) + ); + + FutureResult slowFuture = new AsyncExecution(slowChain, Execution.Context.createContextStub()).search(new Query("?hits=0")); + FutureResult fastFuture = new AsyncExecution(fastChain, Execution.Context.createContextStub()).search(new Query("?hits=0")); + fastFuture.get(); + FutureResult reslist[] = new FutureResult[]{slowFuture,fastFuture}; + List results = AsyncExecution.waitForAll(Arrays.asList(reslist),0); + + //assertTrue(slowFuture.isCancelled()); + assertTrue(fastFuture.isDone() && !fastFuture.isCancelled()); + + assertNotNull(results.get(0).hits().getErrorHit()); + assertNull(results.get(1).hits().getErrorHit()); + } + + public void testSync() { + Query query=new Query("?query=test"); + Searcher searcher=new ResultProducingSearcher(); + Result result=new Execution(searcher, Execution.Context.createContextStub()).search(query); + + assertEquals(1,result.hits().size()); + assertEquals("hello",result.hits().get(0).getField("test")); + } + + public void testSyncThroughSync() { + Query query=new Query("?query=test"); + Searcher searcher=new ResultProducingSearcher(); + Result result=new Execution(new Execution(searcher, Execution.Context.createContextStub())).search(query); + + assertEquals(1,result.hits().size()); + assertEquals("hello",result.hits().get(0).getField("test")); + } + + public void testAsyncThroughSync() { + Query query=new Query("?query=test"); + Searcher searcher=new ResultProducingSearcher(); + FutureResult futureResult=new AsyncExecution(new Execution(searcher, Execution.Context.createContextStub())).search(query); + + List futureResultList=new ArrayList<>(); + futureResultList.add(futureResult); + AsyncExecution.waitForAll(futureResultList,1000); + Result result=futureResult.get(); + + assertEquals(1,result.hits().size()); + assertEquals("hello",result.hits().get(0).getField("test")); + } + + private static class ResultProducingSearcher extends Searcher { + + @Override + public Result search(Query query,Execution execution) { + Result result=new Result(query); + Hit hit=new Hit("test"); + hit.setField("test","hello"); + result.hits().add(hit); + return result; + } + + } + + @SuppressWarnings("deprecation") + public void testAsyncExecutionTimeout() { + Chain chain = new Chain<>(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return new Result(query); + } + }); + Execution execution = new Execution(chain, Execution.Context.createContextStub()); + AsyncExecution async = new AsyncExecution(execution); + FutureResult future = async.searchAndFill(new Query()); + future.get(1, TimeUnit.MILLISECONDS); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java new file mode 100644 index 00000000000..642d8d8cd7e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java @@ -0,0 +1,297 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +/** + * Tests basic search chain execution functionality + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class ExecutionTestCase extends junit.framework.TestCase { + + public void testLinearExecutions() { + // Make a chain + List searchers1=new ArrayList<>(); + searchers1.add(new TestSearcher("searcher1")); + searchers1.add(new TestSearcher("searcher2")); + searchers1.add(new TestSearcher("searcher3")); + searchers1.add(new TestSearcher("searcher4")); + Chain chain1=new Chain<>(new ComponentId("chain1"), searchers1); + // Make another chain containing two of the same searcher instances and two new + List searchers2=new ArrayList<>(searchers1); + searchers2.set(1,new TestSearcher("searcher5")); + searchers2.set(3,new TestSearcher("searcher6")); + Chain chain2=new Chain<>(new ComponentId("chain2"), searchers2); + // Execute both + Query query=new Query("test"); + Result result1=new Execution(chain1, Execution.Context.createContextStub()).search(query); + Result result2=new Execution(chain2, Execution.Context.createContextStub()).search(query); + // Verify results + assertEquals(4,result1.getConcreteHitCount()); + assertNotNull(result1.hits().get("searcher1-1")); + assertNotNull(result1.hits().get("searcher2-1")); + assertNotNull(result1.hits().get("searcher3-1")); + assertNotNull(result1.hits().get("searcher4-1")); + + assertEquals(4,result2.getConcreteHitCount()); + assertNotNull(result2.hits().get("searcher1-2")); + assertNotNull(result2.hits().get("searcher5-1")); + assertNotNull(result2.hits().get("searcher3-2")); + assertNotNull(result2.hits().get("searcher6-1")); + } + + public void testNestedExecution() { + // Make a chain + List searchers1=new ArrayList<>(); + searchers1.add(new FillableTestSearcher("searcher1")); + searchers1.add(new WorkflowSearcher()); + searchers1.add(new TestSearcher("searcher2")); + searchers1.add(new FillingSearcher()); + searchers1.add(new FillableTestSearcherAtTheEnd("searcher3")); + Chain chain1=new Chain<>(new ComponentId("chain1"), searchers1); + // Execute it + Query query=new Query("test"); + Result result1=new Execution(chain1, Execution.Context.createContextStub()).search(query); + // Verify results + assertEquals(7,result1.getConcreteHitCount()); + assertNotNull(result1.hits().get("searcher1-1")); + assertNotNull(result1.hits().get("searcher2-1")); + assertNotNull(result1.hits().get("searcher3-1")); + assertNotNull(result1.hits().get("searcher3-1-filled")); + assertNotNull(result1.hits().get("searcher2-2")); + assertNotNull(result1.hits().get("searcher3-2")); + assertNotNull(result1.hits().get("searcher3-2-filled")); + } + + public void testContextCacheSingleLengthSearchChain() { + IndexFacts[] contextsBefore = new IndexFacts[1]; + IndexFacts[] contextsAfter = new IndexFacts[1]; + List l = new ArrayList<>(1); + l.add(new ContextCacheSearcher(0, contextsBefore, contextsAfter)); + Chain chain = new Chain<>(l); + Query query = new Query("?mutatecontext=0"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(contextsBefore[0], contextsAfter[0]); + assertSame(contextsBefore[0], contextsAfter[0]); + } + + public void testContextCache() { + IndexFacts[] contextsBefore = new IndexFacts[5]; + IndexFacts[] contextsAfter = new IndexFacts[5]; + List l = new ArrayList<>(5); + l.add(new ContextCacheSearcher(0, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(1, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(2, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(3, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(4, contextsBefore, contextsAfter)); + Chain chain = new Chain<>(l); + Query query = new Query("?mutatecontext=2"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + + assertSame(contextsBefore[0], contextsAfter[0]); + assertSame(contextsBefore[1], contextsAfter[1]); + assertSame(contextsBefore[2], contextsAfter[2]); + assertSame(contextsBefore[3], contextsAfter[3]); + assertSame(contextsBefore[4], contextsAfter[4]); + + assertSame(contextsBefore[0], contextsBefore[1]); + assertNotSame(contextsBefore[1], contextsBefore[2]); + assertSame(contextsBefore[2], contextsBefore[3]); + assertSame(contextsBefore[3], contextsBefore[4]); + } + + public void testContextCacheMoreSearchers() { + IndexFacts[] contextsBefore = new IndexFacts[7]; + IndexFacts[] contextsAfter = new IndexFacts[7]; + List l = new ArrayList<>(7); + l.add(new ContextCacheSearcher(0, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(1, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(2, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(3, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(4, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(5, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(6, contextsBefore, contextsAfter)); + Chain chain = new Chain<>(l); + Query query = new Query("?mutatecontext=2,4"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + + assertSame(contextsBefore[0], contextsAfter[0]); + assertSame(contextsBefore[1], contextsAfter[1]); + assertSame(contextsBefore[2], contextsAfter[2]); + assertSame(contextsBefore[3], contextsAfter[3]); + assertSame(contextsBefore[4], contextsAfter[4]); + assertSame(contextsBefore[5], contextsAfter[5]); + assertSame(contextsBefore[6], contextsAfter[6]); + + assertSame(contextsBefore[0], contextsBefore[1]); + assertNotSame(contextsBefore[1], contextsBefore[2]); + assertSame(contextsBefore[2], contextsBefore[3]); + assertNotSame(contextsBefore[3], contextsBefore[4]); + assertSame(contextsBefore[4], contextsBefore[5]); + assertSame(contextsBefore[5], contextsBefore[6]); + } + + @Test + public void testBasicFill() { + Chain chain = new Chain(new FillableResultSearcher()); + Execution execution = new Execution(chain, Execution.Context.createContextStub(null)); + + Result result = execution.search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?presentation.summary=all"))); + assertNotNull(result.hits().get("a")); + assertNull(result.hits().get("a").getField("filled")); + execution.fill(result); + assertTrue((Boolean) result.hits().get("a").getField("filled")); + } + + private static class FillableResultSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + Hit hit = new Hit("a"); + hit.setFillable(); + result.hits().add(hit); + return result; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + for (Hit hit : result.hits().asList()) { + if ( ! hit.isFillable()) continue; + hit.setField("filled",true); + hit.setFilled("all"); + } + } + } + + static class ContextCacheSearcher extends Searcher { + final int index; + final IndexFacts[] contextsBefore; + final IndexFacts[] contextsAfter; + + ContextCacheSearcher(int index, IndexFacts[] contextsBefore, IndexFacts[] contextsAfter) { + this.index = index; + this.contextsBefore = contextsBefore; + this.contextsAfter = contextsAfter; + } + + @Override + public Result search(Query query, Execution execution) { + String s = query.properties().getString("mutatecontext"); + Set indexSet = new HashSet<>(); + for (String num : s.split(",")) { + indexSet.add(Integer.valueOf(num)); + } + + if (indexSet.contains(index)) { + execution.context().setIndexFacts(new IndexFacts()); + } + contextsBefore[index] = execution.context().getIndexFacts(); + Result r = execution.search(query); + contextsAfter[index] = execution.context().getIndexFacts(); + return r; + } + } + + public static class TestSearcher extends Searcher { + + private int counter=1; + + private TestSearcher(String id) { + super(new ComponentId(id)); + } + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + result.hits().add(new Hit(getId().stringValue() + "-" + (counter++))); + return result; + } + + } + + public static class ForwardingSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Chain forwardTo=execution.context().searchChainRegistry().getChain("someChainId"); + return new Execution(forwardTo,execution.context()).search(query); + + } + + } + + public static class FillableTestSearcher extends Searcher { + + private int counter=1; + + private FillableTestSearcher(String id) { + super(new ComponentId(id)); + } + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + Hit hit=new Hit(getId().stringValue() + "-" + counter); + hit.setFillable(); + result.hits().add(hit); + return result; + } + + public @Override void fill(Result result,String summaryClass,Execution execution) { + result.hits().add(new Hit(getId().stringValue() + "-" + (counter++) + "-filled")); // Not something one would normally do in fill + } + + } + + public static class FillableTestSearcherAtTheEnd extends FillableTestSearcher { + + private FillableTestSearcherAtTheEnd(String id) { + super(id); + } + } + + @Before("com.yahoo.search.searchchain.test.ExecutionTestCase$FillableTestSearcherAtTheEnd") + @After("com.yahoo.search.searchchain.test.ExecutionTestCase$TestSearcher") + public static class FillingSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + execution.fill(result); + return result; + } + + } + + @After("com.yahoo.search.searchchain.test.ExecutionTestCase$FillableTestSearcher") + @Before("com.yahoo.search.searchchain.test.ExecutionTestCase$TestSearcher") + public static class WorkflowSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result1=execution.search(query); + Result result2=execution.search(query); + for (Iterator i=result2.hits().iterator(); i.hasNext();) + result1.hits().add(i.next()); + result1.mergeWith(result2); + return result1; + } + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java new file mode 100644 index 00000000000..79881c06852 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.google.common.util.concurrent.AbstractFuture; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentId; +import com.yahoo.processing.response.*; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; +import static org.junit.Assert.*; + +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.yahoo.component.chain.Chain; + +/** + * Tests using the async capabilities of the Processing parent framework of searchers. + * + * @author Jon Bratseth + */ +public class FutureDataTestCase { + + @Test + public void testAsyncFederation() throws InterruptedException, ExecutionException, TimeoutException { + // Setup environment + AsyncProviderSearcher asyncProviderSearcher = new AsyncProviderSearcher(); + Searcher syncProviderSearcher = new SyncProviderSearcher(); + Chain asyncSource = new Chain(new ComponentId("async"),asyncProviderSearcher); + Chain syncSource = new Chain<>(new ComponentId("sync"),syncProviderSearcher); + SearchChainResolver searchChainResolver= + new SearchChainResolver.Builder().addSearchChain(new ComponentId("sync"),new FederationOptions().setUseByDefault(true)). + addSearchChain(new ComponentId("async"),new FederationOptions().setUseByDefault(true)). + build(); + Chain main = new Chain(new FederationSearcher(new ComponentId("federator"),searchChainResolver)); + SearchChainRegistry searchChainRegistry = new SearchChainRegistry(); + searchChainRegistry.register(main); + searchChainRegistry.register(syncSource); + searchChainRegistry.register(asyncSource); + + Result result = new Execution(main, Execution.Context.createContextStub(searchChainRegistry,null)).search(new Query()); + assertNotNull(result); + + HitGroup syncGroup = (HitGroup)result.hits().get("source:sync"); + assertNotNull(syncGroup); + + HitGroup asyncGroup = (HitGroup)result.hits().get("source:async"); + assertNotNull(asyncGroup); + + assertEquals("Got all sync data",3,syncGroup.size()); + assertEquals("sync:0",syncGroup.get(0).getId().toString()); + assertEquals("sync:1",syncGroup.get(1).getId().toString()); + assertEquals("sync:2",syncGroup.get(2).getId().toString()); + + assertTrue(asyncGroup.incoming()==asyncProviderSearcher.incomingData); + assertEquals("Got no async data yet",0,asyncGroup.size()); + asyncProviderSearcher.simulateOneHitIOComplete(new Hit("async:0")); + assertEquals("Got no async data yet, as we haven't completed the incoming buffer and there is no data listener",0,asyncGroup.size()); + asyncProviderSearcher.simulateOneHitIOComplete(new Hit("async:1")); + asyncProviderSearcher.simulateAllHitsIOComplete(); + assertEquals("Got no async data yet, as we haven't pulled it",0,asyncGroup.size()); + asyncGroup.complete().get(); + assertEquals("Completed, so we have the data",2,asyncGroup.size()); + assertEquals("async:0",asyncGroup.get(0).getId().toString()); + assertEquals("async:1",asyncGroup.get(1).getId().toString()); + } + + @Test + public void testFutureData() throws InterruptedException, ExecutionException, TimeoutException { + // Set up + AsyncProviderSearcher futureDataSource=new AsyncProviderSearcher(); + Chain chain=new Chain<>(Collections.singletonList(futureDataSource)); + + // Execute + Query query = new Query(); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + + // Verify the result prior to completion of delayed data + assertEquals("The result has been returned, but no hits are available yet", + 0, result.hits().getConcreteSize()); + + // pretend we're the IO layer and complete delayed data - this is typically done in a callback from jDisc + futureDataSource.simulateOneHitIOComplete(new Hit("hit:0")); + futureDataSource.simulateOneHitIOComplete(new Hit("hit:1")); + futureDataSource.simulateAllHitsIOComplete(); + + assertEquals("Async arriving hits are still not visible because we haven't asked for them", + 0, result.hits().getConcreteSize()); + + // Results with future hit groups will be passed to rendering directly and start rendering immediately. + // For this test we block and wait for the data instead: + result.hits().complete().get(1000, TimeUnit.MILLISECONDS); + assertEquals(2,result.hits().getConcreteSize()); + } + + /** + * A searcher which returns immediately with future data which can then be filled later, + * simulating an async searcher using a separate thread to fill in result data as it becomes available. + */ + public static class AsyncProviderSearcher extends Searcher { + + private IncomingData incomingData = null; + + @Override + public Result search(Query query, Execution execution) { + if (incomingData != null) throw new IllegalArgumentException("This test searcher is one-time use only"); + + HitGroup hitGroup=HitGroup.createAsync("Async source"); + this.incomingData = hitGroup.incoming(); + // A real implementation would do query.properties().get("jdisc.request") here + // to get the jDisc request and use it to spawn a child request to the backend + // which would eventually add to and complete incomingData + return new Result(query,hitGroup); + } + + public void simulateOneHitIOComplete(Hit hit) { + incomingData.add(hit); + } + + public void simulateAllHitsIOComplete() { + incomingData.markComplete(); + } + + } + + public static class SyncProviderSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("sync:0")); + result.hits().add(new Hit("sync:1")); + result.hits().add(new Hit("sync:2")); + return result; + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java new file mode 100644 index 00000000000..ad0c4796549 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import static com.yahoo.search.searchchain.test.SimpleSearchChain.searchChain; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.Version; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; + +/** + * Tests basic search chain functionality - creation, inheritance and ordering + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class SearchChainTestCase extends junit.framework.TestCase { + + public SearchChainTestCase(String name) { + super(name); + } + + public void testEmptySearchChain() { + SearchChain empty = new SearchChain(new ComponentId("empty")); + assertEquals("empty", empty.getId().getName()); + } + + public void testSearchChainCreation() { + assertEquals("test",searchChain.getId().stringValue()); + assertEquals("test",searchChain.getId().getName()); + assertEquals(Version.emptyVersion, searchChain.getId().getVersion()); + assertEquals(new Version(),searchChain.getId().getVersion()); + assertEqualMembers(Arrays.asList("one", "two"), searcherNames(searchChain.searchers())); + } + + public List searcherNames(Collection searchers) { + List names = new ArrayList<>(); + + for (Searcher searcher: searchers) { + names.add(searcher.getId().stringValue()); + } + + Collections.sort(names); + return names; + } + + private void assertEqualMembers(List correct,List test) { + assertEquals(new HashSet<>(correct),new HashSet<>(test)); + } + + public void testSearchChainToStringEmpty() { + assertEquals("chain 'test' []", new Chain<>(new ComponentId("test"), createSearchers(0)).toString()); + } + + public void testSearchChainToStringVeryShort() { + assertEquals("chain 'test' [s1]", new Chain<>(new ComponentId("test"),createSearchers(1)).toString()); + } + + public void testSearchChainToStringShort() { + assertEquals("chain 'test' [s1 -> s2 -> s3]", new Chain<>(new ComponentId("test"),createSearchers(3)).toString()); + } + + public void testSearchChainToStringLong() { + assertEquals("chain 'test' [s1 -> s2 -> ... -> s4]", new Chain<>(new ComponentId("test"),createSearchers(4)).toString()); + } + + public void testSearchChainToStringVeryLong() { + assertEquals("chain 'test' [s1 -> s2 -> ... -> s10]", new Chain<>(new ComponentId("test"),createSearchers(10)).toString()); + } + + private List createSearchers(int count) { + List searchers=new ArrayList<>(count); + for (int i=0; i getSearchChainsForwarded(SearchChainRegistry registry) { + return Arrays.asList( + new ForkingSearcher.CommentedSearchChain("Reason for forwarding to this search chain.", dummySearchChain()), + new ForkingSearcher.CommentedSearchChain(null, dummySearchChain())); + } + + private SearchChain dummySearchChain() { + return new SearchChain(new ComponentId("child-chain"), + new DummySearcher(new ComponentId("child-searcher")) {}); + } + + } + + @Provides("Test") + private static class TestSearcher extends BaseSearcher { + + public TestSearcher(ComponentId id) { + super(id); + } + + } + + private static class DummySearcher extends Searcher { + + public DummySearcher(ComponentId id) { + super(id); + } + + @Override + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + @After("Test") + private static class TestSearcher2 extends BaseSearcher { + + public TestSearcher2(ComponentId id) { + super(id); + } + + @Override + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + private static List twoSearchers(String id1, String id2, boolean ordered) { + List searchers=new ArrayList<>(); + searchers.add(new TestSearcher(new ComponentId(id1))); + searchers.add(createSecondSearcher(new ComponentId(id2), ordered)); + return searchers; + } + + private static Searcher createSecondSearcher(ComponentId componentId, boolean ordered) { + if (ordered) + return new TestSearcher2(componentId); + else + return new TestSearcher(componentId); + } + + private static SearchChain createSearchChain(boolean ordered) { + return new SearchChain(new ComponentId("test"), twoSearchers("one","two", ordered)); + } + + public static final SearchChain searchChain = createSearchChain(false); + public static final SearchChain orderedChain = createSearchChain(true); + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java new file mode 100644 index 00000000000..92091fabbf9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java @@ -0,0 +1,221 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * Tests tracing scenarios where traces from multiple executions over the same query are involved. + * + * @author Jon Bratseth + */ +public class TraceTestCase extends junit.framework.TestCase { + + public void testTracingOnCorrectAPIUseNonParallel() { + assertTracing(true,false); + } + + public void testTracingOnIncorrectAPIUseNonParallel() { + assertTracing(false,false); + } + + public void testTracingOnCorrectAPIUseParallel() { + assertTracing(true, true); + } + + public void testTracingOnIncorrectAPIUseParallel() { + assertTracing(false,true); + } + + @SuppressWarnings("deprecation") + public void assertTracing(boolean carryOverContext,boolean parallel) { + Query query=new Query("?tracelevel=1"); + query.trace("Before execution",1); + Chain forkingChain=new Chain<>(new Tracer("forker"),new Forker(carryOverContext,parallel,new Tracer("branch 1"),new Tracer("branch 2"))); + new Execution(forkingChain, Execution.Context.createContextStub()).search(query); + + // printTrace(query); + + if (carryOverContext) + assertTraceWithChildExecutionMessages(query); + else if (parallel) + assertTrace(query); + else + assertIncorrectlyNestedTrace(query); + + assertCorrectRendering(query); + } + + // The valid and usual trace + private void assertTraceWithChildExecutionMessages(Query query) { + Iterator trace=collectTrace(query).iterator(); + assertEquals("(level start)",trace.next()); + assertEquals(" No query profile is used",trace.next()); + assertEquals(" Before execution",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During forker: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 1: 0",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 2: 0", trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals("(level end)",trace.next()); + assertFalse(trace.hasNext()); + } + + // With incorrect API usage and query cloning (in parallel use) we get a valid trace + // where the message of the execution subtrees is empty rather than "child execution". This is fine. + private void assertTrace(Query query) { + Iterator trace=collectTrace(query).iterator(); + assertEquals("(level start)",trace.next()); + assertEquals(" No query profile is used",trace.next()); + assertEquals(" Before execution",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During forker: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 1: 0",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 2: 0", trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals("(level end)",trace.next()); + assertFalse(trace.hasNext()); + } + + // With incorrect usage and no query cloning the trace nesting becomes incorrect + // but all the trace messages are present. + private void assertIncorrectlyNestedTrace(Query query) { + Iterator trace=collectTrace(query).iterator(); + assertEquals("(level start)",trace.next()); + assertEquals(" No query profile is used",trace.next()); + assertEquals(" Before execution",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During forker: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 1: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 2: 0", trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals("(level end)",trace.next()); + assertFalse(trace.hasNext()); + } + + private void assertCorrectRendering(Query query) { + try { + StringWriter writer=new StringWriter(); + query.getContext(false).render(writer); + String expected= + "\n" + + "\n" + + "

No query profile is used

\n" + + "\n" + + "

Before execution

\n" + + "\n" + + "

\n" + + "

During forker: 0"; + assertEquals(expected,writer.toString().substring(0,expected.length())); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + private List collectTrace(Query query) { + TraceCollector collector=new TraceCollector(); + query.getContext(false).getTrace().accept(collector); + return collector.trace(); + } + + private static class TraceCollector extends TraceVisitor { + + private List trace=new ArrayList<>(); + private StringBuilder indent=new StringBuilder(); + + @Override + public void entering(TraceNode node) { + trace.add(indent + "(level start)"); + indent.append(" "); + } + + @Override + public void leaving(TraceNode end) { + indent.setLength(indent.length()-2); + trace.add(indent + "(level end)"); + } + + @Override + public void visit(TraceNode node) { + if (node.isRoot()) return; + if (node.payload()==null) return; + trace.add(indent + node.payload().toString()); + } + + public List trace() { return trace; } + } + + private static class Tracer extends Searcher { + + private String name; + private int counter=0; + + public Tracer(String name) { + this.name=name; + } + + @Override + public Result search(Query query, Execution execution) { + query.trace("During " + name + ": " + (counter++) ,1); + return execution.search(query); + } + } + + private static class Forker extends Searcher { + + private List branches; + + /** If true, this is using the api as recommended, if false, it is not */ + private boolean carryOverContext; + + /** If true, simulate parallel execution by cloning the query */ + private boolean parallel; + + public Forker(boolean carryOverContext,boolean parallel,Searcher ... branches) { + this.carryOverContext=carryOverContext; + this.parallel=parallel; + this.branches=Arrays.asList(branches); + } + + @SuppressWarnings("deprecation") + @Override + public Result search(Query query, Execution execution) { + Result result=execution.search(query); + for (Searcher branch : branches) { + Query branchQuery=parallel ? query.clone() : query; + Result branchResult= + ( carryOverContext ? new Execution(branch,execution.context()) : new Execution(branch, Execution.Context.createContextStub())).search(branchQuery); + result.hits().add(branchResult.hits()); + result.mergeWith(branchResult); + } + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java new file mode 100644 index 00000000000..954290de6a2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +/** + * Externally provided test for async execution of search chains. + * + * @author Peter Thomas + */ +public class VespaAsyncSearcherTest extends TestCase { + private static class FirstSearcher extends Searcher { + + @Override + public Result search(Query query, Execution exctn) { + int count = 10; + List futures = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + Query subQuery = new Query(); + FutureResult future = new AsyncExecution(exctn) + .search(subQuery); + futures.add(future); + } + AsyncExecution.waitForAll(futures, 10 * 60 * 1000); + return new Result(query); + } + + } + + private static class SecondSearcher extends Searcher { + + @Override + public Result search(Query query, Execution exctn) { + return new Result(query); + } + + } + + public void testAsyncExecution() { + Chain chain = new Chain<>(new FirstSearcher(), + new SecondSearcher()); + Execution execution = new Execution(chain, + Execution.Context.createContextStub(null)); + Query query = new Query(); + // fails with exception on old versions + execution.search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java new file mode 100644 index 00000000000..b112e61cb44 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.CacheControlSearcher; +import junit.framework.TestCase; + +import java.util.List; + +import static com.yahoo.search.searchers.CacheControlSearcher.CACHE_CONTROL_HEADER; + +/** + * Unit test cases for CacheControlSearcher. + * + * @author Frode Lundgren + */ +@SuppressWarnings("deprecation") +public class CacheControlSearcherTestCase extends TestCase { + + private Searcher getDocSource() { + return new Searcher() { + public Result search(Query query, Execution execution) { + Result res = new Result(query); + res.setTotalHitCount(1); + Hit hit = new Hit("http://document/", 1000); + hit.setField("url", "http://document/"); + hit.setField("title", "Article title"); + hit.setField("extsourceid", "12345"); + res.hits().add(hit); + return res; + } + }; + } + + private Chain getSearchChain() { + return new Chain<>(new CacheControlSearcher(), getDocSource()); + } + + private List getCacheControlHeaders(Result result) { + return result.getHeaders(true).get(CACHE_CONTROL_HEADER); + } + + /** + * Assert that cache header ListMap exactly match given array of expected cache headers + * @param values - Array of cache control headers expected, e.g. {"max-age=120", "stale-while-revalidate=3600"} + * @param cacheheaders - The "Cache-Control" headers from the response ListMap + */ + private void assertCacheHeaders(String[] values, List cacheheaders) { + assertNotNull("No headers to test for (was null)", values); + assertTrue("No headers to test for (no elements in array)", values.length > 0); + assertNotNull("No cache headers set in response", cacheheaders); + assertEquals(values.length, cacheheaders.size()); + for (String header : values) { + assertTrue("Cache header does not contain header '" + header + "'", cacheheaders.contains(header)); + } + } + + public void testNoHeader() { + Chain chain = getSearchChain(); + Query query = new Query("?query=foo&custid=foo"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(0, getCacheControlHeaders(result).size()); + } + + public void testInvalidAgeParams() { + Chain chain = getSearchChain(); + + try { + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=foo"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(0, getCacheControlHeaders(result).size()); + fail("Expected exception"); + } + catch (NumberFormatException e) { + // success + } + + try { + Query query = new Query("?query=foo&custid=foo&cachecontrol.staleage=foo"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(0, getCacheControlHeaders(result).size()); + fail("Expected exception"); + } + catch (NumberFormatException e) { + // success + } + } + + public void testMaxAge() { + Chain chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=120"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"max-age=120"}, getCacheControlHeaders(result)); + } + + public void testNoCache() { + Chain chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=120&noCache"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"no-cache"}, getCacheControlHeaders(result)); + + query = new Query("?query=foo&custid=foo&cachecontrol.maxage=120&cachecontrol.nocache=true"); + result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"no-cache"}, getCacheControlHeaders(result)); + } + + public void testStateWhileRevalidate() { + Chain chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.staleage=3600"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"stale-while-revalidate=3600"}, getCacheControlHeaders(result)); + } + + public void testStaleAndMaxAge() { + Chain chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=60&cachecontrol.staleage=3600"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"max-age=60", "stale-while-revalidate=3600"}, getCacheControlHeaders(result)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java new file mode 100644 index 00000000000..5e8596ef16d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.net.SocketAddress; +import java.net.URI; +import java.net.URISyntaxException; + +import org.junit.Test; +import org.mockito.Mockito; + +import com.yahoo.component.chain.Chain; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.Container; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.HttpRequest.Version; +import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.ConnectionControlSearcher; + +/** + * Functionality tests for + * {@link com.yahoo.search.searchers.ConnectionControlSearcher}. + * + * @author Steinar Knutsen + */ +public class ConnectionControlSearcherTestCase { + + @Test + public final void test() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/?connectioncontrol.maxlifetime=1"); + long connectedAtMillis = 0L; + long nowMillis = 2L * 1000L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertEquals("Close", r.getHeaders(false).get("Connection").get(0)); + } + + @Test + public final void testForcedClose() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/?connectioncontrol.maxlifetime=0"); + long connectedAtMillis = 0L; + long nowMillis = 0L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertEquals("Close", r.getHeaders(false).get("Connection").get(0)); + } + + @Test + public final void testNormalCloseWithoutJdisc() { + long nowMillis = 2L; + Query query = new Query("/?connectioncontrol.maxlifetime=1"); + Execution e = new Execution(new Chain(ConnectionControlSearcher.createTestInstance(() -> nowMillis)), + Execution.Context.createContextStub()); + Result r = e.search(query); + assertNull(r.getHeaders(false)); + } + + @Test + public final void testNoMaxLifetime() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/"); + long connectedAtMillis = 0L; + long nowMillis = 0L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertNull(r.getHeaders(false)); + } + + @Test + public final void testYoungEnoughConnection() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/?connectioncontrol.maxlifetime=1"); + long connectedAtMillis = 0L; + long nowMillis = 500L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertNull(r.getHeaders(false)); + } + + + private Result doSearch(URI uri, long connectedAtMillis, long nowMillis) { + SocketAddress remoteAddress = Mockito.mock(SocketAddress.class); + Version version = Version.HTTP_1_1; + Method method = Method.GET; + CurrentContainer container = Mockito.mock(CurrentContainer.class); + Mockito.when(container.newReference(Mockito.any())).thenReturn(Mockito.mock(Container.class)); + final com.yahoo.jdisc.http.HttpRequest serverRequest = com.yahoo.jdisc.http.HttpRequest + .newServerRequest(container, uri, method, version, remoteAddress, connectedAtMillis); + HttpRequest incoming = new HttpRequest(serverRequest, new ByteArrayInputStream(new byte[0])); + Query query = new Query(incoming); + Execution e = new Execution(new Chain(ConnectionControlSearcher.createTestInstance(() -> nowMillis)), + Execution.Context.createContextStub()); + Result r = e.search(query); + return r; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java new file mode 100644 index 00000000000..ff521da1aad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import static org.junit.Assert.*; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.InputCheckingSearcher; +import com.yahoo.text.Utf8; + +/** + * Functional test for InputCheckingSearcher. + * + * @author Steinar Knutsen + */ +public class InputCheckingSearcherTestCase { + + Execution execution; + + @Before + public void setUp() throws Exception { + execution = new Execution(new Chain(new InputCheckingSearcher(MetricReceiver.nullImplementation)), + Execution.Context.createContextStub(new IndexFacts())); + } + + @After + public void tearDown() throws Exception { + execution = null; + } + + @Test + public final void testCommonCase() { + Result r = execution.search(new Query("/search/?query=three+blind+mice")); + assertNull(r.hits().getErrorHit()); + } + + @Test + public final void candidateButAsciiOnly() { + Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a")); + assertNull(r.hits().getErrorHit()); + } + + @Test + public final void candidateButValid() throws UnsupportedEncodingException { + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8"))); + assertNull(r.hits().getErrorHit()); + } + + @Test + public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8"))); + assertNull(r.hits().getErrorHit()); + } + + + @Test + public final void testDoubleEncoded() throws UnsupportedEncodingException { + String rawQuery = "å å å å å å"; + byte[] encodedOnce = Utf8.toBytes(rawQuery); + char[] secondEncodingBuffer = new char[encodedOnce.length]; + for (int i = 0; i < secondEncodingBuffer.length; ++i) { + secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF); + } + String query = new String(secondEncodingBuffer); + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8"))); + assertEquals(1, r.hits().getErrorHit().errors().size()); + } + + @Test + public final void testRepeatedConsecutiveTermsInPhrase() { + Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + assertNotNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + } + @Test + public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { + Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + assertNotNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + } + @Test + public final void testThatMaxRepeatedTermsInPhraseIs10() { + Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a")); + assertNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a")); + assertNotNull(r.hits().getErrorHit()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java b/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java new file mode 100644 index 00000000000..aaad8ba80ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.jdisc.Metric; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** +* @author bratseth +*/ +class MockMetric implements Metric { + + private Map> metrics = new HashMap<>(); + + public Map values(Context context) { + return metricsForContext(context); + } + + @Override + public void set(String key, Number val, Context context) { + metricsForContext(context).put(key, val); + } + + @Override + public void add(String key, Number value, Context context) { + Number previousValue = metricsForContext(context).get(key); + if (previousValue == null) + previousValue = 0; + metricsForContext(context).put(key, value.doubleValue() + previousValue.doubleValue()); + } + + /** Returns the metrics for a given context, never null */ + private Map metricsForContext(Context context) { + Map metricsForContext = metrics.get(context); + if (metricsForContext == null) { + metricsForContext = new HashMap<>(); + metrics.put(context, metricsForContext); + } + return metricsForContext; + } + + @Override + public Context createContext(Map dimensions) { + return new MapContext(dimensions); + } + + /** Creates a context containing a single dimension */ + public Metric.Context createContext(String dimensionName, String dimensionValue) { + if (dimensionName.isEmpty()) + return createContext(Collections.emptyMap()); + return createContext(Collections.singletonMap(dimensionName, dimensionValue)); + } + + private class MapContext implements Metric.Context { + + private final Map dimensions; + + public MapContext(Map dimensions) { + this.dimensions = dimensions; + } + + @Override + public int hashCode() { + return dimensions.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof MapContext)) return false; + return dimensions.equals(((MapContext)o).dimensions); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java new file mode 100644 index 00000000000..9381cf2ab7e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java @@ -0,0 +1,208 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.component.chain.Chain; +import com.yahoo.jdisc.Metric; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.RateLimitingConfig; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.RateLimitingSearcher; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +/** + * A benchmark and multithread stress test of rate limiting. + * The purpose of this is to simulate the environment the rate limiter will work under in production + * and verify that it manages to keep rates more or less within set bounds and does not lead to excessive contention. + * + * @author bratseth + */ +public class RateLimitingBenchmark { + + private final int clientCount = 10; + private final int threadCount = 250; + private final int epochs = 100; // the number of times the sequence of load types are repeated + private final int totalQueriesPerThread = 4 * 1000 * 10; + + // This number produces a theoretical max request rate of 1000/5*threadCount = 50 k rps + // which in practice on my machine is about 40 k rps. + // With the number set to 0 my machine does about 150 k rps. + // This means that peaks (when it is zero) are roughly 3x base. + private final int sleepMsBetweenRequests = 5; + private final int peakDurationMs = 1000; + private final int timeBetweenPeaksMs = 2000; + + private final Chain chain; + private final MockMetric metric; + + private final Map requestCounters = new HashMap<>(); + + public RateLimitingBenchmark() { + RateLimitingConfig.Builder rateLimitingConfig = new RateLimitingConfig.Builder(); + /* Defaults: + rateLimitingConfig.maxAvailableCapacity(10000); + rateLimitingConfig.capacityIncrement(1000); + rateLimitingConfig.recheckForCapacityProbability(0.001); + */ + + rateLimitingConfig.maxAvailableCapacity(10000); + rateLimitingConfig.capacityIncrement(1000); + rateLimitingConfig.recheckForCapacityProbability(0.001); + + ClusterInfoConfig.Builder clusterInfoConfig = new ClusterInfoConfig.Builder(); + clusterInfoConfig.clusterId("testCluster"); + clusterInfoConfig.nodeCount(1); + + this.metric = new MockMetric(); + + chain = new Chain<>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), + new ClusterInfoConfig(clusterInfoConfig), metric)); + + for (int i = 0; i < clientCount ; i++) + requestCounters.put(toClientId(i), new RequestCounts()); + } + + public void run() throws InterruptedException { + long startTime = System.currentTimeMillis(); + runWorkers(); + long totalTime = Math.max(1, System.currentTimeMillis() - startTime); + + double totalAttemptedRate = 0; + for (int i=0; i < clientCount; i++) { + double attemptedRate = requestCounters.get(toClientId(i)).attempted.get() * 1000d / totalTime; + double allowedRate = requestCounters.get(toClientId(i)).allowed.get() * 1000d / totalTime; + System.out.println(String.format(Locale.ENGLISH, + "Client %1$2d: Attempted rate: %2$10.2f. Target allowed rate: %3$10.2f. Allowed rate: %4$10.2f. Rejected requests: %5$8d", + i, attemptedRate, Math.pow(4, i), allowedRate, rejectedRequests(i))); + totalAttemptedRate += attemptedRate; + } + System.out.println(String.format(Locale.ENGLISH, "\nTotal attempted rate: %1$10.2f seconds", totalAttemptedRate)); + System.out.println(String.format(Locale.ENGLISH, "\nTotal time: %1$8.2f seconds", totalTime/1000.0)); + } + + private void runWorkers() { + try { + long startTime = System.currentTimeMillis(); + + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) + threads[i] = new Thread(new Worker(startTime)); + + for (int i = 0; i < threadCount; i++) + threads[i].start(); + + for (int i = 0; i < threadCount; i++) + threads[i].join(); + } + catch (Exception e) { // not production code + throw new RuntimeException(e); + } + } + + private int rejectedRequests(int id) { + Metric.Context context = metric.createContext("id", toClientId(id)); + Number rejectedRequestsMetric = metric.values(context).get("requestsOverQuota"); + if (rejectedRequestsMetric == null) return 0; + return rejectedRequestsMetric.intValue(); + } + + private class Worker implements Runnable { + + private final int sequences = 5; + private final long startTime; + + public Worker(long startTime) { + this.startTime = startTime; + } + + @Override + public void run() { + try { + for (int i = 0; i < epochs; i++) { + issueRequests(this::pickClientFairly); + issueRequests(this::pickClientSkewedToLowerNumbers); + issueRequests(this::pickClientSkewedToHigherNumbers); + issueRequests(this::pickClientFairly); + issueRequests(this::pickClientSkewedToHigherNumbers); + } + } + catch (InterruptedException e) { + // just end + } + } + + private void issueRequests(Supplier clientNumberSupplier) throws InterruptedException { + for (int i = 0; i< totalQueriesPerThread/(epochs * sequences); i++) { + int clientNumber = clientNumberSupplier.get(); + requestCounters.get(toClientId(clientNumber)).addRequest(executeWasAllowed(chain, clientNumber)); + if ( ! isInPeak()) + Thread.sleep(sleepMsBetweenRequests); + } + } + + private boolean isInPeak() { + long timeSinceStart = System.currentTimeMillis() - startTime; + return timeSinceStart % timeBetweenPeaksMs < peakDurationMs; // a peak is at every start of every timeBetweenPeaks interval + } + + protected int pickClientFairly() { + return ThreadLocalRandom.current().nextInt(clientCount); + } + + protected int pickClientSkewedToLowerNumbers() { + int nr = (int)Math.floor((Math.pow(ThreadLocalRandom.current().nextDouble(), 3) * clientCount)); + if (nr > clientCount-1) return clientCount-1; + return nr; + } + + protected int pickClientSkewedToHigherNumbers() { + int nr = (int)Math.floor( ( 1- Math.pow(ThreadLocalRandom.current().nextDouble(), 3)) * clientCount); + if (nr > clientCount-1) return clientCount-1; + return nr; + } + + } + + private String toClientId(int n) { + return "id" + n; + } + + private boolean executeWasAllowed(Chain chain, int id) { + Query query = new Query(); + query.properties().set("rate.id", toClientId(id)); + query.properties().set("rate.cost", 1); + query.properties().set("rate.quota", Math.pow(4, id)); + query.properties().set("rate.idDimension", "id"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + if (result.hits().getError() != null && result.hits().getError().getCode() == 429) + return false; + else + return true; + } + + + public static void main(String[] args) throws InterruptedException { + new RateLimitingBenchmark().run(); + } + + private static class RequestCounts { + + private AtomicInteger attempted = new AtomicInteger(0); + private AtomicInteger allowed = new AtomicInteger(0); + + public void addRequest(boolean wasAllowed) { + attempted.incrementAndGet(); + if (wasAllowed) allowed.incrementAndGet(); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java new file mode 100755 index 00000000000..02d6620df2e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.RateLimitingConfig; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.RateLimitingSearcher; +import com.yahoo.yolean.chain.After; +import org.junit.Test; +import com.yahoo.test.ManualClock; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Unit tests for RateLimitingSearcher + * + * @author bratseth + */ +public class RateLimitingSearcherTestCase { + + @Test + public void testRateLimiting() { + RateLimitingConfig.Builder rateLimitingConfig = new RateLimitingConfig.Builder(); + rateLimitingConfig.maxAvailableCapacity(4); + rateLimitingConfig.capacityIncrement(2); + rateLimitingConfig.recheckForCapacityProbability(1.0); + + ClusterInfoConfig.Builder clusterInfoConfig = new ClusterInfoConfig.Builder(); + clusterInfoConfig.clusterId("testCluster"); + clusterInfoConfig.nodeCount(4); + + ManualClock clock = new ManualClock(); + MockMetric metric = new MockMetric(); + + Chain chain = new Chain("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), + new ClusterInfoConfig(clusterInfoConfig), + metric, clock), + new CostSettingSearcher()); + assertEquals("'rate' request are available initially", 2, tryRequests(chain, "id1")); + assertTrue("However, don't reject if we dryRun", executeWasAllowed(chain, "id1", true)); + clock.advance(Duration.ofMillis(1500)); // causes 2 new requests to become available + assertEquals("'rate' new requests became available", 2, tryRequests(chain, "id1")); + + assertEquals("Another id", 2, tryRequests(chain, "id2")); + + clock.advance(Duration.ofMillis(1000000)); + assertEquals("'maxAvailableCapacity' request became available", 4, tryRequests(chain, "id2")); + + assertFalse("If quota is set to 0, all requests are rejected, even initially", executeWasAllowed(chain, "id3", 0)); + + clock.advance(Duration.ofMillis(1000000)); + assertTrue("A single query which costs more than capacity is allowed as cost is calculated after allowing it", + executeWasAllowed(chain, "id1", 8, 8, false)); + assertFalse("capacity is -4: disallowing", executeWasAllowed(chain, "id1")); + clock.advance(Duration.ofMillis(1000)); + assertFalse("capacity is -2: disallowing", executeWasAllowed(chain, "id1")); + clock.advance(Duration.ofMillis(1000)); + assertFalse("capacity is 0: disallowing", executeWasAllowed(chain, "id1")); + clock.advance(Duration.ofMillis(1000)); + assertTrue(executeWasAllowed(chain, "id1")); + + // check metrics + assertEquals((double)requestsToTry-2 + 1 + requestsToTry-2 + 3, metric.values(metric.createContext("id", "id1")).get("requestsOverQuota")); + assertEquals((double)requestsToTry-2 + requestsToTry-4, metric.values(metric.createContext("id", "id2")).get("requestsOverQuota")); + } + + private int requestsToTry = 50; + + /** + * Try many requests and return how many was allowed. + * This is to avoid testing the exact pattern of request/deny which does not matter + * and is determined by floating point arithmetic details when capacity is close to zero. + */ + private int tryRequests(Chain chain, String id) { + int allowedCount = 0; + for (int i = 0; i < requestsToTry; i++) { + if (executeWasAllowed(chain, id)) + allowedCount++; + } + return allowedCount; + } + + private boolean executeWasAllowed(Chain chain, String id) { + return executeWasAllowed(chain, id, 8); // allowed 8 requests per second over 4 nodes -> 2 per node + } + + private boolean executeWasAllowed(Chain chain, String id, boolean dryRun) { + return executeWasAllowed(chain, id, 8, 1, dryRun); + } + + private boolean executeWasAllowed(Chain chain, String id, int quota) { + return executeWasAllowed(chain, id, quota, 1, false); + } + + private boolean executeWasAllowed(Chain chain, String id, double quota, double cost, boolean dryRun) { + Query query = new Query(); + query.properties().set("rate.id", id); + query.properties().set("cost", cost); // converted to rate.cost by a searcher executing after rate limiting + query.properties().set("rate.quota", quota); + query.properties().set("rate.idDimension", "id"); + query.properties().set("rate.dryRun", dryRun); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + if (result.hits().getError() != null && result.hits().getError().getCode() == 429) + return false; + else + return true; + } + + /** The purpose of this test is simply to verify that cost is picked up after executing the query */ + @After(RateLimitingSearcher.RATE_LIMITING) + private static class CostSettingSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + query.properties().set("rate.cost", query.properties().get("cost")); + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java new file mode 100644 index 00000000000..4f7654ba3c0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.RawSource; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.ValidateMatchPhaseSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author balder + */ +public class ValidateMatchPhaseSearcherTestCase { + + private ValidateMatchPhaseSearcher searcher; + + public ValidateMatchPhaseSearcherTestCase() { + searcher = new ValidateMatchPhaseSearcher( + ConfigGetter.getConfig(AttributesConfig.class, + "raw:", + new RawSource("attribute[4]\n" + + "attribute[0].name ok\n" + + "attribute[0].datatype INT32\n" + + "attribute[0].collectiontype SINGLE\n" + + "attribute[0].fastsearch true\n" + + "attribute[1].name not_fast\n" + + "attribute[1].datatype INT32\n" + + "attribute[1].collectiontype SINGLE\n" + + "attribute[1].fastsearch false\n" + + "attribute[2].name not_numeric\n" + + "attribute[2].datatype STRING\n" + + "attribute[2].collectiontype SINGLE\n" + + "attribute[2].fastsearch true\n" + + "attribute[3].name not_single\n" + + "attribute[3].datatype INT32\n" + + "attribute[3].collectiontype ARRAY\n" + + "attribute[3].fastsearch true" + ))); + } + + private static String getErrorMatch(String attribute) { + return "4: Invalid query parameter: The attribute '" + + attribute + + "' is not available for match-phase. It must be a single value numeric attribute with fast-search."; + } + + private static String getErrorDiversity(String attribute) { + return "4: Invalid query parameter: The attribute '" + + attribute + + "' is not available for match-phase diversification. It must be a single value numeric or string attribute."; + } + + @Test + public void testMatchPhaseAttribute() { + assertEquals("", search("")); + assertEquals("", match("ok")); + assertEquals(getErrorMatch("not_numeric"), match("not_numeric")); + assertEquals(getErrorMatch("not_single"), match("not_single")); + assertEquals(getErrorMatch("not_fast"), match("not_fast")); + assertEquals(getErrorMatch("not_found"), match("not_found")); + } + + @Test + public void testDiversityAttribute() { + assertEquals("", search("")); + assertEquals("", diversify("ok")); + assertEquals("", diversify("not_numeric")); + assertEquals(getErrorDiversity("not_single"), diversify("not_single")); + assertEquals("", diversify("not_fast")); + assertEquals(getErrorDiversity("not_found"), diversify("not_found")); + } + + private String match(String m) { + return search("&ranking.matchPhase.attribute=" + m); + } + + private String diversify(String m) { + return search("&ranking.matchPhase.attribute=ok&ranking.matchPhase.diversity.attribute=" + m); + } + + private String search(String m) { + String q = "/?query=sddocname:test" + m; + Result r = doSearch(searcher, new Query(q), 0, 10); + if (r.hits().getError() != null) { + return r.hits().getError().toString(); + } + return ""; + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain chainedAsSearchChain(Searcher topOfChain) { + List searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java new file mode 100644 index 00000000000..43563e29218 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java @@ -0,0 +1,433 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.statistics; + +import junit.framework.TestCase; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.ElapsedTime; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.search.statistics.TimeTracker.Activity; +import com.yahoo.search.statistics.TimeTracker.SearcherTimer; + +/** + * Check sanity of TimeTracker and ElapsedTime. + * + * @author Steinar Knutsen + */ +public class ElapsedTimeTestCase extends TestCase { + + private static final long[] SEARCH_TIMESEQUENCE = new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L }; + + private static final long[] SEARCH_AND_FILL_TIMESEQUENCE = new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L, + // and here we start filling + 7L, 8L, 9L, 10L, 11L, 12L, 13L }; + + public static class CreativeTimeSource extends TimeTracker.TimeSource { + private int nowIndex = 0; + private long[] now; + + public CreativeTimeSource(long[] now) { + this.now = now; + } + + @Override + long now() { + long present = now[nowIndex++]; + if (present == 0L) { + // defensive coding against the innards of TimeTracker + throw new IllegalStateException("0 is an unsupported time stamp value."); + } + return present; + } + + } + + public static class UselessSearcher extends Searcher { + public UselessSearcher(String name) { + super(new ComponentId(name)); + } + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + } + + private static class AlmostUselessSearcher extends Searcher { + AlmostUselessSearcher(String name) { + super(new ComponentId(name)); + } + + @Override + public Result search(Query query, Execution execution) { + Result r = execution.search(query); + Hit h = new Hit("nalle"); + h.setFillable(); + r.hits().add(h); + return r; + } + } + + private static class NoForwardSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Result r = new Result(query); + Hit h = new Hit("nalle"); + h.setFillable(); + r.hits().add(h); + return r; + } + } + + private class TestingSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Execution exec = new Execution(execution); + exec.timer().injectTimeSource( + new CreativeTimeSource(SEARCH_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + Result r = exec.search(new Query()); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + assertNull(searchers[0].getInvoking(Activity.SEARCH)); + checkTiming(searchers, 1); + return r; + } + } + + private class SecondTestingSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Execution exec = new Execution(execution); + exec.timer().injectTimeSource( + new CreativeTimeSource(SEARCH_AND_FILL_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + Result result = exec.search(new Query()); + exec.fill(result); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + assertNull(searchers[0].getInvoking(Activity.SEARCH)); + checkTiming(searchers, 1); + assertNull(searchers[0].getInvoking(Activity.FILL)); + checkFillTiming(searchers, 1); + return result; + } + } + + private class ShortChainTestingSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Execution exec = new Execution(execution); + exec.timer().injectTimeSource( + new CreativeTimeSource(new long[] { 1L, 2L, 2L })); + exec.context().setDetailedDiagnostics(true); + Result result = exec.search(new Query()); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + assertNull(searchers[0].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.SEARCH)); + assertNull(searchers[1].getReturning(Activity.SEARCH)); + assertNull(searchers[0].getInvoking(Activity.FILL)); + assertNull(searchers[1].getInvoking(Activity.FILL)); + assertTrue(0 < result.getElapsedTime().detailedReport().indexOf("NoForwardSearcher")); + return result; + } + } + + public void testBasic() { + TimeTracker t = new TimeTracker(null); + t.injectTimeSource(new CreativeTimeSource(new long[] {1L, 2L, 3L, 4L})); + Query q = new Query(); + Result r = new Result(q); + t.sampleSearch(0, false); + t.sampleFill(0, false); + t.samplePing(0, false); + t.sampleSearchReturn(0, false, r); + assertEquals(1L, t.first()); + assertEquals(4L, t.last()); + assertEquals(2L, t.firstFill()); + assertEquals(1L, t.searchTime()); + assertEquals(1L, t.fillTime()); + assertEquals(1L, t.pingTime()); + assertEquals(3L, t.totalTime()); + } + + public void testMultiSearchAndPing() { + TimeTracker t = new TimeTracker(null); + t.injectTimeSource(new CreativeTimeSource(new long[] {1L, 4L, 16L, 32L, 64L, 128L, 256L})); + Query q = new Query(); + Result r = new Result(q); + t.sampleSearch(0, false); + t.samplePing(0, false); + t.sampleSearch(0, false); + t.samplePing(0, false); + t.sampleSearch(0, false); + t.sampleFill(0, false); + t.sampleSearchReturn(0, false, r); + assertEquals(1L, t.first()); + assertEquals(256L, t.last()); + assertEquals(128L, t.firstFill()); + assertEquals(83L, t.searchTime()); + assertEquals(128L, t.fillTime()); + assertEquals(44L, t.pingTime()); + assertEquals(255L, t.totalTime()); + ElapsedTime e = new ElapsedTime(); + e.add(t); + e.add(t); + // multiple adds is supposed to be safe + assertEquals(255L, t.totalTime()); + TimeTracker tx = new TimeTracker(null); + tx.injectTimeSource(new CreativeTimeSource(new long[] {1L, 2L, 3L, 4L})); + Query qx = new Query(); + Result rx = new Result(qx); + tx.sampleSearch(0, false); + tx.sampleFill(0, false); + tx.samplePing(0, false); + tx.sampleSearchReturn(0, false, rx); + e.add(tx); + assertEquals(258L, e.totalTime()); + assertEquals(129L, e.fillTime()); + assertEquals(2L, e.firstFill()); + } + + public void testBasicBreakdown() { + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + t.injectTimeSource(new CreativeTimeSource(new long[] { 1L, 2L, 3L, + 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + SearcherTimer[] searchers = t.searcherTracking(); + checkTiming(searchers); + } + + // This test is to make sure the other tests correctly simulate the call + // order into the TimeTracker + public void testBasicBreakdownFullyWiredIn() { + Chain chain = new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.timer().injectTimeSource(new CreativeTimeSource(SEARCH_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + exec.search(new Query()); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + checkTiming(searchers); + } + + + private void checkTiming(SearcherTimer[] searchers) { + checkTiming(searchers, 0); + } + + private void checkTiming(SearcherTimer[] searchers, int offset) { + assertEquals(Long.valueOf(1L), searchers[0 + offset].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[0 + offset].getReturning(Activity.SEARCH)); + } + + public void testBasicBreakdownWithFillFullyWiredIn() { + Chain chain = new Chain<>( + new UselessSearcher("first"), new UselessSearcher("second"), + new AlmostUselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.timer().injectTimeSource( + new CreativeTimeSource(SEARCH_AND_FILL_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + Result result = exec.search(new Query()); + exec.fill(result); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + checkTiming(searchers); + checkFillTiming(searchers); + } + + private void checkFillTiming(SearcherTimer[] searchers) { + checkFillTiming(searchers, 0); + } + + private void checkFillTiming(SearcherTimer[] searchers, int offset) { + assertEquals(Long.valueOf(1L), searchers[0 + offset].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getReturning(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getReturning(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[0 + offset].getReturning(Activity.FILL)); + } + + public void testBasicBreakdownFullyWiredInFirstSearcherNotFirstInChain() { + Chain chain = new Chain<>( + new TestingSearcher(), + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.search(new Query()); + } + + public void testBasicBreakdownWithFillFullyWiredInFirstSearcherNotFirstInChain() { + Chain chain = new Chain<>( + new SecondTestingSearcher(), + new UselessSearcher("first"), new UselessSearcher("second"), + new AlmostUselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.search(new Query()); + } + + public void testTimingWithShortChain() { + Chain chain = new Chain<>( + new ShortChainTestingSearcher(), + new NoForwardSearcher()); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.search(new Query()); + } + + public void testBasicBreakdownReturnInsideSearchChain() { + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + t.injectTimeSource(new CreativeTimeSource(new long[] { 1L, 2L, 3L, + 4L, 5L, 6L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + SearcherTimer[] searchers = t.searcherTracking(); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2].getInvoking(Activity.SEARCH)); + assertNull(searchers[2].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L) ,searchers[1].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L) ,searchers[0].getReturning(Activity.SEARCH)); + } + + public void testBasicBreakdownWithFill() { + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + t.injectTimeSource(new CreativeTimeSource(new long[] { 1L, 2L, 3L, + 4L, 5L, 6L, 7L, 7L, 8L, 9L, 10L})); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + t.sampleFill(0, true); + t.sampleFill(1, true); + t.sampleFillReturn(1, true, null); + t.sampleFillReturn(0, true, null); + SearcherTimer[] searchers = t.searcherTracking(); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[0].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.FILL)); + assertNull(searchers[1].getReturning(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[0].getReturning(Activity.FILL)); + } + + + private void runSomeTraffic(TimeTracker t) { + t.injectTimeSource(new CreativeTimeSource(new long[] { + 1L, 2L, 3L, + // checkpoint 1 + 4L, 5L, + // checkpoint 2 + 6L, 7L, 8L, 9L, + // checkpoint 3 + 10L, 11L, 12L, 13L, + // checkpoint 4 + 14L, 15L, 16L, 17L, + // checkpoint 5 + 18L + })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + // checkpoint 1 + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + // checkpoint 2 + t.sampleFill(1, true); + t.sampleFill(2, true); + t.sampleFillReturn(2, true, null); + t.sampleFillReturn(1, true, null); + // checkpoint 3 + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + // checkpoint 4 + t.sampleFill(1, true); + t.sampleFill(2, true); + t.sampleFillReturn(2, true, null); + t.sampleFillReturn(1, true, null); + // checkpoint 5 + t.sampleSearchReturn(0, true, null); + } + + public void testMixedActivity() { + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + runSomeTraffic(t); + + SearcherTimer[] searchers = t.searcherTracking(); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.SEARCH)); + assertNull(searchers[0].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(2L), searchers[0].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[0].getReturning(Activity.FILL)); + + assertEquals(Long.valueOf(2L), searchers[1].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[1].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(2L), searchers[1].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[1].getReturning(Activity.FILL)); + + assertEquals(Long.valueOf(2L), searchers[2].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[2].getInvoking(Activity.FILL)); + assertNull(searchers[2].getReturning(Activity.SEARCH)); + assertNull(searchers[2].getReturning(Activity.FILL)); + } + + public void testReportGeneration() { + TimeTracker t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + runSomeTraffic(t); + + ElapsedTime elapsed = new ElapsedTime(); + elapsed.add(t); + t = new TimeTracker(new Chain( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + runSomeTraffic(t); + elapsed.add(t); + assertEquals(true, elapsed.hasDetailedData()); + assertEquals("Time use per searcher:" + + " first(QueryProcessing(SEARCH: 2 ms), ResultProcessing(SEARCH: 4 ms, FILL: 4 ms)),\n" + + " second(QueryProcessing(SEARCH: 4 ms, FILL: 4 ms), ResultProcessing(SEARCH: 4 ms, FILL: 4 ms)),\n" + + " third(QueryProcessing(SEARCH: 4 ms, FILL: 4 ms), ResultProcessing()).", + elapsed.detailedReport()); + } + + public static void doInjectTimeSource(TimeTracker t, TimeTracker.TimeSource s) { + t.injectTimeSource(s); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java new file mode 100644 index 00000000000..fba46e1dfbe --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.statistics; + +import static org.junit.Assert.*; + +import java.util.Deque; +import java.util.List; + +import com.yahoo.statistics.Statistics; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.concurrent.LocalInstance; +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.PeakQpsSearcher.QueryRatePerSecond; + +/** + * Check peak QPS aggregation has a chance of working. + * + * @author Steinar Knutsen + */ +public class PeakQpsTestCase { + + static class Producer implements Runnable { + private final ThreadLocalDirectory, Long> rates; + + Producer(ThreadLocalDirectory, Long> rates) { + this.rates = rates; + } + + @Override + public void run() { + LocalInstance, Long> rate = rates.getLocalInstance(); + rates.update(1L, rate); + rates.update(2L, rate); + rates.update(2L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(4L, rate); + rates.update(4L, rate); + rates.update(4L, rate); + rates.update(4L, rate); + } + } + + static class LaterProducer implements Runnable { + private final ThreadLocalDirectory, Long> rates; + + LaterProducer(ThreadLocalDirectory, Long> rates) { + this.rates = rates; + } + + @Override + public void run() { + LocalInstance, Long> rate = rates.getLocalInstance(); + rates.update(2L, rate); + rates.update(2L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(5L, rate); + rates.update(5L, rate); + rates.update(6L, rate); + rates.update(7L, rate); + } + } + + @Test + public void checkBasicDataAggregation() { + ThreadLocalDirectory, Long> directory = PeakQpsSearcher.createDirectory(); + final int threadCount = 20; + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; ++i) { + Producer p = new Producer(directory); + threads[i] = new Thread(p); + threads[i].start(); + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + List> measurements = directory.fetch(); + List results = PeakQpsSearcher.merge(measurements); + assertTrue(results.get(0).when == 1L); + assertTrue(results.get(0).howMany == threadCount); + assertTrue(results.get(1).when == 2L); + assertTrue(results.get(1).howMany == threadCount * 2); + assertTrue(results.get(2).when == 3L); + assertTrue(results.get(2).howMany == threadCount * 3); + assertTrue(results.get(3).when == 4L); + assertTrue(results.get(3).howMany == threadCount * 4); + } + + @Test + public void checkMixedDataAggregation() { + ThreadLocalDirectory, Long> directory = PeakQpsSearcher.createDirectory(); + final int firstThreads = 20; + final int secondThreads = 20; + final int threadCount = firstThreads + secondThreads; + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; ++i) { + if (i < firstThreads) { + Producer p = new Producer(directory); + threads[i] = new Thread(p); + } else { + LaterProducer p = new LaterProducer(directory); + threads[i] = new Thread(p); + } + threads[i].start(); + + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + List> measurements = directory.fetch(); + List results = PeakQpsSearcher.merge(measurements); + assertTrue(results.size() == 7); + assertTrue(results.get(0).when == 1L); + assertTrue(results.get(0).howMany == firstThreads); + assertTrue(results.get(1).when == 2L); + assertTrue(results.get(1).howMany == threadCount * 2); + assertTrue(results.get(2).when == 3L); + assertTrue(results.get(2).howMany == threadCount * 3); + assertTrue(results.get(3).when == 4L); + assertTrue(results.get(3).howMany == firstThreads * 4); + assertTrue(results.get(4).when == 5L); + assertTrue(results.get(4).howMany == secondThreads * 2); + assertTrue(results.get(5).when == 6L); + assertTrue(results.get(5).howMany == secondThreads); + assertTrue(results.get(6).when == 7L); + assertTrue(results.get(6).howMany == secondThreads); + } + + @Test + public void checkSearch() { + MeasureQpsConfig config = new MeasureQpsConfig( + new MeasureQpsConfig.Builder().outputmethod( + MeasureQpsConfig.Outputmethod.METAHIT).queryproperty( + "qpsprobe")); + Searcher s = new PeakQpsSearcher(config, Statistics.nullImplementation); + Chain c = new Chain<>(s); + Execution e = new Execution(c, Execution.Context.createContextStub()); + e.search(new Query("/?query=a")); + new Execution(c, Execution.Context.createContextStub()); + Result r = e.search(new Query("/?query=a&qpsprobe=true")); + final Hit hit = r.hits().get(0); + assertTrue(hit instanceof PeakQpsSearcher.QpsHit); + assertNotNull(hit.fields().get(PeakQpsSearcher.QpsHit.MEAN_QPS)); + assertNotNull(hit.fields().get(PeakQpsSearcher.QpsHit.PEAK_QPS)); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java new file mode 100644 index 00000000000..2f086dbe5a8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.statistics; + +import junit.framework.TestCase; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.TimingSearcher.Parameters; +import com.yahoo.statistics.Statistics; +import com.yahoo.statistics.Value; + +public class TimingSearcherTestCase extends TestCase { + public static class MockValue extends Value { + public int putCount = 0; + + public MockValue() { + super("mock", Statistics.nullImplementation, new Value.Parameters()); + } + + @Override + public void put(double x) { + putCount += 1; + } + } + + public void testMeasurementSearchPath() { + Parameters p = new Parameters("timingtest", TimeTracker.Activity.SEARCH); + TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); + MockValue v = new MockValue(); + ts.setMeasurements(v); + Execution exec = new Execution(ts, Execution.Context.createContextStub()); + Result r = exec.search(new Query("/?query=a")); + Hit f = new Hit("blblbl"); + f.setFillable(); + r.hits().add(f); + exec.fill(r, "whatever"); + exec.fill(r, "lalala"); + exec.ping(new Ping()); + exec.ping(new Ping()); + exec.ping(new Ping()); + assertEquals(1, v.putCount); + } + + public void testMeasurementFillPath() { + Parameters p = new Parameters("timingtest", TimeTracker.Activity.FILL); + TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); + MockValue v = new MockValue(); + ts.setMeasurements(v); + Execution exec = new Execution(ts, Execution.Context.createContextStub()); + Result r = exec.search(new Query("/?query=a")); + Hit f = new Hit("blblbl"); + f.setFillable(); + r.hits().add(f); + exec.fill(r, "whatever"); + exec.fill(r, "lalala"); + exec.ping(new Ping()); + exec.ping(new Ping()); + exec.ping(new Ping()); + assertEquals(2, v.putCount); + } + + public void testMeasurementPingPath() { + Parameters p = new Parameters("timingtest", TimeTracker.Activity.PING); + TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); + MockValue v = new MockValue(); + ts.setMeasurements(v); + Execution exec = new Execution(ts, Execution.Context.createContextStub()); + Result r = exec.search(new Query("/?query=a")); + Hit f = new Hit("blblbl"); + f.setFillable(); + r.hits().add(f); + exec.fill(r, "whatever"); + exec.fill(r, "lalala"); + exec.ping(new Ping()); + exec.ping(new Ping()); + exec.ping(new Ping()); + assertEquals(3, v.putCount); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/test/.gitignore b/container-search/src/test/java/com/yahoo/search/statistics/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java b/container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java new file mode 100644 index 00000000000..bab7c0be548 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.search.Query; + +/** + * Tests the speed of accessing the query + * + * @author Jon Bratseth + */ +public class QueryBenchmark { + + public void run() { + int result=0; + + // Warm-up + out("Warming up..."); + for (int i=0; i<10*1000; i++) + result+=createAndAccessQuery(i); + + long startTime=System.currentTimeMillis(); + out("Running..."); + for (int i=0; i<100*1000; i++) + result+=createAndAccessQuery(i); + out("Ignore this: " + result); // Make sure we are not fooled by optimization by creating an observable result + long endTime=System.currentTimeMillis(); + out("Creating and accessing a query 100.000 times took " + (endTime-startTime) + " ms"); + } + + private final int createAndAccessQuery(int i) { + // 8 sets, 8 gets + + Query query=new Query("?query=test&hits=10&presentation.bolding=true&model.type=all"); + query.properties().set("model.defaultIndex","title"); + query.properties().set("string1","value1:" + i); + query.properties().set("string2","value2:" + i); + query.properties().set("string3","value3:" + i); + int result=((String)query.properties().get("string1")).length(); + result+=((String)query.properties().get("string2")).length(); + result+=((String)query.properties().get("string3")).length(); + result+=((String)query.properties().get("model.defaultIndex")).length(); + + Query clone=query.clone(); + result+=((String)query.properties().get("string1")).length(); + result+=((String)query.properties().get("string2")).length(); + result+=((String)query.properties().get("string3")).length(); + result+=((String)clone.properties().get("model.defaultIndex")).length(); + return result; + } + + private void out(String string) { + System.out.println(string); + } + + public static void main(String[] args) { + new QueryBenchmark().run(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java new file mode 100644 index 00000000000..a9690fd1983 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -0,0 +1,671 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.Highlight; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +import com.yahoo.yolean.Exceptions; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.fail; + +/** + * @author Arne Bergene Fossaa + */ +public class QueryTestCase { + + @Test + public void testSimpleFunctionality() { + Query q = new Query(QueryTestCase.httpEncode("/sdfsd.html?query=this is a simple query&aParameter")); + assertEquals("this is a simple query", q.getModel().getQueryString()); + assertNotNull(q.getModel().getQueryTree()); + assertNull(q.getModel().getDefaultIndex()); + assertEquals("", q.properties().get("aParameter")); + assertNull(q.properties().get("notSetParameter")); + } + + // TODO: YQL work in progress (jon) + @Ignore + @Test + public void testSimpleProgram() { + Query q = new Query(httpEncode("?program=select * where myfield contains(word)")); + assertEquals("", q.getModel().getQueryTree().toString()); + } + + // TODO: YQL work in progress (jon) + @Ignore + @Test + public void testSimpleProgramParameterAlias() throws UnsupportedEncodingException { + Query q = new Query(httpEncode("/sdfsd.html?yql=select * from source where myfield contains(word);")); + assertEquals("", q.getModel().getQueryTree().toString()); + } + + @Test + public void testClone() { + Query q = new Query(httpEncode("/sdfsd.html?query=this+is+a+simple+query&aParameter")); + q.getPresentation().setHighlight(new Highlight()); + Query p = q.clone(); + assertEquals(q, p); + assertEquals(q.hashCode(), p.hashCode()); + + // Make sure we deep clone all mutable objects + + assertNotSame(q, p); + assertNotSame(q.getRanking(), p.getRanking()); + assertNotSame(q.getRanking().getFeatures(), p.getRanking().getFeatures()); + assertNotSame(q.getRanking().getProperties(), p.getRanking().getProperties()); + assertNotSame(q.getRanking().getMatchPhase(), p.getRanking().getMatchPhase()); + assertNotSame(q.getRanking().getMatchPhase().getDiversity(), p.getRanking().getMatchPhase().getDiversity()); + + assertNotSame(q.getPresentation(), p.getPresentation()); + assertNotSame(q.getPresentation().getHighlight(), p.getPresentation().getHighlight()); + assertNotSame(q.getPresentation().getSummaryFields(), p.getPresentation().getSummaryFields()); + + assertNotSame(q.getModel(), p.getModel()); + assertNotSame(q.getModel().getSources(), p.getModel().getSources()); + assertNotSame(q.getModel().getRestrict(), p.getModel().getRestrict()); + assertNotSame(q.getModel().getQueryTree(), p.getModel().getQueryTree()); + } + + private boolean isA(String s) { + return (s.equals("a")); + } + + private void printIt(List l) { + System.out.println(l); + } + + @Test + public void testCloneWithConnectivity() { + List l = new ArrayList(); + l.add("a"); + l.add("b"); + l.add("c"); + l.add("a"); + printIt(l.stream().filter(i -> isA(i)).collect(Collectors.toList())); + printIt(l.stream().filter(i -> ! isA(i)).collect(Collectors.toList())); + + Query q = new Query(); + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + WordItem c = new WordItem("c"); + WordItem d = new WordItem("d"); + WordItem e = new WordItem("e"); + WordItem f = new WordItem("f"); + WordItem g = new WordItem("g"); + + OrItem or = new OrItem(); + or.addItem(c); + or.addItem(d); + + AndItem and1 = new AndItem(); + and1.addItem(a); + and1.addItem(b); + and1.addItem(or); + and1.addItem(e); + + AndItem and2 = new AndItem(); + and2.addItem(f); + and2.addItem(g); + + RankItem rank = new RankItem(); + rank.addItem(and1); + rank.addItem(and2); + + a.setConnectivity(b, 0.1); + b.setConnectivity(c, 0.2); + c.setConnectivity(d, 0.3); + d.setConnectivity(e, 0.4); + e.setConnectivity(f, 0.5); + f.setConnectivity(g, 0.6); + + q.getModel().getQueryTree().setRoot(rank); + Query qClone = q.clone(); + assertEquals(q, qClone); + + RankItem rankClone = (RankItem)qClone.getModel().getQueryTree().getRoot(); + AndItem and1Clone = (AndItem)rankClone.getItem(0); + AndItem and2Clone = (AndItem)rankClone.getItem(1); + OrItem orClone = (OrItem)and1Clone.getItem(2); + + WordItem aClone = (WordItem)and1Clone.getItem(0); + WordItem bClone = (WordItem)and1Clone.getItem(1); + WordItem cClone = (WordItem)orClone.getItem(0); + WordItem dClone = (WordItem)orClone.getItem(1); + WordItem eClone = (WordItem)and1Clone.getItem(3); + WordItem fClone = (WordItem)and2Clone.getItem(0); + WordItem gClone = (WordItem)and2Clone.getItem(1); + + assertTrue(rankClone != rank); + assertTrue(and1Clone != and1); + assertTrue(and2Clone != and2); + assertTrue(orClone != or); + + assertTrue(aClone != a); + assertTrue(bClone != b); + assertTrue(cClone != c); + assertTrue(dClone != d); + assertTrue(eClone != e); + assertTrue(fClone != f); + assertTrue(gClone != g); + + assertTrue(aClone.getConnectedItem() == bClone); + assertTrue(bClone.getConnectedItem() == cClone); + assertTrue(cClone.getConnectedItem() == dClone); + assertTrue(dClone.getConnectedItem() == eClone); + assertTrue(eClone.getConnectedItem() == fClone); + assertTrue(fClone.getConnectedItem() == gClone); + + double delta = 0.0000001; + assertEquals(0.1, aClone.getConnectivity(), delta); + assertEquals(0.2, bClone.getConnectivity(), delta); + assertEquals(0.3, cClone.getConnectivity(), delta); + assertEquals(0.4, dClone.getConnectivity(), delta); + assertEquals(0.5, eClone.getConnectivity(), delta); + assertEquals(0.6, fClone.getConnectivity(), delta); + } + + @Test + public void test_that_cloning_preserves_timeout() { + Query original = new Query(); + original.setTimeout(9876l); + + Query clone = original.clone(); + assertThat(clone.getTimeout(), is(9876l)); + } + + @Test + public void testTimeout() { + // yes, this test depends on numbers which have exact IEEE representations + Query q = new Query(httpEncode("/search?timeout=500")); + assertEquals(500000L, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=500 ms")); + assertEquals(500, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=500.0ms")); + assertEquals(500, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=500.0s")); + assertEquals(500000, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=5ks")); + assertEquals(5000000, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=5000.0 \u00B5s")); + assertEquals(5, q.getTimeout()); + assertEquals(0, q.errors().size()); + + // seconds is unit when unknown unit + q = new Query(httpEncode("/search?timeout=42 yrs")); + assertEquals(42000, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q=new Query(); + q.setTimeout(53L); + assertEquals(53L, q.properties().get("timeout")); + assertEquals(53L, q.getTimeout()); + + // This is the unfortunate consequence of this legacy: + q=new Query(); + q.properties().set("timeout", 53L); + assertEquals(53L * 1000, q.properties().get("timeout")); + assertEquals(53L * 1000, q.getTimeout()); + } + + @Test + public void testUnparseableTimeout() { + try { + new Query(httpEncode("/search?timeout=nalle")); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); + } + } + + @Test + public void testTimeoutInRequestOverridesQueryProfile() { + QueryProfile profile = new QueryProfile("test"); + profile.set("timeout", 318, (QueryProfileRegistry)null); + Query q = new Query(QueryTestCase.httpEncode("/search?timeout=500"), profile.compile(null)); + assertEquals(500000L, q.getTimeout()); + } + + @Test + public void testNotEqual() { + Query q = new Query("/?query=something+test&nocache"); + Query p = new Query("/?query=something+test"); + assertEquals(q,p); + assertEquals(q.hashCode(),p.hashCode()); + Query r = new Query("?query=something+test&hits=5"); + assertNotSame(q,r); + assertNotSame(q.hashCode(),r.hashCode()); + } + + @Test + public void testEqual() { + assertEquals(new Query("?query=12").hashCode(),new Query("?query=12").hashCode()); + assertEquals(new Query("?query=12"),new Query("?query=12")); + } + + @Test + public void testUtf8Decoding() { + Query q = new Query("/?query=beyonc%C3%A9"); + q.getModel().getQueryTree().toString(); + assertEquals("beyonc\u00e9", q.getModel().getQueryTree().toString()); + } + + @Test + public void testDefaultIndex() { + Query q = new Query("?query=hi%20hello%20keyword:kanoo%20" + + "default:munkz%20%22phrases+too%22&default-index=def"); + assertEquals("AND def:hi def:hello keyword:kanoo default:munkz def:\"phrases too\"", + q.getModel().getQueryTree().toString()); + } + + @Test + public void testHashCode() { + Query p = new Query("?query=foo&type=any"); + Query q = new Query("?query=foo&type=all"); + assertTrue(p.hashCode() != q.hashCode()); + } + + @Test + public void testSimpleQueryParsing () { + Query q = new Query("/search?query=foobar&offset=10&hits=20"); + assertEquals("foobar",q.getModel().getQueryTree().toString()); + assertEquals(10,q.getOffset()); + assertEquals(20,q.getHits()); + } + + /** Test that GET parameter names are case in-sensitive */ + @Test + public void testGETParametersCase() { + Query q = new Query("?QUERY=testing&hits=10&oFfSeT=10"); + assertEquals("testing", q.getModel().getQueryString()); + assertEquals(10, q.getHits()); + assertEquals(10, q.getOffset()); + } + + /** Test that we get the last value if a parameter is assigned multiple times */ + @Test + public void testRepeatedParameter() { + Query q = new Query("?query=test&hits=5&hits=10"); + assertEquals(10, q.getHits()); + } + + @Test + public void testNoCache() { + Query q = new Query("search?query=foobar&nocache"); + assertTrue(q.getNoCache()); + } + + @Test + public void testSessionCache() { + Query q = new Query("search?query=foobar&groupingSessionCache"); + assertTrue(q.getGroupingSessionCache()); + q = new Query("search?query=foobar"); + assertFalse(q.getGroupingSessionCache()); + } + + public class TestClass { + private int testInt = 0; + public int getTestInt() { + return testInt; + } + + public void setTestInt(int testInt) { + this.testInt = testInt; + } + + public void setTestInt(String testInt) { + this.testInt = Integer.parseInt(testInt); + } + } + + @Test + public void testSetting() { + Query q = new Query(); + q.properties().set("test", "test"); + assertEquals(q.properties().get("test"), "test"); + + TestClass tc = new TestClass(); + q.properties().set("test", tc); + assertEquals(q.properties().get("test"), tc); + q.properties().set("test.testInt", 1); + assertEquals(q.properties().get("test.testInt"), 1); + } + + @Test + public void testAlias() { + Query q = new Query("search?query=testing&language=en"); + assertEquals(q.getModel().getLanguage(), q.properties().get("model.language")); + } + + @Test + public void testTracing() { + Query q = new Query("?query=foo&traceLevel=2"); + assertEquals(2, q.getTraceLevel()); + q.trace(true, 1, "trace1"); + q.trace(false,2, "trace2"); + q.trace(true, 3, "Ignored"); + q.trace(true, 2, "trace3-1", ", ", "trace3-2"); + q.trace(false,1, "trace4-1", ", ", "trace4-2"); + q.trace(false,3, "Ignored-1", "Ignored-2"); + Set traces = new HashSet<>(); + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) + traces.add(trace); + // for (String s : traces) System.out.println(s); + assertTrue(traces.contains("trace1: [select * from sources * where default contains \"foo\";]")); + assertTrue(traces.contains("trace2")); + assertTrue(traces.contains("trace3-1, trace3-2: [select * from sources * where default contains \"foo\";]")); + assertTrue(traces.contains("trace4-1, trace4-2")); + } + + @Test + public void testNullTracing() { + Query q = new Query("?query=foo&traceLevel=2"); + assertEquals(2, q.getTraceLevel()); + q.trace(false,2, "trace2 ", null); + Set traces = new HashSet<>(); + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) { + traces.add(trace); + } + assertTrue(traces.contains("trace2 null")); + } + + @Test + public void testQueryPropertyResolveTracing() { + QueryProfile testProfile=new QueryProfile("test"); + testProfile.setOverridable("u", false, null); + testProfile.set("d","e", null); + testProfile.set("u","11", null); + testProfile.set("foo.bar", "wiz", null); + Query q = new Query(QueryTestCase.httpEncode("?query=a:>5&a=b&traceLevel=5&sources=a,b&u=12&foo.bar2=wiz2&c.d=foo&queryProfile=test"),testProfile.compile(null)); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + for (String line : traceLines) + System.out.println(line); + assertTrue(contains("query=a:>5 (value from request)",traceLines)); + assertTrue(contains("traceLevel=5 (value from request)",traceLines)); + assertTrue(contains("a=b (value from request)",traceLines)); + assertTrue(contains("sources=[a, b] (value from request)",traceLines)); + assertTrue(contains("d=e (value from query profile)",traceLines)); + assertTrue(contains("u=11 (value from query profile - unoverridable, ignoring request value)",traceLines)); + } + + @Test + public void testNonleafInRequestDoesNotOverrideProfile() { + QueryProfile testProfile=new QueryProfile("test"); + testProfile.set("a.b", "foo", (QueryProfileRegistry)null); + testProfile.freeze(); + { + Query q = new Query("?", testProfile.compile(null)); + assertEquals("foo", q.properties().get("a.b")); + } + + { + Query q = new Query("?a=bar", testProfile.compile(null)); + assertEquals("bar", q.properties().get("a")); + assertEquals("foo", q.properties().get("a.b")); + } + } + + @Test + public void testQueryPropertyResolveTracing2() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.freeze(); + Query q = new Query(QueryTestCase.httpEncode("?query=dvd&a.b=foo&tracelevel=9"), defaultProfile.compile(null)); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + assertTrue(contains("query=dvd (value from request)",traceLines)); + assertTrue(contains("a.b=foo (value from request)",traceLines)); + } + + @Test + public void testQueryPropertyListingAndTrace() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.setDimensions(new String[]{"x"}); + defaultProfile.set("a.b","a.b-x1-value",new String[] {"x1"}, null); + defaultProfile.set("a.b", "a.b-x2-value", new String[]{"x2"}, null); + defaultProfile.freeze(); + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b-x1-value",propertyList.get("a.b")); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + assertTrue(contains("a.b=a.b-x1-value (value from query profile)",traceLines)); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties("a"); + assertEquals(1,propertyList.size()); + assertEquals("a.b-x1-value",propertyList.get("b")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x2"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b-x2-value",propertyList.get("a.b")); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + assertTrue(contains("a.b=a.b-x2-value (value from query profile)",traceLines)); + } + } + + @Test + public void testQueryPropertyListingThreeLevel() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.setDimensions(new String[] {"x"}); + defaultProfile.set("a.b.c", "a.b.c-x1-value", new String[]{"x1"}, null); + defaultProfile.set("a.b.c", "a.b.c-x2-value", new String[]{"x2"}, null); + defaultProfile.freeze(); + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b.c-x1-value",propertyList.get("a.b.c")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties("a"); + assertEquals(1,propertyList.size()); + assertEquals("a.b.c-x1-value",propertyList.get("b.c")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties("a.b"); + assertEquals(1,propertyList.size()); + assertEquals("a.b.c-x1-value",propertyList.get("c")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x2"),defaultProfile.compile(null)); + Map propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b.c-x2-value",propertyList.get("a.b.c")); + } + } + + @Test + public void testQueryPropertyReplacement() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.set("model.queryString","myquery", (QueryProfileRegistry)null); + defaultProfile.set("queryUrl","http://provider:80?query=%{model.queryString}", (QueryProfileRegistry)null); + defaultProfile.freeze(); + + Query q1 = new Query(QueryTestCase.httpEncode(""),defaultProfile.compile(null)); + assertEquals("myquery",q1.getModel().getQueryString()); + assertEquals("http://provider:80?query=myquery",q1.properties().get("queryUrl")); + + Query q2 = new Query(QueryTestCase.httpEncode("?model.queryString=foo"),defaultProfile.compile(null)); + assertEquals("foo",q2.getModel().getQueryString()); + assertEquals("http://provider:80?query=foo",q2.properties().get("queryUrl")); + + Query q3 = new Query(QueryTestCase.httpEncode("?query=foo"),defaultProfile.compile(null)); + assertEquals("foo",q3.getModel().getQueryString()); + assertEquals("http://provider:80?query=foo",q3.properties().get("queryUrl")); + + Query q4 = new Query(QueryTestCase.httpEncode("?query=foo"),defaultProfile.compile(null)); + q4.getModel().setQueryString("bar"); + assertEquals("http://provider:80?query=bar",q4.properties().get("queryUrl")); + } + + @Test + public void testNoQueryString() throws IOException { + Query q = new Query(httpEncode("?tracelevel=1")); + Chain chain = new Chain<>(new RandomSearcher()); + new Execution(chain, Execution.Context.createContextStub()).search(q); + assertNotNull(q.getModel().getQueryString()); + } + + @Test + public void testSetCollapseField() { + Query q = new Query(httpEncode("?collapsefield=foo&presentation.format=tiled")); + assertEquals("foo",q.properties().get("collapsefield")); + assertEquals("tiled", q.properties().get("presentation.format")); + assertEquals("tiled", q.getPresentation().getFormat()); + } + + @Test + public void testSetNullProperty() { + QueryProfile profile = new QueryProfile("test"); + profile.set("property","initialValue", (QueryProfileRegistry)null); + Query query = new Query(httpEncode("?query=test"), profile.compile(null)); + assertEquals("initialValue",query.properties().get("property")); + query.properties().set("property",null); + assertNull(query.properties().get("property")); + } + + @Test + public void testSetNullPropertyNoQueryProfile() { + Query query=new Query(); + query.properties().set("a",null); + assertNull(query.properties().get("a")); + } + + @Test + public void testMissingParameter() { + Query q=new Query("?query=foo&hits="); + assertEquals(0, q.errors().size()); + } + + @Test + public void testModelProperties() { + { + Query query=new Query(); + query.properties().set("model.searchPath", "foo"); + assertEquals("Set dynamic get dynamic works","foo",query.properties().get("model.searchPath")); + assertEquals("Set dynamic get static works","foo",query.getModel().getSearchPath()); + } + + { + Query query=new Query(); + query.getModel().setSearchPath("foo"); + assertEquals("Set static get dynamic works","foo",query.properties().get("model.searchPath")); + assertEquals("Set static get static works","foo",query.getModel().getSearchPath()); + } + + { + Query query=new Query(); + query.properties().set("a","bar"); + assertEquals("bar",query.properties().get("a")); + query.properties().set("a.b","baz"); + assertEquals("baz",query.properties().get("a.b")); + } + } + + @Test + public void testPositiveTerms() { + Query q = new Query(QueryTestCase.httpEncode("/?query=-a \"b c\" d e")); + Item i = q.getModel().getQueryTree().getRoot(); + List l = QueryTree.getPositiveTerms(i); + assertEquals(3, l.size()); + } + + protected boolean contains(String lineSubstring,String[] lines) { + for (String line : lines) + if (line.indexOf(lineSubstring)>=0) return true; + return false; + } + + private static class RandomSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result r=new Result(query); + r.hits().add(new Hit("hello")); + return r; + } + } + + /** + * Url encode the given string, except the characters =?&, such that queries with paths and parameters can + * be written as a single string. + */ + public static String httpEncode(String s) { + try { + if (s == null) return null; + String encoded = URLEncoder.encode(s, "utf-8"); + encoded = encoded.replaceAll("%3F", "?"); + encoded = encoded.replaceAll("%3D", "="); + encoded = encoded.replaceAll("%26", "&"); + return encoded; + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java b/container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java new file mode 100644 index 00000000000..5c77dc0215d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.search.Query; + +/** + * @author Jon Bratseth + */ +public class RequestParameterPreservationTestCase extends junit.framework.TestCase { + + public void testPreservation() { + Query query=new Query("?query=test...&offset=15&hits=10"); + query.setWindow(25,13); + assertEquals(25,query.getOffset()); + assertEquals(13,query.getHits()); + assertEquals("15", query.getHttpRequest().getProperty("offset")); + assertEquals("10", query.getHttpRequest().getProperty("hits")); + assertEquals("test...",query.getHttpRequest().getProperty("query")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java b/container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java new file mode 100644 index 00000000000..450da35b7a4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * Tests the speed of accessing hits in the query by id + * + * @author Jon Bratseth + */ +public class ResultBenchmark { + + public void run() { + int foundCount=0; + + // Warm-up + out("Warming up..."); + Result result=createResult(); + for (int i=0; i<10*1000; i++) + foundCount+=accessResultFiveTimes(result); + foundCount=0; + + long startTime=System.currentTimeMillis(); + out("Running..."); + for (int i=0; i<200*1000; i++) + foundCount+=accessResultFiveTimes(result); + out("Successfully looked up " + foundCount + " hits"); + long endTime=System.currentTimeMillis(); + out("Accessing a result 1.000.000 times took " + (endTime-startTime) + " ms"); + } + + private final Result createResult() { + // 8 sets, 8 gets + Result result=new Result(new Query("?query=test&hits=10&presentation.bolding=true&model.type=all")); + addHits(5,"firstTopLevel",result.hits()); + result.hits().add(addHits(10, "group1hit", new HitGroup())); + addHits(5, "secondTopLevel", result.hits()); + result.hits().add(addHits(10, "group2hit", new HitGroup())); + result.hits().add(addHits(10, "group3hit", new HitGroup())); + return result; + } + + private final HitGroup addHits(int count,String idPrefix,HitGroup to) { + for (int i=1; i<=count; i++) + to.add(new Hit(idPrefix + i,1/i)); + return to; + } + + private final int accessResultFiveTimes(Result result) { + // 8 sets, 8 gets + int foundCount=0; + if (null!=result.hits().get("firstTopLevel1")) + foundCount++; + if (null!=result.hits().get("secondTopLevel3")) + foundCount++; + if (null!=result.hits().get("group3hit5")) + foundCount++; + if (null!=result.hits().get("group1hit2")) + foundCount++; + if (null!=result.hits().get("group2hit4")) + foundCount++; + return foundCount; + } + + private void out(String string) { + System.out.println(string); + } + + public static void main(String[] args) { + new ResultBenchmark().run(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java new file mode 100644 index 00000000000..ac6ceeb5467 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; + +/** + * Smoketest that we remove fields in a sane manner. + * + * @author Steinar Knutsen + */ +public class FieldFilterTestCase { + private static final String FIELD_C = "c"; + private static final String FIELD_B = "b"; + private static final String FIELD_A = "a"; + private Chain searchChain; + private Execution.Context context; + private Execution execution; + + @Before + public void setUp() throws Exception { + Query query = new Query("?query=test"); + + Result result = new Result(query); + Hit hit = createHit("lastHit", .1d, FIELD_A, FIELD_B, FIELD_C); + result.hits().add(hit); + + DocumentSourceSearcher mockBackend = new DocumentSourceSearcher(); + mockBackend.addResult(query, result); + + searchChain = new Chain(new FieldFilter(), + mockBackend); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + + } + + private Hit createHit(String id, double relevancy, String... fieldNames) { + Hit h = new Hit(id, relevancy); + h.setFillable(); + int i = 0; + for (String field : fieldNames) { + h.setField(field, ++i); + } + return h; + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public final void testBasic() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD_B); + Result result = execution.search(query); + execution.fill(result); + assertEquals(1, result.getConcreteHitCount()); + assertFalse(result.hits().get(0).fieldKeys().contains(FIELD_A)); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_B)); + assertFalse(result.hits().get(0).fieldKeys().contains(FIELD_C)); + } + + @Test + public final void testNoFiltering() { + final Query query = new Query("?query=test"); + Result result = execution.search(query); + execution.fill(result); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_A)); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_B)); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_C)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java new file mode 100644 index 00000000000..7834539db72 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -0,0 +1,297 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + +import com.yahoo.search.grouping.GroupingRequest; + +import org.apache.http.client.utils.URIBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.Sorting.AttributeSorter; +import com.yahoo.search.query.Sorting.FieldOrder; +import com.yahoo.search.query.Sorting.LowerCaseSorter; +import com.yahoo.search.query.Sorting.Order; +import com.yahoo.search.query.Sorting.UcaSorter; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.List; + +/** + * Smoke test for first generation YQL+ integration. + */ +public class MinimalQueryInserterTestCase { + private Chain searchChain; + private Execution.Context context; + private Execution execution; + + @Before + public void setUp() throws Exception { + searchChain = new Chain(new MinimalQueryInserter()); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public void requireThatGroupingStepsAreAttachedToQuery() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox';"); + Query query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[]", query); + + assertEquals(1, query.getPresentation().getSummaryFields().size()); + assertEquals("foo", query.getPresentation().getSummaryFields().toArray(new String[1])[0]); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[]all(group(a) each(output(count())))]", query); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count()))) " + + "| all(group(b) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[]all(group(a) each(output(count())))," + + " []all(group(b) each(output(count())))]", query); + } + + @Test + public void requireThatGroupingContinuationsAreAttachedToQuery() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox';"); + Query query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[]", query); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(a) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", query); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(a) each(output(count()))) " + + "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + + "all(group(b) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))," + + " [BCBBBBBDBF, BCBJBPCBJCCJ]all(group(b) each(output(count())))]", query); + } + + @Test + @Ignore + // TODO: YQL work in progress (jon) + public final void testTmp() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + //execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + } + + @Test + public final void testSearch() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + } + + @Test + public final void testUserQueryFailsWithoutArgument() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + } + + @Test + public final void testSearchFromAllSourcesWithUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(0, query.getModel().getSources().size()); + } + + @Test + public final void testSearchFromAllSourcesWithoutUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(0, query.getModel().getSources().size()); + } + + @Test + public final void testSearchFromSomeSourcesWithoutUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(2, query.getModel().getSources().size()); + assertTrue(query.getModel().getSources().contains("sourceA")); + assertTrue(query.getModel().getSources().contains("sourceB")); + } + + @Test + public final void testSearchFromSomeSourcesWithUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(3, query.getModel().getSources().size()); + assertTrue(query.getModel().getSources().contains("sourceA")); + assertTrue(query.getModel().getSources().contains("sourceB")); + assertTrue(query.getModel().getSources().contains("abc")); + } + + @Test + public final void testSearchFromSomeSourcesWithOverlappingUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc,sourceA&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(3, query.getModel().getSources().size()); + assertTrue(query.getModel().getSources().contains("sourceA")); + assertTrue(query.getModel().getSources().contains("sourceB")); + assertTrue(query.getModel().getSources().contains("abc")); + } + + @Test + public final void testLimitAndOffset() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2031offset%207%3B"); + execution.search(query); + assertEquals(7, query.getOffset()); + assertEquals(24, query.getHits()); + assertEquals("select * from sources * where title contains \"madonna\" limit 31 offset 7;", + query.yqlRepresentation()); + } + + @Test + public final void testMaxOffset() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040031offset%2040000%3B"); + Result r = execution.search(query); + assertEquals(1, r.hits().getErrorHit().errors().size()); + ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); + assertEquals(com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code, e.getCode()); + assertTrue(e.getDetailedMessage().indexOf("max offset") >= 0); + } + + @Test + public final void testMaxLimit() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040000offset%207%3B"); + Result r = execution.search(query); + assertEquals(1, r.hits().getErrorHit().errors().size()); + ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); + assertEquals(com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code, e.getCode()); + assertTrue(e.getDetailedMessage().indexOf("max hits") >= 0); + } + + @Test + public final void testTimeout() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051%3B"); + execution.search(query); + assertEquals(51L, query.getTimeout()); + assertEquals("select * from sources * where title contains \"madonna\" timeout 51;", query.yqlRepresentation()); + } + + @Test + public final void testOrdering() { + { + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + assertEquals(2, query.getRanking().getSorting().fieldOrders() + .size()); + assertEquals("something", query.getRanking().getSorting() + .fieldOrders().get(0).getFieldName()); + assertEquals(Order.ASCENDING, query.getRanking().getSorting() + .fieldOrders().get(0).getSortOrder()); + assertEquals("shoesize", query.getRanking().getSorting() + .fieldOrders().get(1).getFieldName()); + assertEquals(Order.DESCENDING, query.getRanking().getSorting() + .fieldOrders().get(1).getSortOrder()); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", query.yqlRepresentation()); + } + { + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + assertEquals("other", query.getRanking().getSorting().fieldOrders() + .get(0).getFieldName()); + assertEquals(Order.ASCENDING, query.getRanking().getSorting() + .fieldOrders().get(0).getSortOrder()); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by other limit 300 timeout 3;", query.yqlRepresentation()); + } + { + String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + { + final FieldOrder fieldOrder = query.getRanking().getSorting() + .fieldOrders().get(0); + assertEquals("other", fieldOrder.getFieldName()); + assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(UcaSorter.class, sorter.getClass()); + final UcaSorter uca = (UcaSorter) sorter; + assertEquals("en_US", uca.getLocale()); + assertEquals(UcaSorter.Strength.IDENTICAL, uca.getStrength()); + } + { + final FieldOrder fieldOrder = query.getRanking().getSorting() + .fieldOrders().get(1); + assertEquals("something", fieldOrder.getFieldName()); + assertEquals(Order.ASCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(LowerCaseSorter.class, sorter.getClass()); + } + assertEquals("select foo from bar where title contains \"madonna\" order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc, [{\"function\": \"lowercase\"}]something limit 300 timeout 3;", + query.yqlRepresentation()); + } + } + + @Test + public final void testStringReprBasicSanity() { + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + assertEquals("select ignoredfield from ignoredsource where [{\"segmenter\": {\"version\": \"1.9\", \"backend\": \"YqlUnitTest\"}}](title contains \"madonna\") order by something, shoesize desc limit 300 timeout 3;", + query.yqlRepresentation(new Tuple2<>("YqlUnitTest", new Version(1, 9)), true)); + } + + + private static void assertGrouping(String expected, Query query) { + List actual = new ArrayList<>(); + for (GroupingRequest request : GroupingRequest.getRequests(query)) { + actual.add(request.continuations().toString() + request.getRootOperation()); + } + assertEquals(expected, actual.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java new file mode 100644 index 00000000000..8c4d8e0fe84 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.assertEquals; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; + +/** + * Check rules for resegmenting words in YQL+ when segmenter is deemed + * incompatible. The class under testing is {@link YqlParser}. + * + * @author Steinar Knutsen + */ +public class ResegmentingTestCase { + private YqlParser parser; + + @Before + public void setUp() throws Exception { + ParserEnvironment env = new ParserEnvironment(); + parser = new YqlParser(env); + } + + @After + public void tearDown() throws Exception { + parser = null; + } + + @Test + public final void testWord() { + assertEquals( + "title:'a b'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}] (title contains \"a b\");")) + .toString()); + } + + @Test + public final void testPhraseSegment() { + assertEquals( + "title:'c d'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseInEquiv() { + assertEquals( + "EQUIV title:a title:'c d'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains" + + " equiv(\"a\"," + + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]\"b\")" + + ")" + + ");")) + .toString()); + } + + @Test + public final void testPhraseSegmentToAndSegment() { + assertEquals( + "SAND title:c title:d", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"andSegmenting\": true}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseSegmentInPhrase() { + assertEquals( + "title:\"a 'c d'\"", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains phrase(\"a\"," + + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]" + + " phrase(\"e\", \"f\"))));")) + .toString()); + } + + @Test + public final void testWordNoImplicitTransforms() { + assertEquals( + "title:a b", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}] (title contains ([{\"implicitTransforms\": false}]\"a b\"));")) + .toString()); + } + + @Test + public final void testPhraseSegmentNoImplicitTransforms() { + assertEquals( + "title:'a b'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"implicitTransforms\": false}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseSegmentToAndSegmentNoImplicitTransforms() { + assertEquals( + "SAND title:a title:b", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"andSegmenting\": true, \"implicitTransforms\": false}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseSegmentInPhraseNoImplicitTransforms() { + assertEquals( + "title:\"a 'e f'\"", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains phrase(\"a\"," + + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"implicitTransforms\": false}]" + + " phrase(\"e\", \"f\"))));")) + .toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java new file mode 100644 index 00000000000..0d81970bdce --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -0,0 +1,280 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + +import org.apache.http.client.utils.URIBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; + +/** + * Tests where you really test YqlParser but need the full Query infrastructure. + * + * @author steinar + */ +public class UserInputTestCase { + + private Chain searchChain; + private Execution.Context context; + private Execution execution; + + @Before + public void setUp() throws Exception { + searchChain = new Chain(new MinimalQueryInserter()); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public final void testSimpleUserInput() { + { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"nalle\";", query.yqlRepresentation()); + } + { + URIBuilder builder = searchUri(); + builder.setParameter("nalle", "bamse"); + builder.setParameter("yql", + "select * from sources * where userInput(@nalle);"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"bamse\";", query.yqlRepresentation()); + } + { + URIBuilder builder = searchUri(); + builder.setParameter("nalle", "bamse"); + builder.setParameter("yql", + "select * from sources * where userInput(nalle);"); + Query query = new Query(builder.toString()); + Result r = execution.search(query); + assertNotNull(r.hits().getError()); + } + } + + @Test + public final void testRawUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"grammar\": \"raw\"}]userInput(\"nal le\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); + } + + @Test + public final void testSegmentedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"nal le\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains ([{\"origin\": {\"original\": \"nal le\", \"offset\": 0, \"length\": 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); + } + + @Test + public final void testSegmentedNoiseUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"^^^^^^^^\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); + } + + @Test + public final void testCustomDefaultIndexUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"defaultIndex\": \"glompf\"}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where glompf contains \"nalle\";", query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputStemming() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"stem\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"stem\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputUnrankedTerms() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"ranked\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"ranked\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputFiltersTerms() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"filter\": true}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"filter\": true}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputCaseNormalization() { + URIBuilder builder = searchUri(); + builder.setParameter( + "yql", + "select * from sources * where [{\"normalizeCase\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"normalizeCase\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputAccentRemoval() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"accentDrop\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"accentDrop\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputPositionData() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"usePositionData\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"usePositionData\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testQueryPropertiesAsStringArguments() { + URIBuilder builder = searchUri(); + builder.setParameter("nalle", "bamse"); + builder.setParameter("meta", "syntactic"); + builder.setParameter("yql", + "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle);"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (foo contains \"bamse\" AND foo contains phrase(\"bamse\", \"syntactic\", \"bamse\"));", query.yqlRepresentation()); + } + + private Query searchAndAssertNoErrors(URIBuilder builder) { + Query query = new Query(builder.toString()); + Result r = execution.search(query); + assertNull(r.hits().getError()); + return query; + } + + private URIBuilder searchUri() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + return builder; + } + + @Test + public final void testEmptyUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where userInput(\"\");"); + assertQueryFails(builder); + } + + @Test + public final void testEmptyUserInputFromQueryProperty() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", + "select * from sources * where userInput(@foo);"); + assertQueryFails(builder); + } + + @Test + public final void testEmptyQueryProperty() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains @foo);"); + assertQueryFails(builder); + } + + @Test + public final void testEmptyQueryPropertyInsideExpression() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", + "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo);"); + assertQueryFails(builder); + } + + @Test + public final void testCompositeWithoutArguments() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and foo contains phrase();"); + searchAndAssertNoErrors(builder); + builder = searchUri(); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains phrase());"); + assertQueryFails(builder); + } + + @Test + public final void testAnnoyingPlacementOfNonEmpty() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where bar contains \"a\" and foo contains nonEmpty(phrase(\"a\", \"b\"));"); + assertQueryFails(builder); + } + + private void assertQueryFails(URIBuilder builder) { + Result r = execution.search(new Query(builder.toString())); + assertEquals(INVALID_QUERY_PARAMETER.code, r.hits().getError().getCode()); + } + + @Test + public final void testAllowEmptyUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + searchAndAssertNoErrors(builder); + } + + @Test + public final void testAllowEmptyNullFromQueryParsing() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ",,,,,,,,"); + builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + searchAndAssertNoErrors(builder); + } + + @Test + public final void testDisallowEmptyNullFromQueryParsing() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ",,,,,,,,"); + builder.setParameter("yql", "select * from sources * where userInput(@foo);"); + assertQueryFails(builder); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java new file mode 100644 index 00000000000..d9d8eb1b14b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -0,0 +1,404 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + +import com.yahoo.search.Query; +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.AttributeFunction; +import com.yahoo.search.grouping.request.CountAggregator; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.MarkerWordItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; + +import java.util.Arrays; + +public class VespaSerializerTestCase { + + private static final String SELECT = "select ignoredfield from sourceA where "; + private YqlParser parser; + + @Before + public void setUp() throws Exception { + ParserEnvironment env = new ParserEnvironment(); + parser = new YqlParser(env); + } + + @After + public void tearDown() throws Exception { + parser = null; + } + + @Test + public void requireThatGroupingRequestsAreSerialized() { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(new WordItem("foo")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\")", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("a")) + .addChild(new EachOperation().addOutput(new CountAggregator()))); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| all(group(attribute(a)) each(output(count())))", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("b")) + .addChild(new EachOperation().addOutput(new CountAggregator()))); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| all(group(attribute(a)) each(output(count()))) " + + "| all(group(attribute(b)) each(output(count())))", + VespaSerializer.serialize(query)); + } + + @Test + public void requireThatGroupingContinuationsAreSerialized() { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(new WordItem("foo")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\")", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("a")) + .addChild(new EachOperation().addOutput(new CountAggregator())), + Continuation.fromString("BCBCBCBEBG"), + Continuation.fromString("BCBKCBACBKCCK")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(attribute(a)) each(output(count())))", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("b")) + .addChild(new EachOperation().addOutput(new CountAggregator())), + Continuation.fromString("BCBBBBBDBF"), + Continuation.fromString("BCBJBPCBJCCJ")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(attribute(a)) each(output(count()))) " + + "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + + "all(group(attribute(b)) each(output(count())))", + VespaSerializer.serialize(query)); + } + + @Test + public final void testAnd() { + parseAndConfirm("(description contains \"a\" AND title contains \"that\")"); + } + + private void parseAndConfirm(String expected) { + parseAndConfirm(expected, expected); + } + + private void parseAndConfirm(String expected, String toParse) { + QueryTree item = parser + .parse(new Parsable() + .setQuery(SELECT + toParse + ";")); + // System.out.println(item.toString()); + String q = VespaSerializer.serialize(item.getRoot()); + assertEquals(expected, q); + } + + @Test + public final void testAndNot() { + parseAndConfirm("(description contains \"a\") AND !(title contains \"that\")"); + } + + @Test + public final void testEquiv() { + parseAndConfirm("title contains equiv(\"a\", \"b\")"); + } + + @Test + public final void testNear() { + parseAndConfirm("title contains near(\"a\", \"b\")"); + parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); + } + + @Test + public final void testNumbers() { + parseAndConfirm("title = 500"); + parseAndConfirm("title > 500"); + parseAndConfirm("title < 500"); + } + + @Test + public final void testAnnotatedNumbers() { + parseAndConfirm("title = ([{\"filter\": true}]500)"); + parseAndConfirm("title > ([{\"filter\": true}]500)"); + parseAndConfirm("title < ([{\"filter\": true}](-500))"); + parseAndConfirm("title <= ([{\"filter\": true}](-500))", "([{\"filter\": true}](-500)) >= title"); + parseAndConfirm("title <= ([{\"filter\": true}](-500))"); + } + + @Test + public final void testRange() { + parseAndConfirm("range(title, 1, 500)"); + } + + @Test + public final void testAnnotatedRange() { + parseAndConfirm("[{\"filter\": true}]range(title, 1, 500)"); + } + + @Test + public final void testOrderedNear() { + parseAndConfirm("title contains onear(\"a\", \"b\")"); + } + + @Test + public final void testOr() { + parseAndConfirm("(description contains \"a\" OR title contains \"that\")"); + } + + @Test + public final void testDotProduct() { + parseAndConfirm("dotProduct(description, {\"a\": 1, \"b\": 2})"); + } + + @Test + public final void testPredicate() { + parseAndConfirm("predicate(boolean,{\"gender\":\"male\"},{\"age\":25L})"); + parseAndConfirm("predicate(boolean,{\"gender\":\"male\",\"hobby\":\"music\",\"hobby\":\"hiking\"}," + + "{\"age\":25L})", + "predicate(boolean,{\"gender\":\"male\",\"hobby\":[\"music\",\"hiking\"]},{\"age\":25})"); + parseAndConfirm("predicate(boolean,{\"0x3\":{\"gender\":\"male\"},\"0x1\":{\"hobby\":\"music\"},\"0x1\":{\"hobby\":\"hiking\"}},{\"0x80ffffffffffffff\":{\"age\":23L}})", + "predicate(boolean,{\"0x3\":{\"gender\":\"male\"},\"0x1\":{\"hobby\":[\"music\",\"hiking\"]}},{\"0x80ffffffffffffff\":{\"age\":23L}})"); + parseAndConfirm("predicate(boolean,0,0)"); + parseAndConfirm("predicate(boolean,0,0)","predicate(boolean,null,void)"); + parseAndConfirm("predicate(boolean,0,0)","predicate(boolean,{},{})"); + } + + @Test + public final void testPhrase() { + parseAndConfirm("description contains phrase(\"a\", \"b\")"); + } + + @Test + public final void testAnnotatedPhrase() { + parseAndConfirm("description contains ([{\"id\": 1}]phrase(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedNear() { + parseAndConfirm("description contains ([{\"distance\": 37}]near(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedOnear() { + parseAndConfirm("description contains ([{\"distance\": 37}]onear(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedEquiv() { + parseAndConfirm("description contains ([{\"id\": 1}]equiv(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedPhraseSegment() { + PhraseSegmentItem phraseSegment = new PhraseSegmentItem("abc", true, false); + phraseSegment.addItem(new WordItem("a", "indexNamePlaceholder")); + phraseSegment.addItem(new WordItem("b", "indexNamePlaceholder")); + phraseSegment.setIndexName("someIndexName"); + phraseSegment.setLabel("labeled"); + phraseSegment.lock(); + String q = VespaSerializer.serialize(phraseSegment); + assertEquals("someIndexName contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"label\": \"labeled\"}]phrase(\"a\", \"b\"))", q); + } + + @Test + public final void testAnnotatedAndSegment() { + AndSegmentItem andSegment = new AndSegmentItem("abc", true, false); + andSegment.addItem(new WordItem("a", "indexNamePlaceholder")); + andSegment.addItem(new WordItem("b", "indexNamePlaceholder")); + andSegment.setLabel("labeled"); + andSegment.lock(); + String q = VespaSerializer.serialize(andSegment); + assertEquals("indexNamePlaceholder contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"a\", \"b\"))", q); + } + + @Test + public final void testPhraseWithAnnotations() { + parseAndConfirm("description contains phrase(([{\"id\": 15}]\"a\"), \"b\")"); + } + + @Test + public final void testPhraseSegmentInPhrase() { + parseAndConfirm("description contains phrase(\"a\", \"b\", ([{\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}}]phrase(\"c\", \"d\")))"); + } + + @Test + public final void testRank() { + parseAndConfirm("rank(a contains \"A\", b contains \"B\")"); + } + + @Test + public final void testWand() { + parseAndConfirm("wand(description, {\"a\": 1, \"b\": 2})"); + } + + @Test + public final void testWeakAnd() { + parseAndConfirm("weakAnd(a contains \"A\", b contains \"B\")"); + } + + @Test + public final void testAnnotatedWeakAnd() { + parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{\"" + YqlParser.SCORE_THRESHOLD + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10, \"" + YqlParser.SCORE_THRESHOLD + + "\": 20}]weakAnd(a contains \"A\", b contains \"B\"))"); + } + + @Test + public final void testWeightedSet() { + parseAndConfirm("weightedSet(description, {\"a\": 1, \"b\": 2})"); + } + + @Test + public final void testAnnotatedWord() { + parseAndConfirm("description contains ([{\"andSegmenting\": true}]\"a\")"); + parseAndConfirm("description contains ([{\"weight\": 37}]\"a\")"); + parseAndConfirm("description contains ([{\"id\": 37}]\"a\")"); + parseAndConfirm("description contains ([{\"filter\": true}]\"a\")"); + parseAndConfirm("description contains ([{\"ranked\": false}]\"a\")"); + parseAndConfirm("description contains ([{\"significance\": 37.0}]\"a\")"); + parseAndConfirm("description contains ([{\"implicitTransforms\": false}]\"a\")"); + parseAndConfirm("(description contains ([{\"connectivity\": {\"id\": 2, \"weight\": 0.42}, \"id\": 1}]\"a\") AND description contains ([{\"id\": 2}]\"b\"))"); + } + + @Test + public final void testPrefix() { + parseAndConfirm("description contains ([{\"prefix\": true}]\"a\")"); + } + + @Test + public final void testSuffix() { + parseAndConfirm("description contains ([{\"suffix\": true}]\"a\")"); + } + + @Test + public final void testSubstring() { + parseAndConfirm("description contains ([{\"substring\": true}]\"a\")"); + } + + @Test + public final void testExoticItemTypes() { + Item item = MarkerWordItem.createEndOfHost(); + String q = VespaSerializer.serialize(item); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"$\")", q); + } + + @Test + public final void testEmptyIndex() { + Item item = new WordItem("nalle", true); + String q = VespaSerializer.serialize(item); + assertEquals("default contains \"nalle\"", q); + } + + @Test + public final void testLongAndNot() { + NotItem item = new NotItem(); + item.addItem(new WordItem("a")); + item.addItem(new WordItem("b")); + item.addItem(new WordItem("c")); + item.addItem(new WordItem("d")); + String q = VespaSerializer.serialize(item); + assertEquals("(default contains ([{\"implicitTransforms\": false}]\"a\")) AND !(default contains ([{\"implicitTransforms\": false}]\"b\") OR default contains ([{\"implicitTransforms\": false}]\"c\") OR default contains ([{\"implicitTransforms\": false}]\"d\"))", q); + } + + @Test + public final void testPhraseAsOperatorArgument() { + // flattening phrases is a feature, not a bug + parseAndConfirm("description contains phrase(\"a\", \"b\", \"c\")", + "description contains phrase(\"a\", phrase(\"b\", \"c\"))"); + parseAndConfirm("description contains equiv(\"a\", phrase(\"b\", \"c\"))"); + } + + private static void newGroupingRequest(Query query, GroupingOperation grouping, Continuation... continuations) { + GroupingRequest request = GroupingRequest.newInstance(query); + request.setRootOperation(grouping); + request.continuations().addAll(Arrays.asList(continuations)); + } + + @Test + public final void testNumberTypeInt() { + parseAndConfirm("title = 500"); + parseAndConfirm("title > 500"); + parseAndConfirm("title < (-500)"); + parseAndConfirm("title >= (-500)"); + parseAndConfirm("title <= (-500)"); + parseAndConfirm("range(title, 0, 500)"); + } + + @Test + public final void testNumberTypeLong() { + parseAndConfirm("title = 549755813888L"); + parseAndConfirm("title > 549755813888L"); + parseAndConfirm("title < (-549755813888L)"); + parseAndConfirm("title >= (-549755813888L)"); + parseAndConfirm("title <= (-549755813888L)"); + parseAndConfirm("range(title, -549755813888L, 549755813888L)"); + } + + @Test + public final void testNumberTypeFloat() { + parseAndConfirm("title = 500.0"); // silly + parseAndConfirm("title > 500.0"); + parseAndConfirm("title < (-500.0)"); + parseAndConfirm("title >= (-500.0)"); + parseAndConfirm("title <= (-500.0)"); + parseAndConfirm("range(title, 0.0, 500.0)"); + } + + @Test + public final void testAnnotatedLong() { + parseAndConfirm("title >= ([{\"id\": 2014}](-549755813888L))"); + } + + @Test + public final void testHitLimit() { + parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); + parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); + parseAndConfirm("[{\"hitLimit\": 89}]range(title, 1, 500)"); + } + + @Test + public final void testOpenIntervals() { + parseAndConfirm("range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"bounds\": \"open\"}]range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"id\": 500, \"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0)"); + } + + @Test + public final void testRegExp() { + parseAndConfirm("foo matches \"a b\""); + } + + @Test + public final void testWordAlternatives() { + parseAndConfirm("foo contains" + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))"); + } + + @Test + public final void testWordAlternativesInPhrase() { + parseAndConfirm("foo contains phrase(\"forest\"," + + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))" + + ")"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java new file mode 100644 index 00000000000..2ba5a781ab5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; +import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; + +/** + * Test translation of fields and sources in YQL+ to the associated concepts in + * Vespa. + */ +public class YqlFieldAndSourceTestCase { + private static final String FIELD1 = "field1"; + private static final String FIELD2 = "field2"; + private static final String FIELD3 = "field3"; + private static final String THIRD_OPTION = "THIRD_OPTION"; + + private Chain searchChain; + private Execution.Context context; + private Execution execution; + + + @Before + public void setUp() throws Exception { + Query query = new Query("?query=test"); + + Result result = new Result(query); + Hit hit = createHit("lastHit", .1d, FIELD1, FIELD2, FIELD3); + result.hits().add(hit); + + DocumentSourceSearcher mockBackend = new DocumentSourceSearcher(); + mockBackend.addResult(query, result); + + mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, Arrays.asList(FIELD1, FIELD2)); + mockBackend.addSummaryClassByCopy(Execution.ATTRIBUTEPREFETCH, Arrays.asList(FIELD2)); + mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); + + DocumentdbInfoConfig config = new DocumentdbInfoConfig( + new DocumentdbInfoConfig.Builder() + .documentdb(buildDocumentdbArray())); + + searchChain = new Chain(new FieldFiller(config), + mockBackend); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + } + + private Hit createHit(String id, double relevancy, String... fieldNames) { + Hit h = new Hit(id, relevancy); + h.setFillable(); + int i = 0; + for (String field : fieldNames) { + h.setField(field, ++i); + } + return h; + } + + private List buildDocumentdbArray() { + List configArray = new ArrayList( + 1); + configArray.add(new Documentdb.Builder().summaryclass( + buildSummaryclassArray()).name("defaultsearchdefinition")); + + return configArray; + } + + private List buildSummaryclassArray() { + return Arrays.asList( + new Summaryclass.Builder() + .id(0) + .name(DEFAULT_SUMMARY_CLASS) + .fields(Arrays.asList(new Fields.Builder().name(FIELD1) + .type("string"), + new Fields.Builder().name(FIELD2) + .type("string"))), + new Summaryclass.Builder() + .id(1) + .name(Execution.ATTRIBUTEPREFETCH) + .fields(Arrays.asList(new Fields.Builder().name(FIELD2) + .type("string"))), + new Summaryclass.Builder() + .id(2) + .name(THIRD_OPTION) + .fields(Arrays.asList(new Fields.Builder().name(FIELD3) + .type("string")))); + + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public final void testTrivial() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD1); + Result result = execution.search(query); + execution.fill(result); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + + @Test + public final void testWithOnlyAttribute() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD2); + Result result = execution.search(query); + execution.fill(result, THIRD_OPTION); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); + assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertTrue(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + + @Test + public final void testWithOnlyDiskfieldCorrectClassRequested() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD3); + Result result = execution.search(query); + execution.fill(result, THIRD_OPTION); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); + assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + @Test + public final void testTrivialCaseWithOnlyDiskfieldWrongClassRequested() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD1); + Result result = execution.search(query); + execution.fill(result, THIRD_OPTION); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); + assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java new file mode 100644 index 00000000000..c9d73853cca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -0,0 +1,928 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import com.yahoo.component.Version; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Alias; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Command; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.Sorting.AttributeSorter; +import com.yahoo.search.query.Sorting.FieldOrder; +import com.yahoo.search.query.Sorting.LowerCaseSorter; +import com.yahoo.search.query.Sorting.Order; +import com.yahoo.search.query.Sorting.UcaSorter; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Specification for the conversion of YQL+ expressions to Vespa search queries. + * + * @author steinar + * @author stiankri + */ +public class YqlParserTestCase { + + private final YqlParser parser = new YqlParser(new ParserEnvironment()); + + @Test + public void requireThatDefaultsAreSane() { + assertTrue(parser.isQueryParser()); + assertNull(parser.getDocTypes()); + } + + @Test + public void requireThatGroupingStepCanBeParsed() { + assertParse("select foo from bar where baz contains 'cox';", + "baz:cox"); + assertEquals("[]", + toString(parser.getGroupingSteps())); + + assertParse("select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count())));", + "baz:cox"); + assertEquals("[[]all(group(a) each(output(count())))]", + toString(parser.getGroupingSteps())); + + assertParse("select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count()))) " + + "| all(group(b) each(output(count())));", + "baz:cox"); + assertEquals("[[]all(group(a) each(output(count())))," + + " []all(group(b) each(output(count())))]", + toString(parser.getGroupingSteps())); + } + + @Test + public void requireThatGroupingContinuationCanBeParsed() { + assertParse("select foo from bar where baz contains 'cox' " + + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())));", + "baz:cox"); + assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", + toString(parser.getGroupingSteps())); + + assertParse("select foo from bar where baz contains 'cox' " + + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count()))) " + + "| [{ 'continuations': ['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]all(group(b) each(output(count())));", + "baz:cox"); + assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))," + + " [BCBBBBBDBF, BCBJBPCBJCCJ]all(group(b) each(output(count())))]", + toString(parser.getGroupingSteps())); + } + + @Test + public void test() { + assertParse("select foo from bar where title contains \"madonna\";", + "title:madonna"); + } + + @Test + public void testOr() { + assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\";", + "OR title:madonna title:saint"); + assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\" or title " + + "contains \"angel\";", + "OR title:madonna title:saint title:angel"); + } + + @Test + public void testAnd() { + assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\";", + "AND title:madonna title:saint"); + assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\" and title " + + "contains \"angel\";", + "AND title:madonna title:saint title:angel"); + } + + @Test + public void testAndNot() { + assertParse("select foo from bar where title contains \"madonna\" and !(title contains \"saint\");", + "+title:madonna -title:saint"); + } + + @Test + public void testLessThan() { + assertParse("select foo from bar where price < 500;", "price:<500"); + assertParse("select foo from bar where 500 < price;", "price:>500"); + } + + @Test + public void testGreaterThan() { + assertParse("select foo from bar where price > 500;", "price:>500"); + assertParse("select foo from bar where 500 > price;", "price:<500"); + } + + @Test + public void testLessThanOrEqual() { + assertParse("select foo from bar where price <= 500;", "price:[;500]"); + assertParse("select foo from bar where 500 <= price;", "price:[500;]"); + } + + @Test + public void testGreaterThanOrEqual() { + assertParse("select foo from bar where price >= 500;", "price:[500;]"); + assertParse("select foo from bar where 500 >= price;", "price:[;500]"); + } + + @Test + public void testEquality() { + assertParse("select foo from bar where price = 500;", "price:500"); + assertParse("select foo from bar where 500 = price;", "price:500"); + } + + @Test + public void testNegativeLessThan() { + assertParse("select foo from bar where price < -500;", "price:<-500"); + assertParse("select foo from bar where -500 < price;", "price:>-500"); + } + + @Test + public void testNegativeGreaterThan() { + assertParse("select foo from bar where price > -500;", "price:>-500"); + assertParse("select foo from bar where -500 > price;", "price:<-500"); + } + + @Test + public void testNegativeLessThanOrEqual() { + assertParse("select foo from bar where price <= -500;", "price:[;-500]"); + assertParse("select foo from bar where -500 <= price;", "price:[-500;]"); + } + + @Test + public void testNegativeGreaterThanOrEqual() { + assertParse("select foo from bar where price >= -500;", "price:[-500;]"); + assertParse("select foo from bar where -500 >= price;", "price:[;-500]"); + } + + @Test + public void testNegativeEquality() { + assertParse("select foo from bar where price = -500;", "price:-500"); + assertParse("select foo from bar where -500 = price;", "price:-500"); + } + + @Test + public void testAnnotatedLessThan() { + assertParse("select foo from bar where price < ([{\"filter\": true}](-500));", "|price:<-500"); + assertParse("select foo from bar where ([{\"filter\": true}]500) < price;", "|price:>500"); + } + + @Test + public void testAnnotatedGreaterThan() { + assertParse("select foo from bar where price > ([{\"filter\": true}]500);", "|price:>500"); + assertParse("select foo from bar where ([{\"filter\": true}](-500)) > price;", "|price:<-500"); + } + + @Test + public void testAnnotatedLessThanOrEqual() { + assertParse("select foo from bar where price <= ([{\"filter\": true}](-500));", "|price:[;-500]"); + assertParse("select foo from bar where ([{\"filter\": true}]500) <= price;", "|price:[500;]"); + } + + @Test + public void testAnnotatedGreaterThanOrEqual() { + assertParse("select foo from bar where price >= ([{\"filter\": true}]500);", "|price:[500;]"); + assertParse("select foo from bar where ([{\"filter\": true}](-500)) >= price;", "|price:[;-500]"); + } + + @Test + public void testAnnotatedEquality() { + assertParse("select foo from bar where price = ([{\"filter\": true}](-500));", "|price:-500"); + assertParse("select foo from bar where ([{\"filter\": true}]500) = price;", "|price:500"); + } + + @Test + public void testTermAnnotations() { + assertEquals("merkelapp", + getRootWord("select foo from bar where baz contains " + + "([ {\"label\": \"merkelapp\"} ]\"colors\");").getLabel()); + assertEquals("another", + getRootWord("select foo from bar where baz contains " + + "([ {\"annotations\": {\"cox\": \"another\"}} ]\"colors\");").getAnnotation("cox")); + assertEquals(23.0, getRootWord("select foo from bar where baz contains " + + "([ {\"significance\": 23.0} ]\"colors\");").getSignificance(), 1E-6); + assertEquals(23, getRootWord("select foo from bar where baz contains " + + "([ {\"id\": 23} ]\"colors\");").getUniqueID()); + assertEquals(150, getRootWord("select foo from bar where baz contains " + + "([ {\"weight\": 150} ]\"colors\");").getWeight()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"usePositionData\": false} ]\"colors\");").usePositionData()); + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"filter\": true} ]\"colors\");").isFilter()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"ranked\": false} ]\"colors\");").isRanked()); + + Substring origin = getRootWord("select foo from bar where baz contains " + + "([ {\"origin\": {\"original\": \"abc\", \"offset\": 1, \"length\": 2}} ]" + + "\"colors\");").getOrigin(); + assertEquals("abc", origin.string); + assertEquals(1, origin.start); + assertEquals(3, origin.end); + } + + @Test + public void testPhrase() { + assertParse("select foo from bar where baz contains phrase(\"a\", \"b\");", + "baz:\"a b\""); + } + + @Test + public void testNestedPhrase() { + assertParse("select foo from bar where baz contains phrase(\"a\", \"b\", phrase(\"c\", \"d\"));", + "baz:\"a b c d\""); + } + + @Test + public void testNestedPhraseSegment() { + assertParse("select foo from bar where baz contains " + + "phrase(\"a\", \"b\", [ {\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}} ]" + + "phrase(\"c\", \"d\"));", + "baz:\"a b 'c d'\""); + } + + @Test + public void testStemming() { + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"stem\": false} ]\"colors\");").isStemmed()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"stem\": true} ]\"colors\");").isStemmed()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "\"colors\";").isStemmed()); + } + + @Test + public void testAccentDropping() { + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"accentDrop\": false} ]\"colors\");").isNormalizable()); + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"accentDrop\": true} ]\"colors\");").isNormalizable()); + assertTrue(getRootWord("select foo from bar where baz contains " + + "\"colors\";").isNormalizable()); + } + + @Test + public void testCaseNormalization() { + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"normalizeCase\": false} ]\"colors\");").isLowercased()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"normalizeCase\": true} ]\"colors\");").isLowercased()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "\"colors\";").isLowercased()); + } + + @Test + public void testSegmentingRule() { + assertEquals(SegmentingRule.PHRASE, + getRootWord("select foo from bar where baz contains " + + "([ {\"andSegmenting\": false} ]\"colors\");").getSegmentingRule()); + assertEquals(SegmentingRule.BOOLEAN_AND, + getRootWord("select foo from bar where baz contains " + + "([ {\"andSegmenting\": true} ]\"colors\");").getSegmentingRule()); + assertEquals(SegmentingRule.LANGUAGE_DEFAULT, + getRootWord("select foo from bar where baz contains " + + "\"colors\";").getSegmentingRule()); + } + + @Test + public void testNfkc() { + assertEquals("a\u030a", + getRootWord("select foo from bar where baz contains " + + "([ {\"nfkc\": false} ]\"a\\u030a\");").getWord()); + assertEquals("\u00e5", + getRootWord("select foo from bar where baz contains " + + "([ {\"nfkc\": true} ]\"a\\u030a\");").getWord()); + assertEquals("\u00e5", + getRootWord("select foo from bar where baz contains " + + "\"a\\u030a\";").getWord()); + } + + @Test + public void testImplicitTransforms() { + assertFalse(getRootWord("select foo from bar where baz contains ([ {\"implicitTransforms\": " + + "false} ]\"cox\");").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains ([ {\"implicitTransforms\": " + + "true} ]\"cox\");").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains \"cox\";").isFromQuery()); + } + + @Test + public void testConnectivity() { + QueryTree parsed = parse("select foo from bar where " + + "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}}]\"madonna\") " + + "and title contains ([{\"id\": 2}]\"saint\") " + + "and title contains ([{\"id\": 3}]\"angel\");"); + assertEquals("AND title:madonna title:saint title:angel", + parsed.toString()); + AndItem root = (AndItem)parsed.getRoot(); + WordItem first = (WordItem)root.getItem(0); + WordItem second = (WordItem)root.getItem(1); + WordItem third = (WordItem)root.getItem(2); + assertTrue(first.getConnectedItem() == third); + assertEquals(first.getConnectivity(), 7.0d, 1E-6); + assertNull(second.getConnectedItem()); + + assertParseFail("select foo from bar where " + + "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}}]\"madonna\") " + + "and title contains ([{\"id\": 2}]\"saint\") " + + "and title contains ([{\"id\": 3}]\"angel\");", + new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " + + "which does not exist in the query.")); + } + + @Test + public void testAnnotatedPhrase() { + QueryTree parsed = + parse("select foo from bar where baz contains ([{\"label\": \"hello world\"}]phrase(\"a\", \"b\"));"); + assertEquals("baz:\"a b\"", parsed.toString()); + PhraseItem phrase = (PhraseItem)parsed.getRoot(); + assertEquals("hello world", phrase.getLabel()); + } + + @Test + public void testRange() { + QueryTree parsed = parse("select foo from bar where range(baz,1,8);"); + assertEquals("baz:[1;8]", parsed.toString()); + } + + @Test + public void testNegativeRange() { + QueryTree parsed = parse("select foo from bar where range(baz,-8,-1);"); + assertEquals("baz:[-8;-1]", parsed.toString()); + } + + @Test + public void testRangeIllegalArguments() { + assertParseFail("select foo from bar where range(baz,cox,8);", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + } + + @Test + public void testNear() { + assertParse("select foo from bar where description contains near(\"a\", \"b\");", + "NEAR(2) description:a description:b"); + assertParse("select foo from bar where description contains ([ {\"distance\": 100} ]near(\"a\", \"b\"));", + "NEAR(100) description:a description:b"); + } + + @Test + public void testOrderedNear() { + assertParse("select foo from bar where description contains onear(\"a\", \"b\");", + "ONEAR(2) description:a description:b"); + assertParse("select foo from bar where description contains ([ {\"distance\": 100} ]onear(\"a\", \"b\"));", + "ONEAR(100) description:a description:b"); + } + + //This test is order dependent. Fix this!! + @Test + public void testWand() { + assertParse("select foo from bar where wand(description, {\"a\":1, \"b\":2});", + "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); + assertParse("select foo from bar where [ {\"scoreThreshold\": 13.3, \"targetNumHits\": 7, " + + "\"thresholdBoostFactor\": 2.3} ]wand(description, {\"a\":1, \"b\":2});", + "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); + } + + @Test + public void testNumericWand() { + String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}"; + assertParse("select foo from bar where wand(description, [[11,1], [37,2]]);", numWand); + assertParse("select foo from bar where wand(description, [[11L,1], [37L,2]]);", numWand); + assertParseFail("select foo from bar where wand(description, 12);", + new IllegalArgumentException("Expected ARRAY or MAP, got LITERAL.")); + } + + @Test + //This test is order dependent. Fix it! + public void testWeightedSet() { + assertParse("select foo from bar where weightedSet(description, {\"a\":1, \"b\":2});", + "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}"); + assertParseFail("select foo from bar where weightedSet(description, {\"a\":g, \"b\":2});", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + assertParseFail("select foo from bar where weightedSet(description);", + new IllegalArgumentException("Expected 2 arguments, got 1.")); + } + + //This test is order dependent. Fix it! + @Test + public void testDotProduct() { + assertParse("select foo from bar where dotProduct(description, {\"a\":1, \"b\":2});", + "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}"); + assertParse("select foo from bar where dotProduct(description, {\"a\":2});", + "DOTPRODUCT description{[2]:\"a\"}"); + } + + @Test + public void testPredicate() { + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23L});", + "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23});", + "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); + assertParse("select foo from bar where predicate(predicate_field, 0, void);", + "PREDICATE_QUERY_ITEM "); + } + + @Test + public void testPredicateWithSubQueries() { + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"0x03\":{\"gender\":\"male\"},\"0x01\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"0x80ffffffffffffff\":{\"age\":23L}});", + "PREDICATE_QUERY_ITEM gender=male[0x3], hobby=music[0x1], hobby=hiking[0x1], age:23[0x80ffffffffffffff]"); + assertParseFail("select foo from bar where predicate(foo, null, {\"0x80000000000000000\":{\"age\":23}});", + new NumberFormatException("Too long subquery string: 0x80000000000000000")); + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"[0,1]\":{\"gender\":\"male\"},\"[0]\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"[62, 63]\":{\"age\":23L}});", + "PREDICATE_QUERY_ITEM gender=male[0x3], hobby=music[0x1], hobby=hiking[0x1], age:23[0xc000000000000000]"); + } + + @Test + public void testRank() { + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\");", + "RANK a:A b:B"); + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\", c " + + "contains \"C\");", + "RANK a:A b:B c:C"); + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\" or c " + + "contains \"C\");", + "RANK a:A (OR b:B c:C)"); + } + + @Test + public void testWeakAnd() { + assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\");", + "WAND(100) a:A b:B"); + assertParse("select foo from bar where [{\"targetNumHits\": 37}]weakAnd(a contains \"A\", " + + "b contains \"B\");", + "WAND(37) a:A b:B"); + + QueryTree tree = parse("select foo from bar where [{\"scoreThreshold\": 41}]weakAnd(a " + + "contains \"A\", b contains \"B\");"); + assertEquals("WAND(100) a:A b:B", tree.toString()); + assertEquals(WeakAndItem.class, tree.getRoot().getClass()); + assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); + } + + @Test + public void testEquiv() { + assertParse("select foo from bar where fieldName contains equiv(\"A\",\"B\");", + "EQUIV fieldName:A fieldName:B"); + assertParse("select foo from bar where fieldName contains " + + "equiv(\"ny\",phrase(\"new\",\"york\"));", + "EQUIV fieldName:ny fieldName:\"new york\""); + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\");", + new IllegalArgumentException("Expected 2 or more arguments, got 1.")); + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", nalle(void));", + new IllegalArgumentException("Expected function 'phrase', got 'nalle'.")); + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", 42);", + new ClassCastException("Cannot cast java.lang.Integer to java.lang.String")); + } + + @Test + public void testAffixItems() { + assertRootClass("select foo from bar where baz contains ([ {\"suffix\": true} ]\"colors\");", + SuffixItem.class); + assertRootClass("select foo from bar where baz contains ([ {\"prefix\": true} ]\"colors\");", + PrefixItem.class); + assertRootClass("select foo from bar where baz contains ([ {\"substring\": true} ]\"colors\");", + SubstringItem.class); + assertParseFail("select foo from bar where description contains ([ {\"suffix\": true, " + + "\"prefix\": true} ]\"colors\");", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + assertParseFail("select foo from bar where description contains ([ {\"suffix\": true, " + + "\"substring\": true} ]\"colors\");", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + } + + @Test + public void testLongNumberInSimpleExpression() { + assertParse("select foo from bar where price = 8589934592L;", + "price:8589934592"); + } + + @Test + public void testNegativeLongNumberInSimpleExpression() { + assertParse("select foo from bar where price = -8589934592L;", + "price:-8589934592"); + } + + @Test + public void testSources() { + assertSources("select foo from sourceA where price <= 500;", + Arrays.asList("sourceA")); + } + + @Test + public void testWildCardSources() { + assertSources("select foo from sources * where price <= 500;", + Collections.emptyList()); + } + + @Test + public void testMultiSources() { + assertSources("select foo from sources sourceA, sourceB where price <= 500;", + Arrays.asList("sourceA", "sourceB")); + } + + @Test + public void testFields() { + assertSummaryFields("select fieldA from bar where price <= 500;", + Arrays.asList("fieldA")); + assertSummaryFields("select fieldA, fieldB from bar where price <= 500;", + Arrays.asList("fieldA", "fieldB")); + assertSummaryFields("select fieldA, fieldB, fieldC from bar where price <= 500;", + Arrays.asList("fieldA", "fieldB", "fieldC")); + assertSummaryFields("select * from bar where price <= 500;", + Collections.emptyList()); + } + + @Test + public void testFieldsRoot() { + assertParse("select * from bar where price <= 500;", + "price:[;500]"); + } + + @Test + public void testOffset() { + assertParse("select foo from bar where title contains \"madonna\" offset 37;", + "title:madonna"); + assertEquals(Integer.valueOf(37), parser.getOffset()); + } + + @Test + public void testLimit() { + assertParse("select foo from bar where title contains \"madonna\" limit 29;", + "title:madonna"); + assertEquals(Integer.valueOf(29), parser.getHits()); + } + + @Test + public void testOffsetAndLimit() { + assertParse("select foo from bar where title contains \"madonna\" limit 31 offset 29;", + "title:madonna"); + assertEquals(Integer.valueOf(29), parser.getOffset()); + assertEquals(Integer.valueOf(2), parser.getHits()); + + assertParse("select * from bar where title contains \"madonna\" limit 41 offset 37;", + "title:madonna"); + assertEquals(Integer.valueOf(37), parser.getOffset()); + assertEquals(Integer.valueOf(4), parser.getHits()); + } + + @Test + public void testTimeout() { + assertParse("select * from bar where title contains \"madonna\" timeout 7;", + "title:madonna"); + assertEquals(Integer.valueOf(7), parser.getTimeout()); + + assertParse("select foo from bar where title contains \"madonna\" limit 600 timeout 3;", + "title:madonna"); + assertEquals(Integer.valueOf(3), parser.getTimeout()); + } + + @Test + public void testOrdering() { + assertParse("select foo from bar where title contains \"madonna\" order by something asc, " + + "shoesize desc limit 600 timeout 3;", + "title:madonna"); + assertEquals(2, parser.getSorting().fieldOrders().size()); + assertEquals("something", parser.getSorting().fieldOrders().get(0).getFieldName()); + assertEquals(Order.ASCENDING, parser.getSorting().fieldOrders().get(0).getSortOrder()); + assertEquals("shoesize", parser.getSorting().fieldOrders().get(1).getFieldName()); + assertEquals(Order.DESCENDING, parser.getSorting().fieldOrders().get(1).getSortOrder()); + + assertParse("select foo from bar where title contains \"madonna\" order by other limit 600 " + + "timeout 3;", + "title:madonna"); + assertEquals("other", parser.getSorting().fieldOrders().get(0).getFieldName()); + assertEquals(Order.ASCENDING, parser.getSorting().fieldOrders().get(0).getSortOrder()); + } + + @Test + public void testAnnotatedOrdering() { + assertParse( + "select foo from bar where title contains \"madonna\"" + + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc" + + " limit 600" + " timeout 3;", "title:madonna"); + final FieldOrder fieldOrder = parser.getSorting().fieldOrders().get(0); + assertEquals("other", fieldOrder.getFieldName()); + assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(UcaSorter.class, sorter.getClass()); + final UcaSorter uca = (UcaSorter) sorter; + assertEquals("en_US", uca.getLocale()); + assertEquals(UcaSorter.Strength.IDENTICAL, uca.getStrength()); + } + + @Test + public void testMultipleAnnotatedOrdering() { + assertParse( + "select foo from bar where title contains \"madonna\"" + + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc," + + " [{\"function\": \"lowercase\"}]something asc" + + " limit 600" + " timeout 3;", "title:madonna"); + { + final FieldOrder fieldOrder = parser.getSorting().fieldOrders() + .get(0); + assertEquals("other", fieldOrder.getFieldName()); + assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(UcaSorter.class, sorter.getClass()); + final UcaSorter uca = (UcaSorter) sorter; + assertEquals("en_US", uca.getLocale()); + assertEquals(UcaSorter.Strength.IDENTICAL, uca.getStrength()); + } + { + final FieldOrder fieldOrder = parser.getSorting().fieldOrders() + .get(1); + assertEquals("something", fieldOrder.getFieldName()); + assertEquals(Order.ASCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(LowerCaseSorter.class, sorter.getClass()); + } + } + + @Test + public void testSegmenting() { + assertParse("select * from bar where ([{\"segmenter\": {\"version\": \"58.67.49\", \"backend\": " + + "\"yell\"}}] title contains \"madonna\");", + "title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("58.67.49"), parser.getSegmenterVersion()); + + assertParse("select * from bar where ([{\"segmenter\": {\"version\": \"8.7.3\", \"backend\": " + + "\"yell\"}}]([{\"targetNumHits\": 9999438}] weakAnd(format contains \"online\", title contains " + + "\"madonna\")));", + "WAND(9999438) format:online title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("8.7.3"), parser.getSegmenterVersion()); + + assertParse("select * from bar where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": " + + "\"yell\"}}] ([{\"targetNumHits\": 99909438}] weakAnd(format contains \"online\", title contains " + + "\"madonna\"));", + "WAND(99909438) format:online title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("18.47.39"), parser.getSegmenterVersion()); + + assertParse("select * from bar where [{\"targetNumHits\": 99909438}] weakAnd(format contains " + + "\"online\", title contains \"madonna\");", + "WAND(99909438) format:online title:madonna"); + assertNull(parser.getSegmenterBackend()); + assertNull(parser.getSegmenterVersion()); + + assertParse("select * from bar where [{\"segmenter\": {\"version\": \"58.67.49\", \"backend\": " + + "\"yell\"}}](title contains \"madonna\") order by shoesize;", + "title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("58.67.49"), parser.getSegmenterVersion()); + } + + @Test + public void testNegativeHitLimit() { + assertParse( + "select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1);", + "foo:[0;1;-38]"); + } + + @Test + public void testRangeSearchHitPopulationOrdering() { + assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true}]range(foo, 0, 1);", "foo:[0;1;38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": false}]range(foo, 0, 1);", "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": true}]range(foo, 0, 1);", "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": false}]range(foo, 0, 1);", "foo:[0;1;38]"); + + boolean gotExceptionFromParse = false; + try { + parse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true, \"descending\": false}]range(foo, 0, 1);"); + } catch (IllegalArgumentException e) { + assertTrue("Expected information about abuse of settings.", + e.getMessage().contains("both ascending and descending ordering set")); + gotExceptionFromParse = true; + } + assertTrue(gotExceptionFromParse); + } + + @Test + public void testOpenIntervals() { + assertParse("select * from sources * where range(title, 0.0, 500.0);", + "title:[0.0;500.0]"); + assertParse( + "select * from sources * where [{\"bounds\": \"open\"}]range(title, 0.0, 500.0);", + "title:<0.0;500.0>"); + assertParse( + "select * from sources * where [{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0);", + "title:<0.0;500.0]"); + assertParse( + "select * from sources * where [{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0);", + "title:[0.0;500.0>"); + } + + @Test + public void testInheritedAnnotations() { + { + QueryTree x = parse("select * from sources * where ([{\"ranked\": false}](foo contains \"a\" and bar contains \"b\")) or foor contains ([{\"ranked\": false}]\"c\");"); + List terms = QueryTree.getPositiveTerms(x); + assertEquals(3, terms.size()); + for (IndexedItem term : terms) { + assertFalse(((Item) term).isRanked()); + } + } + { + QueryTree x = parse("select * from sources * where [{\"ranked\": false}](foo contains \"a\" and bar contains \"b\");"); + List terms = QueryTree.getPositiveTerms(x); + assertEquals(2, terms.size()); + for (IndexedItem term : terms) { + assertFalse(((Item) term).isRanked()); + } + } + } + + @Test + public void testMoreInheritedAnnotations() { + final String yqlQuery = "select * from sources * where " + + "([{\"ranked\": false}](foo contains \"a\" " + + "and ([{\"ranked\": true}](bar contains \"b\" " + + "or ([{\"ranked\": false}](foo contains \"c\" " + + "and foo contains ([{\"ranked\": true}]\"d\")))))));"; + QueryTree x = parse(yqlQuery); + List terms = QueryTree.getPositiveTerms(x); + assertEquals(4, terms.size()); + for (IndexedItem term : terms) { + switch (term.getIndexedString()) { + case "a": + case "c": + assertFalse(((Item) term).isRanked()); + break; + case "b": + case "d": + assertTrue(((Item) term).isRanked()); + break; + default: + fail(); + } + } + } + + @Test + public void testFieldAliases() { + IndexInfoConfig modelConfig = new IndexInfoConfig(new IndexInfoConfig.Builder().indexinfo(new Indexinfo.Builder() + .name("music").command(new Command.Builder().indexname("title").command("index")) + .alias(new Alias.Builder().alias("song").indexname("title")))); + IndexModel model = new IndexModel(modelConfig, (QrSearchersConfig)null); + + IndexFacts indexFacts = new IndexFacts(model); + ParserEnvironment parserEnvironment = new ParserEnvironment().setIndexFacts(indexFacts); + YqlParser configuredParser = new YqlParser(parserEnvironment); + QueryTree x = configuredParser.parse(new Parsable() + .setQuery("select * from sources * where title contains \"a\" and song contains \"b\";")); + List terms = QueryTree.getPositiveTerms(x); + assertEquals(2, terms.size()); + for (IndexedItem term : terms) { + assertEquals("title", term.getIndexName()); + } + } + + @Test + public void testRegexp() { + QueryTree x = parse("select * from sources * where foo matches \"a b\";"); + Item root = x.getRoot(); + assertSame(RegExpItem.class, root.getClass()); + assertEquals("a b", ((RegExpItem) root).stringValue()); + } + + @Test + public void testWordAlternatives() { + QueryTree x = parse("select * from sources * where foo contains alternatives({\"trees\": 1.0, \"tree\": 0.7});"); + Item root = x.getRoot(); + assertSame(WordAlternativesItem.class, root.getClass()); + WordAlternativesItem alternatives = (WordAlternativesItem) root; + checkWordAlternativesContent(alternatives); + } + + @Test + public void testWordAlternativesWithOrigin() { + QueryTree x = parse("select * from sources * where foo contains" + + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + "alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + Item root = x.getRoot(); + assertSame(WordAlternativesItem.class, root.getClass()); + WordAlternativesItem alternatives = (WordAlternativesItem) root; + checkWordAlternativesContent(alternatives); + Substring origin = alternatives.getOrigin(); + assertEquals(1, origin.start); + assertEquals(6, origin.end); + assertEquals("trees", origin.getValue()); + assertEquals(" trees ", origin.getSuperstring()); + } + + @Test + public void testWordAlternativesInPhrase() { + QueryTree x = parse("select * from sources * where" + + " foo contains phrase(\"forest\", alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + Item root = x.getRoot(); + assertSame(PhraseItem.class, root.getClass()); + PhraseItem phrase = (PhraseItem) root; + assertEquals(2, phrase.getItemCount()); + assertEquals("forest", ((WordItem) phrase.getItem(0)).getWord()); + checkWordAlternativesContent((WordAlternativesItem) phrase.getItem(1)); + } + + private void checkWordAlternativesContent(WordAlternativesItem alternatives) { + boolean seenTree = false; + boolean seenForest = false; + final String forest = "trees"; + final String tree = "tree"; + assertEquals(2, alternatives.getAlternatives().size()); + for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) { + if (tree.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenTree); + seenTree = true; + assertEquals(.7d, alternative.exactness, 1e-15d); + } else if (forest.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenForest); + seenForest = true; + assertEquals(1.0d, alternative.exactness, 1e-15d); + } else { + fail("Unexpected term: " + alternative.word); + } + } + } + + private void assertParse(String yqlQuery, String expectedQueryTree) { + assertEquals(expectedQueryTree, parse(yqlQuery).toString()); + } + + private void assertParseFail(String yqlQuery, Throwable expectedException) { + try { + parse(yqlQuery); + } catch (Throwable t) { + assertEquals(expectedException.getClass(), t.getClass()); + assertEquals(expectedException.getMessage(), t.getMessage()); + return; + } + fail("Parse succeeded: " + yqlQuery); + } + + private void assertSources(String yqlQuery, Collection expectedSources) { + parse(yqlQuery); + assertEquals(new HashSet<>(expectedSources), parser.getYqlSources()); + } + + private void assertSummaryFields(String yqlQuery, Collection expectedSummaryFields) { + parse(yqlQuery); + assertEquals(new HashSet<>(expectedSummaryFields), parser.getYqlSummaryFields()); + } + + private WordItem getRootWord(String yqlQuery) { + Item root = parse(yqlQuery).getRoot(); + assertTrue(root instanceof WordItem); + return (WordItem)root; + } + + private void assertRootClass(String yqlQuery, Class expectedRootClass) { + assertEquals(expectedRootClass, parse(yqlQuery).getRoot().getClass()); + } + + private QueryTree parse(String yqlQuery) { + return parser.parse(new Parsable().setQuery(yqlQuery)); + } + + private static String toString(List steps) { + List actual = new ArrayList<>(steps.size()); + for (VespaGroupingStep step : steps) { + actual.add(step.continuations().toString() + + step.getOperation()); + } + return actual.toString(); + } +} diff --git a/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java b/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java new file mode 100644 index 00000000000..6fd5c238d6f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text.interpretation.test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import junit.framework.TestCase; + +import com.yahoo.text.interpretation.AnnotationClass; +import com.yahoo.text.interpretation.Annotations; +import com.yahoo.text.interpretation.Interpretation; +import com.yahoo.text.interpretation.Span; + +/** + * @author Arne Bergene Fossaa + */ +public class AnnotationTestCase extends TestCase { + + public void testSimpleAnnotations() { + Interpretation i= new Interpretation("new york hotel"); + i.annotate("sentence").put("isValid",true); + i.annotate(0,3,"token"); + i.annotate(0,8,"place_name").put("^taxonomy:place_category","city"); + i.annotate(0,8,"place_name").put("woeid","12345678"); + //i.getInterpretationAnnotation().put("domain","jaffa"); + i.setProbability(0.5); + + assertNotNull(i.get("sentence")); + } + + public void testAnnotationAPI() { + Interpretation a = new Interpretation("new york hotel"); + + a.annotate(0,3,"token"); + a.annotate(0,8,"state").put("name","New York"); + a.annotate(0,8,"state").put("country","US"); + a.annotate(0,8,"state").put("coast","east"); + a.annotate(9,14,"business"); + a.annotate(4,8,"token"); + a.annotate(9,14,"token"); + + for(Span span : a.getTokens()) { + assertTrue(span.hasClass(new AnnotationClass("token"))); + } + + Set annotationClasses = a.getClasses(0,3); + Set testClass = new HashSet<>(Arrays.asList( + new AnnotationClass("token"), new AnnotationClass("state"))); + assertEquals(testClass,annotationClasses); + + assertNull(a.get("state","country")); + assertEquals("US", a.get(0,8,"state","country")); + + assertEquals("new york", a.root().getSubSpans().get(0).getText()); + assertEquals("hotel", a.root().getSubSpans().get(1).getText()); + assertEquals(2,a.root().getSubSpans().size()); + + + //Test scoring + a.setProbability(5); + Interpretation b = new Interpretation("new york hotel"); + b.setProbability(3); + + //Test the interpretation API + a.annotate("vespa_query"); + + assertNotNull(a.get("vespa_query")); + + //This is bad about the API, getTokens may not necessairily return what a user thinks a token is + //But it should still be tested + a.annotate(0,1,"n"); + Set testSet = new HashSet<>(Arrays.asList("n","york","hotel")); + for(Span span:a.getTokens()) { + assertTrue(testSet.remove(span.getText())); + } + assertEquals(0,testSet.size()); + } + + //The following testcase is a test with the api on a use_case, no cornercases here + public void testUsability() { + + Interpretation interpretation = new Interpretation("new york crab pizza"); + interpretation.annotate(0,8,"place_name").put("^taxonomy:place_category","city"); + interpretation.annotate(0,8,"place_name").put("woe_id",2459115); + interpretation.annotate(9,13,"food"); + interpretation.annotate(14,19,"food"); + + //Here we want to write code that finds out if the interpretation + //matches pizza and toppings. + + List pizzaSpans = interpretation.getTermSpans("pizza"); + + if(pizzaSpans.size() > 0) { + //We know that we have pizza, now we want to get some topping + //In a perfect world, pizza topping would have its own annotation class + //but for now, we'll just accept terms that have been tokenized with food + + List toppings = new ArrayList<>(); + for(Annotations annotations :interpretation.getAll("food")) { + if(!annotations.getSubString().equalsIgnoreCase("pizza")) { + toppings.add(annotations.getSubString()); + } + } + //We also want to find out where we should search for pizza places + //Since we know that our interpreter engine is smart, we know + //that all spans that has the annotation "place_name" has a "woe_id". + int woe_id = 0; + + for(Annotations annotations :interpretation.getAll("place_name")) { + //This will return either 0 or throw a bad exception + //if a number is not found + woe_id = annotations.getInteger("woe_id"); + } + assertEquals(Arrays.asList("crab"),toppings); + assertEquals(2459115,woe_id); + + } + + } +} diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/ListMergerTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/ListMergerTestCase.java new file mode 100644 index 00000000000..55af3dd43e5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/ListMergerTestCase.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.vespa.streamingvisitors.ListMerger; + +import java.util.List; +import java.util.LinkedList; + +/** + * @author Ulf Carlin + */ +public class ListMergerTestCase extends junit.framework.TestCase { + private void initializeLists(List list1, List list2, int entryCount, int padding) { + for (int i = 0; i < entryCount; i++) { + if ((i % 2) == 0) { + list1.add("String " + String.format("%0" + padding + "d", (i+1))); + } else { + list2.add("String " + String.format("%0" + padding + "d", (i+1))); + } + } + } + + private void verifyList(List list, int entryCount, int padding) { + assertEquals(entryCount, list.size()); + for (int i = 0; i < entryCount; i++) { + assertEquals("String " + String.format("%0" + padding + "d", (i+1)), list.get(i)); + } + } + + public void testMergeLists() { + int entryCount = 6; + int padding = (int)Math.log10(entryCount) + 1; + + List list1 = new LinkedList<>(); + List list2 = new LinkedList<>(); + initializeLists(list1, list2, entryCount, padding); + + List newList = ListMerger.mergeIntoArrayList(list1, list2); + verifyList(newList, entryCount, padding); + + newList = ListMerger.mergeIntoArrayList(list1, list2, entryCount/2); + verifyList(newList, entryCount/2, padding); + + ListMerger.mergeLinkedLists(list1, list2, entryCount/2); + verifyList(list1, entryCount/2, padding); + } + + public void testMergeListsReversed() { + int entryCount = 6; + int padding = (int)Math.log10(entryCount) + 1; + + List list1 = new LinkedList<>(); + List list2 = new LinkedList<>(); + initializeLists(list2, list1, entryCount, padding); + + List newList = ListMerger.mergeIntoArrayList(list1, list2); + verifyList(newList, entryCount, padding); + + newList = ListMerger.mergeIntoArrayList(list1, list2, entryCount/2); + verifyList(newList, entryCount/2, padding); + + ListMerger.mergeLinkedLists(list1, list2, entryCount/2); + verifyList(list1, entryCount/2, padding); + } + + /* + public void testMergeListsPerformance() { + int entryCount = 2000000; // 2000000 + int padding = (int)Math.log10(entryCount) + 1; + + List list1 = new LinkedList(); + List list2 = new LinkedList(); + initializeLists(list1, list2, entryCount, padding); + + long startTime = System.currentTimeMillis(); + //List newList = ListMerger.mergeIntoArrayList(list1, list2); + //List newList = ListMerger.mergeIntoArrayList(list1, list2, entryCount/2); + ListMerger.mergeLinkedLists(list1, list2, entryCount); + //ListMerger.mergeLinkedLists(list1, list2, entryCount/2); + long endTime = System.currentTimeMillis(); + long elapsedTime = endTime - startTime; + double seconds = elapsedTime / 1.0E03; + System.out.println ("Elapsed Time = " + seconds + " seconds"); + //assertEquals(entryCount/2, newList.size()); + } + */ +} diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java new file mode 100644 index 00000000000..901c9aa79d4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/MetricsSearcherTestCase.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vdslib.VisitorStatistics; +import org.junit.Test; + +import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author Ulf Carlin + */ +public class MetricsSearcherTestCase { + private MetricsSearcher metricsSearcher = new MetricsSearcher(); + private MockBackend backend = new MockBackend(); + private Chain chain = new Chain<>(metricsSearcher, backend); + private Execution.Context context = Execution.Context.createContextStub(null); + private MetricsSearcher.Stats expStatsLt1 = new MetricsSearcher.Stats(); + private static final String LOADTYPE1 = "lt1"; + private MetricsSearcher.Stats expStatsLt2 = new MetricsSearcher.Stats(); + private static final String LOADTYPE2 = "lt2"; + + private void verifySearch(String metricParam, String message, String detailedMessage) { + Result result = new Execution(chain, context).search(new Query("?query=test&" + metricParam)); + assertEquals(1, result.hits().size()); + if (message == null) { + assertEquals("news:0", result.hits().get(0).getId().toString()); + } else { + assertNotNull(result.hits().getError()); + assertTrue("Expected '" + message + "' to be contained in '" + + result.hits().getErrorHit().errors().iterator().next().getMessage() + "'", + result.hits().getErrorHit().errors().iterator().next().getMessage().contains(message)); + assertTrue("Expected '" + detailedMessage + "' to be contained in '" + + result.hits().getErrorHit().errors().iterator().next().getDetailedMessage() + "'", + result.hits().getErrorHit().errors().iterator().next().getDetailedMessage().contains(detailedMessage)); + } + + if (metricParam == null) { + return; + } + + MetricsSearcher.Stats expStats; + MetricsSearcher.Stats actualStats; + if (metricParam.contains(LOADTYPE1)) { + expStats = expStatsLt1; + actualStats = metricsSearcher.statMap.get(LOADTYPE1); + } else { + expStats = expStatsLt2; + actualStats = metricsSearcher.statMap.get(LOADTYPE2); + } + + expStats.count++; + if (message == null) { + expStats.ok++; + } else { + expStats.failed++; + } + if (metricParam.contains(LOADTYPE1)) { + expStats.dataStreamed += 16; + expStats.documentsStreamed += 2; + } + + assertEquals(expStats.count, actualStats.count); + assertEquals(expStats.ok, actualStats.ok); + assertEquals(expStats.failed, actualStats.failed); + assertEquals(expStats.dataStreamed, actualStats.dataStreamed); + assertEquals(expStats.documentsStreamed, actualStats.documentsStreamed); + } + + @Test + public void testBasics() { + // Start counting at -1 since count is reset upon the first query by MetricsSearcher.search + expStatsLt1.count--; + String[] loadTypes = { LOADTYPE1, LOADTYPE2}; + for (String loadType : loadTypes) { + verifySearch("streaming.loadtype="+loadType, null, null); + verifySearch("metricsearcher.id="+loadType, null, null); + verifySearch(null, null, null); + verifySearch("streaming.loadtype="+loadType, "Backend communication error", "Detailed error message"); + } + + } + + @Test + public void searcherDoesNotTryToDereferenceNullQueryContext() { + backend.setImplicitlyCreateContext(false); + // This will crash with an NPE if the searcher does not cope with null + // query contexts. + new Execution(chain, context).search(new Query("?query=test&streaming.loadtype=" + LOADTYPE1)); + } + + private static class MockBackend extends Searcher { + private int sequenceNumber = 0; + private VisitorStatistics visitorStats = new VisitorStatistics(); + private boolean implicitlyCreateContext = true; + + private MockBackend() { + visitorStats.setBucketsVisited(1); + visitorStats.setBytesReturned(8); + visitorStats.setBytesVisited(16); + visitorStats.setDocumentsReturned(1); + visitorStats.setDocumentsVisited(2); + } + + public void setImplicitlyCreateContext(boolean implicitlyCreateContext) { + this.implicitlyCreateContext = implicitlyCreateContext; + } + + public @Override Result search(Query query, Execution execution) { + if (implicitlyCreateContext) { + String loadType = query.properties().getString("streaming.loadtype"); + assignContextProperties(query, loadType); + } + + Result result = new Result(query); + if (sequenceNumber == 3 || sequenceNumber == 7) { + result.hits().addError(ErrorMessage.createBackendCommunicationError("Detailed error message")); + } else { + result.hits().add(new Hit("news:0")); + } + sequenceNumber++; + return result; + } + + private void assignContextProperties(Query query, String loadType) { + if (loadType != null && loadType.equals(LOADTYPE1)) { + query.getContext(true).setProperty(VdsStreamingSearcher.STREAMING_STATISTICS, visitorStats); + } else { + query.getContext(true).setProperty(VdsStreamingSearcher.STREAMING_STATISTICS, null); + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java new file mode 100644 index 00000000000..1931dd2179e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java @@ -0,0 +1,295 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.document.select.parser.TokenMgrError; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.prelude.fastsearch.*; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.VisitorStatistics; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * @author Ulf Carlin + */ +public class VdsStreamingSearcherTestCase { + public static final String USERDOC_ID_PREFIX = "userdoc:namespace:1:userspecific"; + public static final String GROUPDOC_ID_PREFIX = "groupdoc:namespace:group1:userspecific"; + + private static class MockVisitor implements Visitor { + private Query query; + String searchCluster; + Route route; + int totalHitCount; + private final List hits = new ArrayList<>(); + private final Map summaryMap = new HashMap<>(); + private final List groupings = new ArrayList<>(); + + MockVisitor(Query query, String searchCluster, Route route) { + this.query = query; + this.searchCluster = searchCluster; + this.route = route; + } + + @Override + public void doSearch() throws InterruptedException, ParseException, TimeoutException { + String queryString = query.getModel().getQueryString(); + if (queryString.compareTo("parseexception") == 0) { + throw new ParseException("Parsing failed"); + } else if (queryString.compareTo("tokenizeexception") == 0) { + throw new TokenMgrError("Tokenization failed", 0); + } else if (queryString.compareTo("interruptedexception") == 0) { + throw new InterruptedException("Interrupted"); + } else if (queryString.compareTo("timeoutexception") == 0) { + throw new TimeoutException("Timed out"); + } else if (queryString.compareTo("illegalargumentexception") == 0) { + throw new IllegalArgumentException("Illegal argument"); + } else if (queryString.compareTo("nosummary") == 0) { + String docId = USERDOC_ID_PREFIX + 0; + totalHitCount = 1; + hits.add(new SearchResult.Hit(docId, 1.0)); + } else if (queryString.compareTo("nosummarytofill") == 0) { + addResults(USERDOC_ID_PREFIX, 1, true); + } else if (queryString.compareTo("oneuserhit") == 0) { + addResults(USERDOC_ID_PREFIX, 1, false); + } else if (queryString.compareTo("twouserhits") == 0) { + addResults(USERDOC_ID_PREFIX, 2, false); + } else if (queryString.compareTo("twogrouphitsandoneuserhit") == 0) { + addResults(GROUPDOC_ID_PREFIX, 2, false); + addResults(USERDOC_ID_PREFIX, 1, false); + } else if (queryString.compareTo("onegroupinghit") == 0) { + groupings.add(new Grouping()); + } + + } + + private void addResults(String idPrefix, int hitCount, boolean emptyDocsum) { + totalHitCount += hitCount; + for (int i=0; i getHits() { + return hits; + } + + @Override + public Map getSummaryMap() { + return summaryMap; + } + + @Override + public int getTotalHitCount() { + return totalHitCount; + } + + @Override + public List getGroupings() { + return groupings; + } + } + + private static class MockVisitorFactory implements VisitorFactory { + @Override + public Visitor createVisitor(Query query, String searchCluster, Route route) { + return new MockVisitor(query, searchCluster, route); + } + } + + private static Result executeQuery(VdsStreamingSearcher searcher, Query query) { + QueryPacket queryPacket = QueryPacket.create(query); + CacheKey cacheKey = new CacheKey(queryPacket); + Execution execution = new Execution(new Execution.Context(null, null, null, null, null)); + return searcher.doSearch2(query, queryPacket, cacheKey, execution); + } + + private static Query[] generateTestQueries(String queryString) { + Query[] queries = new Query[4]; // Increase coverage + for (int i = 0; i getter = new ConfigGetter<>(DocumentdbInfoConfig.class); + DocumentdbInfoConfig config = getter.getConfig("file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"); + searcher.init(new SummaryParameters("default"), + new ClusterParams("clusterName"), + new CacheParams(100, 1e64), + config); + + // Magic query values are used to trigger specific behaviors from mock visitor. + checkError(searcher, "/?query=noselection", + "Backend communication error", "Streaming search needs one and only one"); + checkError(searcher, "/?streaming.userid=1&query=parseexception", + "Backend communication error", "Failed to parse document selection string"); + checkError(searcher, "/?streaming.userid=1&query=tokenizeexception", + "Backend communication error", "Failed to tokenize document selection string"); + checkError(searcher, "/?streaming.userid=1&query=interruptedexception", + "Backend communication error", "Interrupted"); + checkError(searcher, "/?streaming.userid=1&query=timeoutexception", + "Timed out", "Timed out"); + checkError(searcher, "/?streaming.userid=1&query=illegalargumentexception", + "Backend communication error", "Illegal argument"); + checkError(searcher, "/?streaming.userid=1&query=nosummary", + "Backend communication error", "Did not find summary for hit with document id"); + checkError(searcher, "/?streaming.userid=1&query=nosummarytofill", + "Timed out", "Missing hit summary data for 1 hits"); + + checkSearch(searcher, "/?streaming.userid=1&query=oneuserhit", 1, USERDOC_ID_PREFIX); + checkSearch(searcher, "/?streaming.userid=1&query=oneuserhit&sorting=%2Bsurname", 1, USERDOC_ID_PREFIX); + checkSearch(searcher, "/?streaming.selection=id.user%3D%3d1&query=twouserhits", 2, USERDOC_ID_PREFIX); + checkSearch(searcher, "/?streaming.groupname=group1&query=twogrouphitsandoneuserhit", 2, GROUPDOC_ID_PREFIX); + + checkGrouping(searcher, "/?streaming.selection=true&query=onegroupinghit", 1); + } + + @Test + public void testTrivialitiesToIncreaseCoverage() { + VdsStreamingSearcher searcher = new VdsStreamingSearcher(); + + assertNull(searcher.getSearchClusterConfigId()); + String searchClusterConfigId = "searchClusterConfigId"; + searcher.setSearchClusterConfigId(searchClusterConfigId); + assertEquals(searchClusterConfigId, searcher.getSearchClusterConfigId()); + + assertNull(searcher.getStorageClusterRouteSpec()); + String storageClusterRouteSpec = "storageClusterRouteSpec"; + searcher.setStorageClusterRouteSpec(storageClusterRouteSpec); + assertEquals(storageClusterRouteSpec, searcher.getStorageClusterRouteSpec()); + + Pong pong = searcher.ping(new Ping(), new Execution(new Execution.Context(null, null, null, null, null))); + assertEquals(0, pong.getErrorSize()); + } + + @Test + public void testVerifyDocId() { + Query generalQuery = new Query("/?streaming.selection=true&query=test"); + Query user1Query = new Query("/?streaming.userid=1&query=test"); + Query group1Query = new Query("/?streaming.groupname=group1&query=test"); + String userId1 = "userdoc:namespace:1:userspecific"; + String userId2 = "userdoc:namespace:2:userspecific"; + String groupId1 = "groupdoc:namespace:group1:userspecific"; + String groupId2 = "groupdoc:namespace:group2:userspecific"; + String orderIdGroup1 = "orderdoc(3,1):storage_test:group1:0:userspecific"; + String orderIdGroup2 = "orderdoc(5,2):storage_test:group2:0:userspecific"; + String orderIdUser1 = "orderdoc(3,1):storage_test:1:0:userspecific"; + String orderIdUser2 = "orderdoc(5,2):storage_test:2:0:userspecific"; + String badId = "unknowscheme:namespace:something"; + + assertTrue(VdsStreamingSearcher.verifyDocId(userId1, generalQuery, true)); + + assertTrue(VdsStreamingSearcher.verifyDocId(userId1, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(userId2, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(groupId1, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(groupId2, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(orderIdGroup1, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(orderIdGroup2, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(orderIdUser1, generalQuery, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(orderIdUser2, generalQuery, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(badId, generalQuery, false)); + + assertTrue(VdsStreamingSearcher.verifyDocId(userId1, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(userId2, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(groupId1, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(groupId2, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(orderIdGroup1, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(orderIdGroup2, user1Query, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(orderIdUser1, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(orderIdUser2, user1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(badId, user1Query, false)); + + assertFalse(VdsStreamingSearcher.verifyDocId(userId1, group1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(userId2, group1Query, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(groupId1, group1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(groupId2, group1Query, false)); + assertTrue(VdsStreamingSearcher.verifyDocId(orderIdGroup1, group1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(orderIdGroup2, group1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(orderIdUser1, group1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(orderIdUser2, group1Query, false)); + assertFalse(VdsStreamingSearcher.verifyDocId(badId, group1Query, false)); + } +} diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java new file mode 100644 index 00000000000..8c6322256fc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -0,0 +1,560 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; +import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; +import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.prelude.fastsearch.TimeoutException; +import com.yahoo.search.Query; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.text.Utf8String; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vespa.objects.BufferSerializer; +import org.junit.Test; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author Ulf Carlin + */ +public class VdsVisitorTestCase { + private LoadTypeSet loadTypeSet = new LoadTypeSet(); + + public VdsVisitorTestCase() { + loadTypeSet.addLoadType(1, "low", DocumentProtocol.Priority.LOW_1); + loadTypeSet.addLoadType(2, "normal", DocumentProtocol.Priority.NORMAL_1); + } + + private SearchResult createSR(String docId, double rank) { + BufferSerializer serializer = new BufferSerializer(); + serializer.putInt(null, 2); // total hits + serializer.putInt(null, 1); // hit count + serializer.put(null, docId); + serializer.putDouble(null, rank); + serializer.putInt(null, 0); // sort blob count + serializer.putInt(null, 0); // aggregator count + serializer.putInt(null, 0); // grouping count + serializer.getBuf().flip(); + return new SearchResult(serializer); + } + + private DocumentSummary createDS(String docId) { + BufferSerializer serializer = new BufferSerializer(); + serializer.putInt(null, 0); // old seq id + serializer.putInt(null, 1); // summary count + serializer.put(null, docId); + serializer.putInt(null, 1); // summary size + serializer.putInt(null, 0); // summary buffer + serializer.getBuf().flip(); + return new DocumentSummary(serializer); + } + + private QueryResultMessage createQRM(String docId, double rank) { + QueryResultMessage qrm = new QueryResultMessage(); + qrm.setSearchResult(createSR(docId, rank)); + qrm.setSummary(createDS(docId)); + return qrm; + } + + private SearchResultMessage createSRM(String docId, double rank) { + SearchResultMessage srm = new SearchResultMessage(); + srm.setSearchResult(createSR(docId, rank)); + return srm; + } + + private DocumentSummaryMessage createDSM(String docId) { + DocumentSummaryMessage dsm = new DocumentSummaryMessage(); + dsm.setDocumentSummary(createDS(docId)); + return dsm; + } + + private Message createM() { + return new Message() { + @Override + public Utf8String getProtocol() { + return null; + } + + @Override + public int getType() { + return 0; + } + }; + } + + private class QueryArguments { + // General query parameters + String query = "test"; + long timeout = 5; + int offset = 0; + int hits = 10; + int traceLevel = 0; + String summary = null; + String profile = null; + String location = null; // "pos.ll=N37.416383;W122.024683" requires PosSearcher? + String sortSpec = null; + String rankProperties = null; + + // Streaming query parameters + String userId = null; + String groupName = null; + String selection = null; + boolean headersOnly = false; + long from = 0; + long to = 0; + String loadTypeName = null; + DocumentProtocol.Priority priority = null; + String ordering = null; + int maxBucketsPerVisitor = 0; + + // Parameters in query object + boolean defineGrouping = false; // "select=all(group(customer) each(output(count())))" requires GroupingQueryParser? + + void setNonDefaults() { + query = "newquery"; + timeout = 10; + offset = 1; + hits = 1; + traceLevel = 100; + summary = "fancysummary"; + profile = "fancyprofile"; + location = "(1,10000,2000,0,1,0)"; + sortSpec = "+surname -yearofbirth"; + rankProperties = "rankfeature.something=2"; + + userId = "1234"; + groupName = null; + selection = null; + headersOnly = true; + from = 123; + to = 456; + loadTypeName = "low"; + priority = DocumentProtocol.Priority.HIGH_2; + ordering = "-"; + maxBucketsPerVisitor = 2; + + defineGrouping = true; + } + } + + private Query buildQuery(QueryArguments qa) throws Exception { + StringBuilder queryString = new StringBuilder(); + queryString.append("/?query=").append(qa.query); + if (qa.timeout != 5) { + queryString.append("&timeout=").append(qa.timeout); + } + if (qa.offset != 0) { + queryString.append("&offset=").append(qa.offset); + } + if (qa.hits != 10) { + queryString.append("&hits=").append(qa.hits); + } + if (qa.traceLevel != 0) { + queryString.append("&tracelevel=").append(qa.traceLevel); + } + if (qa.summary != null) { + queryString.append("&summary=").append(qa.summary); + } + if (qa.profile != null) { + queryString.append("&ranking.profile=").append(qa.profile); + } + if (qa.location != null) { + queryString.append("&location=").append(qa.location); + } + if (qa.sortSpec != null) { + queryString.append("&sorting=").append(URLEncoder.encode(qa.sortSpec, "UTF-8")); + } + if (qa.rankProperties != null) { + queryString.append("&").append(qa.rankProperties); + } + + if (qa.userId != null) { + queryString.append("&streaming.userid=").append(qa.userId); + } + if (qa.groupName != null) { + queryString.append("&streaming.groupname=").append(qa.groupName); + } + if (qa.selection != null) { + queryString.append("&streaming.selection=").append(URLEncoder.encode(qa.selection, "UTF-8")); + } + if (qa.headersOnly) { + queryString.append("&streaming.headersonly=").append(qa.headersOnly); + } + if (qa.from != 0) { + queryString.append("&streaming.fromtimestamp=").append(qa.from); + } + if (qa.to != 0) { + queryString.append("&streaming.totimestamp=").append(qa.to); + } + if (qa.loadTypeName != null) { + queryString.append("&streaming.loadtype=").append(qa.loadTypeName); + } + if (qa.priority != null) { + queryString.append("&streaming.priority=").append(qa.priority); + } + if (qa.ordering != null) { + queryString.append("&streaming.ordering=").append(URLEncoder.encode(qa.ordering, "UTF-8")); + } + if (qa.maxBucketsPerVisitor != 0) { + queryString.append("&streaming.maxbucketspervisitor=").append(qa.maxBucketsPerVisitor); + } + //System.out.println("query string="+queryString.toString()); + + Query query = new Query(queryString.toString()); + if (qa.defineGrouping) { + List groupingList = new ArrayList<>(); + groupingList.add(new Grouping()); + query.properties().set(GroupingExecutor.class.getName() + ".GroupingList", groupingList); + } + return query; + } + + private void verifyVisitorParameters(VisitorParameters params, QueryArguments qa, String searchCluster, Route route) { + //System.out.println("params="+params); + // Verify parameters based on properties + if (qa.userId != null) { + assertEquals("id.user=="+qa.userId, params.getDocumentSelection()); + } else if (qa.groupName != null) { + assertEquals("id.group==\""+qa.groupName+"\"", params.getDocumentSelection()); + } else if (qa.selection != null) { + assertEquals(qa.selection, params.getDocumentSelection()); + } else { + assertEquals("", params.getDocumentSelection()); + } + assertEquals(qa.headersOnly, params.getVisitHeadersOnly()); + assertEquals(qa.from, params.getFromTimestamp()); + assertEquals(qa.to, params.getToTimestamp()); + if (qa.loadTypeName != null && loadTypeSet.getNameMap().get(qa.loadTypeName) != null) { + LoadType expectedLoadType = loadTypeSet.getNameMap().get(qa.loadTypeName); + assertEquals(expectedLoadType, params.getLoadType()); + if (qa.priority != null) { + assertEquals(qa.priority, params.getPriority()); + } else { + assertEquals(expectedLoadType.getPriority(), params.getPriority()); + } + } else { + assertEquals(LoadType.DEFAULT, params.getLoadType()); + if (qa.priority != null) { + assertEquals(qa.priority, params.getPriority()); + } else { + assertEquals(DocumentProtocol.Priority.VERY_HIGH, params.getPriority()); + } + } + if (qa.ordering != null) { + assertEquals(VdsVisitor.getOrdering(qa.ordering), params.getVisitorOrdering()); + assertEquals(qa.offset+qa.hits, params.getMaxFirstPassHits()); + if (qa.maxBucketsPerVisitor != 0) { + assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); + } else { + assertEquals(1, params.getMaxBucketsPerVisitor()); + } + assertEquals(true, params.getDynamicallyIncreaseMaxBucketsPerVisitor()); + } else { + assertEquals(0, params.getVisitorOrdering()); + assertEquals(-1, params.getMaxFirstPassHits()); + if (qa.maxBucketsPerVisitor != 0) { + assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); + } else { + assertEquals(Integer.MAX_VALUE, params.getMaxBucketsPerVisitor()); + } + assertEquals(false, params.getDynamicallyIncreaseMaxBucketsPerVisitor()); + } + + // Verify parameters based only on query + assertEquals(qa.timeout*1000, params.getTimeoutMs()); + assertEquals("searchvisitor", params.getVisitorLibrary()); + assertEquals(Integer.MAX_VALUE, params.getMaxPending()); + assertEquals(qa.traceLevel, params.getTraceLevel()); + + // Verify library parameters + //System.err.println("query="+new String(params.getLibraryParameters().get("query"))); + assertNotNull(params.getLibraryParameters().get("query")); // TODO: Check contents + //System.err.println("query="+new String(params.getLibraryParameters().get("querystackcount"))); + assertNotNull(params.getLibraryParameters().get("querystackcount")); // TODO: Check contents + assertEquals(searchCluster, new String(params.getLibraryParameters().get("searchcluster"))); + if (qa.summary != null) { + assertEquals(qa.summary, new String(params.getLibraryParameters().get("summaryclass"))); + } else { + assertEquals("default", new String(params.getLibraryParameters().get("summaryclass"))); + } + assertEquals(Integer.toString(qa.offset+qa.hits), new String(params.getLibraryParameters().get("summarycount"))); + if (qa.profile != null) { + assertEquals(qa.profile, new String(params.getLibraryParameters().get("rankprofile"))); + } else { + assertEquals("default", new String(params.getLibraryParameters().get("rankprofile"))); + } + //System.err.println("queryflags="+new String(params.getLibraryParameters().get("queryflags"))); + assertNotNull(params.getLibraryParameters().get("queryflags")); // TODO: Check contents + if (qa.location != null) { + assertEquals(qa.location, new String(params.getLibraryParameters().get("location"))); + } else { + assertNull(params.getLibraryParameters().get("location")); + } + if (qa.rankProperties != null) { + //System.err.println("rankProperties="+new String(params.getLibraryParameters().get("rankproperties"))); + assertNotNull(params.getLibraryParameters().get("rankproperties")); // TODO: Check contents + } else { + assertNull(params.getLibraryParameters().get("rankproperties")); + } + if (qa.defineGrouping) { + //System.err.println("aggregation="+new String(params.getLibraryParameters().get("aggregation"))); + assertNotNull(params.getLibraryParameters().get("aggregation")); // TODO: Check contents + } else { + assertNull(params.getLibraryParameters().get("aggregation")); + } + if (qa.sortSpec != null) { + assertEquals(qa.sortSpec, new String(params.getLibraryParameters().get("sort"))); + } else { + assertNull(params.getLibraryParameters().get("sort")); + } + + assertEquals(route, params.getRoute()); + } + + @Test + public void testGetQueryFlags() { + assertEquals(0x00028000, VdsVisitor.getQueryFlags(new Query("/?query=test"))); + assertEquals(0x00028080, VdsVisitor.getQueryFlags(new Query("/?query=test&hitcountestimate=true"))); + assertEquals(0x00068000, VdsVisitor.getQueryFlags(new Query("/?query=test&rankfeatures=true"))); + assertEquals(0x00068080, VdsVisitor.getQueryFlags(new Query("/?query=test&hitcountestimate=true&rankfeatures=true"))); + + Query query= new Query("/?query=test"); + assertEquals(0x00028000, VdsVisitor.getQueryFlags(query)); + query.setNoCache(true); + assertEquals(0x00038000, VdsVisitor.getQueryFlags(query)); + query.getRanking().setFreshness("now"); + assertEquals(0x0003a000, VdsVisitor.getQueryFlags(query)); + } + + @Test + public void testGetOrdering() { + assertEquals(OrderingSpecification.ASCENDING, VdsVisitor.getOrdering("+")); + assertEquals(OrderingSpecification.DESCENDING, VdsVisitor.getOrdering("-")); + try { + VdsVisitor.getOrdering("illegalValue"); + assertTrue("Method expected to throw RuntimeException", false); + } catch (RuntimeException e) { + assertEquals("Ordering must be on the format {+/-}", e.getMessage()); + } + } + + @Test + public void testBasics() throws Exception { + Route route = Route.parse("storageClusterRouteSpec"); + String searchCluster = "searchClusterConfigId"; + MockVisitorSessionFactory factory = new MockVisitorSessionFactory(loadTypeSet); + + // Default values and no selection + QueryArguments qa = new QueryArguments(); + verifyVisitorOk(factory, qa, route, searchCluster); + + // Groupdoc + qa.groupName = "group"; + qa.maxBucketsPerVisitor = 2; // default ordering, non-default maxBucketsPerVisitor + qa.loadTypeName = "normal"; // non-default loadTypeName, default priority + verifyVisitorOk(factory, qa, route, searchCluster); + + qa.loadTypeName = "unknown"; // unknown loadTypeName, default priority + verifyVisitorOk(factory, qa, route, searchCluster); + + qa.priority = DocumentProtocol.Priority.NORMAL_2; // unknown loadTypeName, non-default priority + verifyVisitorOk(factory, qa, route, searchCluster); + + // Orderdoc + qa.groupName = null; + qa.selection = "id.group==\"group1\" and id.order(3,1)>=0"; + qa.ordering = "+"; // non-default ordering, default maxBucketsPerVisitor + qa.maxBucketsPerVisitor = 0; + qa.loadTypeName = null; // default loadTypeName, non-default priority + verifyVisitorOk(factory, qa, route, searchCluster); + + // Userdoc and lots of non-default parameters + qa.setNonDefaults(); + verifyVisitorOk(factory, qa, route, searchCluster); + } + + @Test + public void testFailures() throws Exception { + Route route = Route.parse("storageClusterRouteSpec"); + String searchCluster = "searchClusterConfigId"; + MockVisitorSessionFactory factory = new MockVisitorSessionFactory(loadTypeSet); + + // Default values and no selection + QueryArguments qa = new QueryArguments(); + + factory.failQuery = true; + verifyVisitorFails(factory, qa, route, searchCluster); + + factory.failQuery = false; + factory.timeoutQuery = true; + verifyVisitorFails(factory, qa, route, searchCluster); + } + + private void verifyVisitorOk(MockVisitorSessionFactory factory, QueryArguments qa, Route route, String searchCluster) throws Exception { + VdsVisitor visitor = new VdsVisitor(buildQuery(qa), searchCluster, route, factory); + visitor.doSearch(); + verifyVisitorParameters(factory.getParams(), qa, searchCluster, route); + supplyResults(visitor); + verifyResults(qa, visitor); + } + + private void verifyVisitorFails(MockVisitorSessionFactory factory, QueryArguments qa, Route route, String searchCluster) throws Exception { + VdsVisitor visitor = new VdsVisitor(buildQuery(qa), searchCluster, route, factory); + try { + visitor.doSearch(); + assertTrue("Visitor did not fail", false); + } catch (TimeoutException te) { + assertTrue("Got TimeoutException unexpectedly", factory.timeoutQuery); + } catch (IllegalArgumentException iae) { + assertTrue("Got IllegalArgumentException unexpectedly", factory.failQuery); + } + } + + private void supplyResults(VdsVisitor visitor) { + AckToken ackToken = null; + visitor.onMessage(createQRM("doc:0", 0.3), ackToken); + visitor.onMessage(createSRM("doc:1", 1.0), ackToken); + visitor.onMessage(createSRM("doc:2", 0.5), ackToken); + visitor.onMessage(createDSM("doc:1"), ackToken); + visitor.onMessage(createDSM("doc:2"), ackToken); + try { + visitor.onMessage(createM(), ackToken); + assertTrue("Unsupported message did not cause exception", false); + } catch (UnsupportedOperationException uoe) { + assertTrue(uoe.getMessage().contains("VdsVisitor can only accept query result, search result, and documentsummary messages")); + } + } + + private void verifyResults(QueryArguments qa, VdsVisitor visitor) { + assertEquals(6, visitor.getTotalHitCount()); + assertEquals(Math.min(3 - qa.offset, qa.hits), visitor.getHits().size()); + assertEquals(3, visitor.getSummaryMap().size()); + assertEquals(0, visitor.getGroupings().size()); + assertNull(visitor.getStatistics()); + + for (int i=0; i