diff options
Diffstat (limited to 'container-search')
177 files changed, 651 insertions, 13111 deletions
diff --git a/container-search/CMakeLists.txt b/container-search/CMakeLists.txt index dcf6c3461d3..a6099fea63b 100644 --- a/container-search/CMakeLists.txt +++ b/container-search/CMakeLists.txt @@ -6,13 +6,10 @@ install_config_definition(src/main/resources/configdefinitions/federation.def se install_config_definition(src/main/resources/configdefinitions/fs4.def container.search.fs4.def) install_config_definition(src/main/resources/configdefinitions/index-info.def search.config.index-info.def) install_config_definition(src/main/resources/configdefinitions/keyvalue.def prelude.searcher.keyvalue.def) -install_config_definition(src/main/resources/configdefinitions/legacy-emulation.def container.search.legacy-emulation.def) install_config_definition(src/main/resources/configdefinitions/lowercasing.def search.querytransform.lowercasing.def) install_config_definition(src/main/resources/configdefinitions/measure-qps.def search.statistics.measure-qps.def) install_config_definition(src/main/resources/configdefinitions/page-templates.def search.pagetemplates.page-templates.def) install_config_definition(src/main/resources/configdefinitions/provider.def search.federation.provider.def) -install_config_definition(src/main/resources/configdefinitions/qr-binary-cache-region.def search.cache.qr-binary-cache-region.def) -install_config_definition(src/main/resources/configdefinitions/qr-binary-cache.def search.cache.qr-binary-cache.def) install_config_definition(src/main/resources/configdefinitions/qr-monitor.def prelude.cluster.qr-monitor.def) install_config_definition(src/main/resources/configdefinitions/qr-quotetable.def prelude.searcher.qr-quotetable.def) install_config_definition(src/main/resources/configdefinitions/qr-start.def search.config.qr-start.def) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 829c86711ef..e37836a5f06 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1653,347 +1653,6 @@ ], "fields": [] }, - "com.yahoo.prelude.templates.Context": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void setXmlEscape(boolean)", - "public boolean getXmlEscape()", - "public abstract java.lang.Object put(java.lang.String, java.lang.Object)", - "public abstract java.lang.Object get(java.lang.String)", - "public abstract java.lang.Object remove(java.lang.Object)", - "public void setBoldOpenTag(java.lang.String)", - "public void setBoldCloseTag(java.lang.String)", - "public void setSeparatorTag(java.lang.String)", - "protected java.lang.Object normalizeValue(java.lang.Object)", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public abstract java.util.Collection getKeys()", - "public boolean isUtf8Output()", - "public void setUtf8Output(boolean)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.DefaultTemplateSet": { - "superClass": "com.yahoo.prelude.templates.UserTemplate", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "protected void <init>(java.lang.String)", - "public void <init>()", - "public com.yahoo.text.XMLWriter wrapWriter(java.io.Writer)", - "public void header(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void footer(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void hit(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void hitFooter(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void error(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void noHits(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected static void renderCoverageAttributes(com.yahoo.search.result.Coverage, com.yahoo.text.XMLWriter)", - "protected void renderHitAttributes(com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderHitGroup(com.yahoo.search.result.HitGroup, com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderGroup(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderHitGroupOfTypeGroupHit(com.yahoo.prelude.templates.Context, com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderId(com.yahoo.net.URI, com.yahoo.text.XMLWriter)", - "protected void renderHitFields(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderField(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.util.Map$Entry, com.yahoo.text.XMLWriter)", - "protected void renderFieldContent(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.lang.String, com.yahoo.text.XMLWriter)", - "protected boolean shouldRenderField(com.yahoo.search.result.Hit, java.lang.String)", - "public bridge synthetic void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic java.io.Writer wrapWriter(java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.FormattingOptions": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void formatFieldAsAttribute(java.lang.String, java.lang.String)", - "public java.util.Set fieldsAsAttributes()", - "public java.lang.String getAttributeName(java.lang.String)", - "public void formatFieldWithSubtype(java.lang.String, java.lang.String, java.lang.String, java.lang.String)", - "public com.yahoo.prelude.templates.FormattingOptions$SubtypeField getSubtype(java.lang.String)", - "public void formatFieldWithSubtype(java.lang.String, java.lang.String, java.lang.String)", - "public com.yahoo.prelude.templates.FormattingOptions$SubtypeFieldWithPrefix getSubtypeWithPrefix(java.lang.String)", - "public boolean shouldRenderField(java.lang.String)", - "public void setFieldToRender(java.lang.String)", - "public void setFieldNotToRender(java.lang.String)" - ], - "fields": [ - "public static final java.lang.String DEFAULT_TYPE_ATTRIBUTE_NAME" - ] - }, - "com.yahoo.prelude.templates.GenericTemplateSet": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String getName()", - "public java.lang.String getMimeType()", - "public java.lang.String getEncoding()", - "public final java.lang.String getRequestedEncoding(com.yahoo.search.Query)", - "public com.yahoo.prelude.templates.Template getTemplate(java.lang.String)", - "public void setTemplate(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void setTemplateNotNull(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void setHighlightTags(java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public void setSummaryClass(java.lang.String)", - "public java.lang.String getSummaryClass()" - ], - "fields": [ - "public static final java.lang.String DEFAULT_MIMETYPE", - "public static final java.lang.String DEFAULT_ENCODING" - ] - }, - "com.yahoo.prelude.templates.HitContext": { - "superClass": "com.yahoo.prelude.templates.Context", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.search.result.Hit, com.yahoo.prelude.templates.Context)", - "public java.lang.Object put(java.lang.String, java.lang.Object)", - "public java.lang.Object get(java.lang.String)", - "public java.lang.Object remove(java.lang.Object)", - "public java.util.Collection getKeys()", - "public void setBoldOpenTag(java.lang.String)", - "public void setBoldCloseTag(java.lang.String)", - "public void setSeparatorTag(java.lang.String)", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public boolean isUtf8Output()", - "public void setUtf8Output(boolean)", - "public void setXmlEscape(boolean)", - "public boolean getXmlEscape()", - "protected java.lang.Object normalizeValue(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.LogExceptionUserTemplateDelegator": { - "superClass": "com.yahoo.prelude.templates.UserTemplate", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.prelude.templates.UserTemplate)", - "public com.yahoo.prelude.templates.Context createContext()", - "public java.io.Writer wrapWriter(java.io.Writer)", - "public boolean isDefaultTemplateSet()", - "public java.lang.String getSummaryClass()", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public void setSummaryClass(java.lang.String)", - "public void setHighlightTags(java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String getName()", - "public java.lang.String getMimeType()", - "public java.lang.String getEncoding()", - "public com.yahoo.prelude.templates.Template getTemplate(java.lang.String)", - "public void setTemplate(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void setTemplateNotNull(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void queryContext(com.yahoo.prelude.templates.Context, java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.MapContext": { - "superClass": "com.yahoo.prelude.templates.Context", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public java.lang.Object get(java.lang.String)", - "public java.lang.Object put(java.lang.String, java.lang.Object)", - "public java.lang.Object remove(java.lang.Object)", - "public java.util.Collection getKeys()" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.PageTemplateSet": { - "superClass": "com.yahoo.prelude.templates.TiledTemplateSet", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.text.XMLWriter wrapWriter(java.io.Writer)", - "public void header(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void footer(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderSection(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "public void hit(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void hitFooter(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public java.lang.String toString()", - "public bridge synthetic void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic java.io.Writer wrapWriter(java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.SearchRendererAdaptor": { - "superClass": "com.yahoo.search.rendering.Renderer", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.prelude.templates.UserTemplate)", - "public void init()", - "public static void callRender(java.io.OutputStream, com.yahoo.search.Result)", - "public java.lang.String getEncoding()", - "public java.lang.String getMimeType()", - "public java.lang.String getDefaultSummaryClass()", - "public void render(java.io.Writer, com.yahoo.search.Result)", - "public java.io.Writer wrapWriter(java.io.Writer)", - "public void beginResult(java.io.Writer, com.yahoo.search.Result)", - "public void endResult(java.io.Writer, com.yahoo.search.Result)", - "public void error(java.io.Writer, java.util.Collection)", - "public void emptyResult(java.io.Writer, com.yahoo.search.Result)", - "public void queryContext(java.io.Writer, com.yahoo.search.query.context.QueryContext)", - "public void renderHit(java.io.Writer, com.yahoo.search.result.Hit, int)", - "public static boolean simpleRenderDefaultErrorHit(java.io.Writer, com.yahoo.search.result.ErrorHit)", - "public static void renderMessageDefaultErrorHit(com.yahoo.text.XMLWriter, com.yahoo.processing.request.ErrorMessage)", - "public boolean simpleRenderErrorHit(java.io.Writer, com.yahoo.search.result.ErrorHit)", - "public static void rendererErrorHitMessageMessage(com.yahoo.text.XMLWriter, com.yahoo.search.result.ErrorHit, com.yahoo.processing.request.ErrorMessage)", - "public com.yahoo.prelude.templates.UserTemplate getAdaptee()" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.Template": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>()", - "public abstract void render(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract java.lang.String getName()" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.TemplateSet": { - "superClass": "com.yahoo.prelude.templates.UserTemplate", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public static com.yahoo.prelude.templates.UserTemplate getDefault()", - "public com.yahoo.prelude.templates.Template getHeader(com.yahoo.search.Result)", - "public void setHeader(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getFooter(com.yahoo.search.Result)", - "public void setFooter(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getNohits(com.yahoo.search.Result)", - "public com.yahoo.prelude.templates.Template getQueryContext(com.yahoo.search.Result)", - "public void setQueryContext(com.yahoo.prelude.templates.Template)", - "public void setNohits(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getError(com.yahoo.search.Result)", - "public void setError(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getHit(com.yahoo.search.result.Hit)", - "public void setHit(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getHitFooter(com.yahoo.search.result.Hit)", - "public java.lang.String toString()", - "public void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void queryContext(com.yahoo.prelude.templates.Context, java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.TiledTemplateSet": { - "superClass": "com.yahoo.prelude.templates.DefaultTemplateSet", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String)", - "public com.yahoo.text.XMLWriter wrapWriter(java.io.Writer)", - "public void header(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderHitAttributes(com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderField(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.util.Map$Entry, com.yahoo.text.XMLWriter)", - "protected void renderHitFields(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected boolean shouldRenderField(com.yahoo.search.result.Hit, java.lang.String)", - "public void hit(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderHitGroup(com.yahoo.search.result.HitGroup, com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderSection(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderSectionContent(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderProvider(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderProviderHitAttributes(com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderProviderHitFields(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderProviderField(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.util.Map$Entry, com.yahoo.text.XMLWriter)", - "public java.lang.String toString()", - "public bridge synthetic void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic java.io.Writer wrapWriter(java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.UserTemplate": { - "superClass": "com.yahoo.prelude.templates.GenericTemplateSet", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public void <init>(java.lang.String)", - "public java.io.Writer wrapWriter(java.io.Writer)", - "public com.yahoo.prelude.templates.Context createContext()", - "public boolean isDefaultTemplateSet()", - "public abstract void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void queryContext(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public static void dumpAndXMLQuoteUTF8(com.yahoo.io.ByteWriter, byte[])", - "public static boolean dumpBytes(com.yahoo.io.ByteWriter, com.yahoo.prelude.fastsearch.FastHit, java.lang.String)" - ], - "fields": [] - }, "com.yahoo.search.Query$Type": { "superClass": "java.lang.Enum", "interfaces": [], @@ -2044,7 +1703,6 @@ "public long getStartTime()", "public long getDurationTime()", "public long getTimeLeft()", - "public boolean requestHasProperty(java.lang.String)", "public long getTimeout()", "public void setTimeout(long)", "public void setTimeout(java.lang.String)", @@ -2061,8 +1719,6 @@ "public void setHits(int)", "public void setOffset(int)", "public void setWindow(int, int)", - "public void setCompress(boolean)", - "public boolean getCompress()", "public java.lang.String toString()", "public java.lang.String toDetailString()", "public int encode(java.nio.ByteBuffer)", @@ -2081,7 +1737,6 @@ "public com.yahoo.search.query.Ranking getRanking()", "public com.yahoo.search.query.Model getModel()", "public com.yahoo.container.jdisc.HttpRequest getHttpRequest()", - "public com.yahoo.search.query.SessionId getSessionId(boolean)", "public com.yahoo.search.query.SessionId getSessionId()", "public com.yahoo.search.query.SessionId getSessionId(java.lang.String)", "public boolean hasEncodableProperties()", @@ -2146,7 +1801,6 @@ "public void setCoverage(com.yahoo.search.result.Coverage)", "public com.yahoo.search.result.Coverage getCoverage(boolean)", "public com.yahoo.collections.ListMap getHeaders(boolean)", - "public com.yahoo.search.result.Templating getTemplating()", "public bridge synthetic com.yahoo.component.provider.ListenableFreezableClass clone()", "public bridge synthetic com.yahoo.component.provider.FreezableClass clone()", "public bridge synthetic java.lang.Object clone()" @@ -2209,7 +1863,6 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.cluster.NodeManager, java.lang.String)", "public void <init>(com.yahoo.search.cluster.NodeManager)", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", "public void add(java.lang.Object, boolean)", @@ -2445,10 +2098,8 @@ "public java.util.TimeZone getTimeZone()", "public com.yahoo.search.grouping.GroupingRequest setTimeZone(java.util.TimeZone)", "public com.yahoo.search.grouping.result.RootGroup getResultGroup(com.yahoo.search.Result)", - "public com.yahoo.search.grouping.GroupingRequest setResultGroup(com.yahoo.search.grouping.result.RootGroup)", "public java.util.List continuations()", "public static com.yahoo.search.grouping.GroupingRequest newInstance(com.yahoo.search.Query)", - "public static java.util.List getRequests(com.yahoo.search.Query)", "public java.lang.String toString()" ], "fields": [] @@ -4138,19 +3789,6 @@ ], "fields": [] }, - "com.yahoo.search.grouping.request.YmumValue": { - "superClass": "com.yahoo.search.grouping.request.DocumentValue", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.search.grouping.request.YmumValue copy()", - "public bridge synthetic com.yahoo.search.grouping.request.GroupingExpression copy()" - ], - "fields": [] - }, "com.yahoo.search.grouping.request.ZCurveXFunction": { "superClass": "com.yahoo.search.grouping.request.FunctionNode", "interfaces": [], @@ -4449,8 +4087,6 @@ ], "methods": [ "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.container.core.ContainerHttpConfig)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.container.core.QrTemplatesConfig, com.yahoo.container.protect.FreezeDetector, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry)", "protected void destroy()", "public final com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)", "public com.yahoo.search.Result searchAndFill(com.yahoo.search.Query, com.yahoo.component.chain.Chain, com.yahoo.search.searchchain.SearchChainRegistry)", @@ -5249,7 +4885,6 @@ "methods": [ "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", "public void <init>(com.yahoo.search.Query)", - "public void traceLanguage()", "public com.yahoo.language.Language getParsingLanguage()", "public com.yahoo.language.Language getParsingLanguage(java.lang.String)", "public com.yahoo.language.Language getLanguage()", @@ -5289,9 +4924,7 @@ "public com.yahoo.search.searchchain.Execution getExecution()", "public static com.yahoo.search.query.Model getFrom(com.yahoo.search.Query)", "public java.lang.String toString()", - "public void prepare(com.yahoo.search.query.Ranking)", - "public void setYTraceHeaderToNext(java.lang.String)", - "public java.lang.String getYTraceHeaderToNext()" + "public void prepare(com.yahoo.search.query.Ranking)" ], "fields": [ "public static final java.lang.String MODEL", @@ -5337,8 +4970,6 @@ "public void setSummary(java.lang.String)", "public boolean getBolding()", "public void setBolding(boolean)", - "public boolean getReportCoverage()", - "public void setReportCoverage(boolean)", "public com.yahoo.component.ComponentSpecification getRenderer()", "public void setRenderer(com.yahoo.component.ComponentSpecification)", "public java.lang.String getFormat()", @@ -5357,7 +4988,6 @@ "public static final java.lang.String BOLDING", "public static final java.lang.String TIMING", "public static final java.lang.String SUMMARY", - "public static final java.lang.String REPORT_COVERAGE", "public static final java.lang.String SUMMARY_FIELDS", "public static final java.lang.String FORMAT" ] @@ -5694,7 +5324,6 @@ ], "methods": [ "public java.lang.String toString()", - "public static com.yahoo.search.query.UniqueRequestId next()", "public static com.yahoo.search.query.UniqueRequestId next(java.lang.String)" ], "fields": [] @@ -6587,9 +6216,7 @@ "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public final com.yahoo.search.Query getParentQuery()" ], - "fields": [ - "public static final com.yahoo.processing.request.CompoundName[] PER_SOURCE_QUERY_PROPERTIES" - ] + "fields": [] }, "com.yahoo.search.query.properties.QueryPropertyAliases": { "superClass": "com.yahoo.search.query.properties.PropertyAliases", @@ -6830,7 +6457,7 @@ "public static final java.lang.String ENABLE", "public static final java.lang.String FACTOR", "public static final java.lang.String TAILCOST", - "public java.lang.Boolean enable" + "public static final com.yahoo.processing.request.CompoundName enableProperty" ] }, "com.yahoo.search.query.rewrite.QueryRewriteSearcher": { @@ -7110,50 +6737,6 @@ ], "fields": [] }, - "com.yahoo.search.rendering.DefaultRenderer$RenderingVisitor": { - "superClass": "com.yahoo.yolean.trace.TraceVisitor", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.text.XMLWriter, long)", - "public void entering(com.yahoo.yolean.trace.TraceNode)", - "public void leaving(com.yahoo.yolean.trace.TraceNode)", - "public void visit(com.yahoo.yolean.trace.TraceNode)" - ], - "fields": [] - }, - "com.yahoo.search.rendering.DefaultRenderer": { - "superClass": "com.yahoo.processing.rendering.AsynchronousSectionedRenderer", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.util.concurrent.Executor)", - "public void init()", - "public java.lang.String getEncoding()", - "public java.lang.String getMimeType()", - "public void error(com.yahoo.text.XMLWriter, com.yahoo.search.Result)", - "public void queryContext(com.yahoo.text.XMLWriter, com.yahoo.search.query.context.QueryContext, com.yahoo.search.Query)", - "public static boolean simpleRenderDefaultErrorHit(com.yahoo.text.XMLWriter, com.yahoo.search.result.ErrorHit)", - "public static void renderMessageDefaultErrorHit(com.yahoo.text.XMLWriter, com.yahoo.search.result.ErrorMessage)", - "public void beginResponse(java.io.OutputStream)", - "public final java.lang.String getRequestedEncoding(com.yahoo.search.Query)", - "public void beginList(com.yahoo.processing.response.DataList)", - "public void data(com.yahoo.processing.response.Data)", - "public void endList(com.yahoo.processing.response.DataList)", - "public void endResponse()" - ], - "fields": [ - "public static final java.lang.String DEFAULT_MIMETYPE", - "public static final java.lang.String DEFAULT_ENCODING" - ] - }, "com.yahoo.search.rendering.JsonRenderer": { "superClass": "com.yahoo.processing.rendering.AsynchronousSectionedRenderer", "interfaces": [], @@ -7216,6 +6799,7 @@ ], "fields": [ "public static final com.yahoo.component.ComponentId xmlRendererId", + "public static final com.yahoo.component.ComponentId pageRendererId", "public static final com.yahoo.component.ComponentId jsonRendererId", "public static final com.yahoo.component.ComponentId defaultRendererId" ] @@ -7361,8 +6945,6 @@ "public void <init>(long, long)", "public void <init>(long, long, int)", "public void <init>(long, long, int, int)", - "public void <init>(long, int, boolean)", - "public void <init>(long, int, boolean, int)", "public com.yahoo.search.result.Coverage setSoonActive(long)", "public com.yahoo.search.result.Coverage setDegradedReason(int)", "public com.yahoo.search.result.Coverage setNodesTried(int)", @@ -7400,7 +6982,6 @@ "public void <init>(java.lang.String, com.yahoo.search.result.ErrorMessage)", "public void <init>(java.lang.String, java.util.List)", "public void setSource(java.lang.String)", - "public com.yahoo.search.result.ErrorMessage getMainError()", "public void addError(com.yahoo.search.result.ErrorMessage)", "public void addErrors(com.yahoo.search.result.ErrorHit)", "public java.util.Iterator errorIterator()", @@ -7428,7 +7009,6 @@ ], "methods": [ "public abstract void setSource(java.lang.String)", - "public abstract com.yahoo.search.result.ErrorMessage getMainError()", "public abstract void addError(com.yahoo.search.result.ErrorMessage)", "public abstract void addErrors(com.yahoo.search.result.ErrorHit)", "public abstract java.util.Iterator errorIterator()", @@ -7587,30 +7167,17 @@ "public java.lang.Object removeField(java.lang.String)", "protected boolean hasField(java.lang.String)", "protected boolean hasFields()", - "public void changeFieldKey(java.lang.String, java.lang.String)", - "public com.yahoo.prelude.hitfield.HitField buildHitField(java.lang.String)", "public com.yahoo.prelude.hitfield.HitField buildHitField(java.lang.String, boolean)", - "public com.yahoo.prelude.hitfield.HitField buildHitField(java.lang.String, boolean, boolean)", "public java.util.Set types()", - "public java.lang.String getTypeString()", - "public int getAddNumber()", - "public void setAddNumber(int)", "public boolean isMeta()", "public void setMeta(boolean)", "public boolean isAuxiliary()", "public void setAuxiliary(boolean)", - "public int getSourceNumber()", - "public void setSourceNumber(int)", "public com.yahoo.search.Query getQuery()", "public com.yahoo.processing.Request request()", "public final void setQuery(com.yahoo.search.Query)", - "public java.lang.String getFieldXML(java.lang.String)", - "public java.lang.String getUnboldedField(java.lang.String, boolean)", "public void setSearcherSpecificMetaData(com.yahoo.search.Searcher, java.lang.Object)", "public java.lang.Object getSearcherSpecificMetaData(com.yahoo.search.Searcher)", - "protected final void setFilledInternal(java.util.Set)", - "protected final java.util.Set getFilledInternal()", - "public static java.lang.String stripCharacter(char, java.lang.String)", "protected void close()", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -7662,7 +7229,6 @@ "public com.yahoo.search.result.Hit remove(java.lang.String)", "public com.yahoo.search.result.Hit remove(com.yahoo.net.URI)", "public com.yahoo.search.result.Hit remove(int)", - "public void setError(com.yahoo.search.result.ErrorMessage)", "public void addError(com.yahoo.search.result.ErrorMessage)", "public com.yahoo.search.result.ErrorHit getErrorHit()", "public com.yahoo.search.result.DefaultErrorHit removeErrorHit()", @@ -7853,31 +7419,6 @@ ], "fields": [] }, - "com.yahoo.search.result.Templating": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.search.Result)", - "public int getFirstHitNo()", - "public long getNextFirstHitNo()", - "public long getNextLastHitNo()", - "public int getLastHitNo()", - "public int getPrevFirstHitNo()", - "public int getPrevLastHitNo()", - "public java.lang.String getNextResultURL()", - "public java.lang.String getPreviousResultURL()", - "public java.lang.String getCurrentResultURL()", - "public com.yahoo.prelude.templates.UserTemplate getTemplates()", - "public void setTemplates(com.yahoo.prelude.templates.UserTemplate)", - "public com.yahoo.processing.rendering.Renderer getRenderer()", - "public void setRenderer(com.yahoo.processing.rendering.Renderer)", - "public boolean usesDefaultTemplate()" - ], - "fields": [] - }, "com.yahoo.search.searchchain.AsyncExecution": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/container-search/pom.xml b/container-search/pom.xml index 13e05bb1e81..35b1477f478 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> @@ -21,6 +21,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-library</artifactId> <scope>test</scope> @@ -107,14 +112,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </dependency> diff --git a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java index a3b8e4f29a2..7dabb52a233 100644 --- a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java @@ -76,7 +76,7 @@ public class GetDocSumsPacket extends Packet { boolean useQueryCache = query.getRanking().getQueryCache(); // If feature cache is used we need to include the sessionId as key. if (useQueryCache) { // TODO: Move this decision (and the key) to ranking - query.getRanking().getProperties().put(sessionIdKey, query.getSessionId(false).toString()); + query.getRanking().getProperties().put(sessionIdKey, query.getSessionId().toString()); } // set the default features diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 109ecfd29f8..8433939090b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -53,7 +53,6 @@ public class Index { private boolean uriIndex = false; private boolean hostIndex = false; private StemMode stemMode = StemMode.NONE; - private Attribute[] matchGroup = null; private boolean isAttribute = false; private boolean isDefaultPosition = false; private boolean dynamicSummary=false; @@ -165,8 +164,6 @@ public class Index { setAttribute(true); } else if (commandString.equals("default-position")) { setDefaultPosition(true); - } else if (commandString.startsWith("match-group ")) { - setMatchGroup(commandString.substring(12).split(" ")); } else if (commandString.equals("plain-tokens")) { setPlainTokens(true); } else if (commandString.equals("multivalue")) { @@ -269,19 +266,6 @@ public class Index { return "(null)".equals(name); } - public Attribute[] getMatchGroup() { // TODO: Not in use on Vespa 6 - return matchGroup; - } - - public void setMatchGroup(String[] attributes) { - Attribute[] a = new Attribute[attributes.length]; - - for (int i = 0; i < attributes.length; i++) { - a[i] = new Attribute(attributes[i].trim()); - } - this.matchGroup = a; - } - public boolean isAttribute() { return isAttribute; } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 448d8e7855f..2fcd2466dd8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -44,12 +44,8 @@ public class IndexFacts { /** * The name of the default search definition, which is the union of all * known document types. - * - * @deprecated do not use */ - // TODO: Make this package private in Vespa 7 - @Deprecated // OK - public static final String unionName = "unionOfAllKnown"; + static final String unionName = "unionOfAllKnown"; /** A search definition which contains the union of all settings. */ @SuppressWarnings("deprecation") @@ -106,31 +102,6 @@ public class IndexFacts { return clusters != null ? clusters : Collections.<String>emptyList(); } - /** - * Public only for testing. - * - * @deprecated set at creation time - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setClusters(Map<String, List<String>> clusters) { - ensureNotFrozen(); - this.clusters = clusters; - clusterByDocument = invert(clusters); - } - - /** - * @deprecated set indexes at creation time instead - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setSearchDefinitions(Map<String, SearchDefinition> searchDefinitions, - SearchDefinition unionSearchDefinition) { - ensureNotFrozen(); - this.searchDefinitions = searchDefinitions; - this.unionSearchDefinition = unionSearchDefinition; - } - private boolean isInitialized() { return searchDefinitions.size() > 0; } @@ -312,58 +283,6 @@ public class IndexFacts { } } - - /** - * Add a string to be accepted as an index name when parsing a - * query. - * - * For testing only. - * - * @param sdName name of search definition containing index, if null, modify default set - * @param indexName name of index, actual or otherwise - * @deprecated set indexes at creation time instead - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void addIndex(String sdName, String indexName) { - ensureNotFrozen(); - - SearchDefinition sd; - if (sdName == null) { - sd = unionSearchDefinition; - } else if (searchDefinitions.containsKey(sdName)) { - sd = searchDefinitions.get(sdName); - } else { - sd = new SearchDefinition(sdName); - searchDefinitions.put(sdName, sd); - } - sd.getOrCreateIndex(indexName); - unionSearchDefinition.getOrCreateIndex(indexName); - } - - /** - * Adds an index to the specified index, and the default index settings, - * overriding any current settings for this index - * @deprecated set indexes at creation time instead - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void addIndex(String sdName, Index index) { - ensureNotFrozen(); - - SearchDefinition sd; - if (sdName == null) { - sd = unionSearchDefinition; - } else if (searchDefinitions.containsKey(sdName)) { - sd = searchDefinitions.get(sdName); - } else { - sd = new SearchDefinition(sdName); - searchDefinitions.put(sdName, sd); - } - sd.addIndex(index); - unionSearchDefinition.addIndex(index); - } - public String getDefaultPosition(String sdName) { SearchDefinition sd; if (sdName == null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index d2950078868..dd2bd1ee2f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -39,8 +39,10 @@ public final class IndexModel { /** * Use IndexModel as a pure wrapper for the parameters given. + * + * @deprecated use the constructor without the third parameter */ - // TODO: Deprecate on Vespa 7 and remove on Vespa 8 + @Deprecated // TODO: Remove Vespa 8 public IndexModel(Map<String, List<String>> masterClusters, Map<String, SearchDefinition> searchDefinitions, SearchDefinition unionSearchDefinition) { @@ -90,7 +92,6 @@ public final class IndexModel { IndexInfoConfig.Indexinfo.Command command = j.next(); sd.addCommand(command.indexname(),command.command()); } - sd.fillMatchGroups(); searchDefinitions.put(info.name(), sd); } @@ -127,7 +128,6 @@ public final class IndexModel { } } - union.fillMatchGroups(); return union; } @@ -135,7 +135,8 @@ public final class IndexModel { public Map<String, SearchDefinition> getSearchDefinitions() { return searchDefinitions; } - // TODO: Deprecate on Vespa 7 and make package scope on Vespa 8 + /** @deprecated do not use */ + @Deprecated // TODO: Remove on Vespa 8 public SearchDefinition getUnionSearchDefinition() { return unionSearchDefinition; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index ba3ff2eda00..cef64c293af 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -46,7 +46,7 @@ public class Pong { public int getErrorSize() { return errors.size(); } - + /** Returns the number of active documents in the backend responding in this Pong, if available */ public Optional<Long> activeDocuments() { if ( ! pongPacket.isPresent()) return Optional.empty(); diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java index aa76bd5912a..7859a9698d9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java @@ -96,19 +96,4 @@ public class SearchDefinition { } } - public void fillMatchGroups() { - for (Index i : indices.values()) { - Attribute[] matchGroup = i.getMatchGroup(); - if (matchGroup == null) { - continue; - } - for (Attribute a : matchGroup) { - Index m = getIndex(a.name); - if (m != null) { - a.setTokenizedContent(!m.isAttribute()); - } - } - } - } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index fd150f975c3..92b6eef906e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -11,7 +11,6 @@ import com.yahoo.concurrent.Receiver.MessageState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.fs4.mplex.Backend; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.net.HostName; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.prelude.fastsearch.FS4ResourcePool; @@ -100,7 +99,6 @@ public class ClusterSearcher extends Searcher { QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, DocumentdbInfoConfig documentDbConfig, - LegacyEmulationConfig emulationConfig, QrMonitorConfig monitorConfig, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, @@ -133,7 +131,7 @@ public class ClusterSearcher extends Searcher { SummaryParameters docSumParams = new SummaryParameters(qrsConfig .com().yahoo().prelude().fastsearch().FastSearcher().docsum() .defaultclass()); - + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { String docTypeName = docDb.name(); documentTypes.add(docTypeName); @@ -146,7 +144,7 @@ public class ClusterSearcher extends Searcher { if (searchClusterConfig.indexingmode() == STREAMING) { VdsStreamingSearcher searcher = vdsCluster(fs4ResourcePool.getServerId(), searchClusterIndex, - searchClusterConfig, cacheParams, emulationConfig, docSumParams, + searchClusterConfig, cacheParams, docSumParams, documentDbConfig); addBackendSearcher(searcher); } else { @@ -156,7 +154,7 @@ public class ClusterSearcher extends Searcher { if ( ! isRemote(searchClusterConfig.dispatcher(dispatcherIndex).host())) { Backend dispatchBackend = createBackend(searchClusterConfig.dispatcher(dispatcherIndex)); FastSearcher searcher = searchDispatch(searchClusterIndex, fs4ResourcePool, - cacheParams, emulationConfig, docSumParams, + cacheParams, docSumParams, documentDbConfig, dispatchBackend, dispatcher, dispatcherIndex); addBackendSearcher(searcher); } @@ -192,22 +190,21 @@ public class ClusterSearcher extends Searcher { } private static ClusterParams makeClusterParams(int searchclusterIndex, - LegacyEmulationConfig emulConfig, int dispatchIndex) { - return new ClusterParams("sc" + searchclusterIndex + ".num" + dispatchIndex, emulConfig); + return new ClusterParams("sc" + searchclusterIndex + ".num" + dispatchIndex); } private static FastSearcher searchDispatch(int searchclusterIndex, FS4ResourcePool fs4ResourcePool, CacheParams cacheParams, - LegacyEmulationConfig emulConfig, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig, Backend backend, Dispatcher dispatcher, int dispatcherIndex) { - ClusterParams clusterParams = makeClusterParams(searchclusterIndex, emulConfig, dispatcherIndex); - return new FastSearcher(backend, fs4ResourcePool, dispatcher, docSumParams, clusterParams, cacheParams, + ClusterParams clusterParams = makeClusterParams(searchclusterIndex, + dispatcherIndex); + return new FastSearcher(backend, fs4ResourcePool, dispatcher, docSumParams, clusterParams, cacheParams, documentdbInfoConfig); } @@ -215,14 +212,13 @@ public class ClusterSearcher extends Searcher { int searchclusterIndex, QrSearchersConfig.Searchcluster searchClusterConfig, CacheParams cacheParams, - LegacyEmulationConfig emulConfig, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig) { if (searchClusterConfig.searchdef().size() != 1) { throw new IllegalArgumentException("Search clusters in streaming search shall only contain a single searchdefinition : " + searchClusterConfig.searchdef()); } - ClusterParams clusterParams = makeClusterParams(searchclusterIndex, emulConfig, 0); + ClusterParams clusterParams = makeClusterParams(searchclusterIndex, 0); VdsStreamingSearcher searcher = (VdsStreamingSearcher) VespaBackEndSearcher .getSearcher("com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher"); searcher.setSearchClusterConfigId(searchClusterConfig.rankprofiles().configid()); @@ -319,7 +315,7 @@ public class ClusterSearcher extends Searcher { if (invalidInDocTypes != null && !invalidInDocTypes.isEmpty()) { String plural = invalidInDocTypes.size() > 1 ? "s" : ""; - return new Result(query, + return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested rank profile '" + rankProfile + "' is undefined for document type" + plural + " '" + StringUtils.join(invalidInDocTypes.iterator(), ", ") + "'")); @@ -384,7 +380,7 @@ public class ClusterSearcher extends Searcher { if (query.getTimeout() <= maxQueryTimeout) return; if (query.isTraceable(2)) { - query.trace("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + + query.trace("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + maxQueryTimeout + " ms). Setting timeout to " + maxQueryTimeout + " ms.", 2); } query.setTimeout(maxQueryTimeout); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java index 3dfa506a967..2787447e791 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.container.search.LegacyEmulationConfig; - /** * Helper class for carrying around cluster-related * config parameters to the FastSearcher class. @@ -12,21 +10,12 @@ import com.yahoo.container.search.LegacyEmulationConfig; public class ClusterParams { public final String searcherName; - public final LegacyEmulationConfig emulation; - - /** - * For testcases only - */ - public ClusterParams(String name) { - this(name, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } /** * Make up full ClusterParams */ - public ClusterParams(String name, LegacyEmulationConfig cfg) { + public ClusterParams(String name) { this.searcherName = name; - this.emulation = cfg; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index cdf696ebe1e..20615a099b6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -4,7 +4,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.yahoo.data.access.Inspector; -import com.yahoo.container.search.LegacyEmulationConfig; import java.util.ArrayList; import java.util.HashMap; @@ -41,8 +40,7 @@ public class DocsumDefinition { this.fieldNameToIndex = fieldNameToIndexBuilder.build(); } - // TODO: Remove LegacyEmulationConfig (the config, not just the usage) on Vespa 7 - DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { + DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config) { this.name = config.name(); List<DocsumField> fieldsBuilder = new ArrayList<>(); @@ -51,7 +49,7 @@ public class DocsumDefinition { for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { // no, don't switch the order of the two next lines :) fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); - fieldsBuilder.add(DocsumField.create(field.name(), field.type(), emulConfig)); + fieldsBuilder.add(DocsumField.create(field.name(), field.type())); if (field.dynamic()) dynamic = true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 07997f0c8f6..7745a71b24f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -7,7 +7,6 @@ import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.ConfigurationException; -import com.yahoo.container.search.LegacyEmulationConfig; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -31,27 +30,15 @@ public final class DocsumDefinitionSet { private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); private final Map<String, DocsumDefinition> definitionsByName; - private final LegacyEmulationConfig emulationConfig; public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this(config, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { - this(toDocsums(config, emulConfig), emulConfig); + this(toDocsums(config)); } public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions) { - this(docsumDefinitions, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions, LegacyEmulationConfig emulConfig) { this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); - this.emulationConfig = emulConfig; } - public LegacyEmulationConfig legacyEmulationConfig() { return emulationConfig; } - /** * Returns the summary definition of the given name, or the default if not found. * @@ -111,10 +98,10 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config) { Collection<DocsumDefinition> docsums = new ArrayList<>(); for (int i = 0; i < config.summaryclass().size(); ++i) - docsums.add(new DocsumDefinition(config.summaryclass(i), emulConfig)); + docsums.add(new DocsumDefinition(config.summaryclass(i))); if (docsums.isEmpty()) log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); return docsums; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 777919286dd..4f52ef91725 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.data.access.Inspector; import com.yahoo.log.LogLevel; @@ -31,19 +30,14 @@ public abstract class DocsumField { constructors.put(typename, constructor); } - DocsumField create(String typename, String name, LegacyEmulationConfig emulConfig) + DocsumField create(String typename, String name) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { DocsumField f = constructors.get(typename).newInstance(name); - f.emulConfig = emulConfig; return f; } } - private LegacyEmulationConfig emulConfig; - - final LegacyEmulationConfig getEmulConfig() { return emulConfig; } - static { fieldFactory = new FieldFactory(); @@ -75,14 +69,9 @@ public abstract class DocsumField { this.name = name; } - /* For unit test only */ public static DocsumField create(String name, String typename) { - return create(name, typename, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public static DocsumField create(String name, String typename, LegacyEmulationConfig emulConfig) { try { - return fieldFactory.create(typename, name, emulConfig); + return fieldFactory.create(typename, name); } catch (Exception e) { throw new RuntimeException("Unknown field type '" + typename + "'", e); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 0ae0983a1ae..aa492524c83 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -2,7 +2,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; -import com.yahoo.container.search.LegacyEmulationConfig; import java.util.ArrayList; import java.util.Collection; @@ -29,8 +28,8 @@ public class DocumentDatabase { private final ImmutableMap<String, RankProfile> rankProfiles; - public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { - this(documentDb.name(), new DocsumDefinitionSet(documentDb, emulConfig), toRankProfiles(documentDb.rankprofile())); + public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb) { + this(documentDb.name(), new DocsumDefinitionSet(documentDb), toRankProfiles(documentDb.rankprofile())); } public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection<RankProfile> rankProfiles) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 28d426cec06..8d73f6795d4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -77,7 +77,6 @@ public class FastHit extends Hit { // Note: This constructor is only used for tests, production use is always of the empty constructor public FastHit(String uri, double relevance, String source) { setId(uri); - super.setField("uri", uri); // TODO: Remove on Vespa 7 setRelevance(new Relevance(relevance)); setSource(source); types().add("summary"); @@ -97,13 +96,6 @@ public class FastHit extends Hit { URI uri = super.getId(); if (uri != null) return uri; - // TODO: Remove on Vespa 7, this should be one of the last vestiges of URL field magic - Object uriField = getField("uri"); - if (uriField != null) { - setId(uriField.toString()); - return super.getId(); - } - // Fallback to index:[source]/[partid]/[id] if (indexUri != null) return indexUri; StringBuilder sb = new StringBuilder(64); @@ -315,21 +307,6 @@ public class FastHit extends Hit { return super.hasFields(); } - /** - * Changes the key under which a value is found. This is useful because it allows keys to be changed - * without accessing the value (which may be lazily created). - * - * @deprecated do not use - */ - @Deprecated // OK - @Override - @SuppressWarnings("deprecation") - public void changeFieldKey(String oldKey, String newKey) { - Object value = removeField(oldKey); - if (value != null) - setField(newKey, value); - } - private Object getSummaryValue(String name) { if (removedFields != null && removedFields.contains(name)) return null; @@ -357,13 +334,6 @@ public class FastHit extends Hit { } } - /** @deprecated do not use */ - // TODO: Make private on Vespa 7 - @Deprecated // OK - public static String asHexString(GlobalId gid) { - return asHexString(new StringBuilder(), gid).toString(); - } - private static StringBuilder asHexString(StringBuilder sb, GlobalId gid) { byte[] rawGid = gid.getRawId(); for (byte b : rawGid) { @@ -539,7 +509,7 @@ public class FastHit extends Hit { DocsumField fieldType = type.getField(name); if (fieldType == null) return null; Inspector fieldValue = data.field(name); - if ( ! fieldValue.valid() && ! fieldType.getEmulConfig().forceFillEmptyFields()) return null; + if ( ! fieldValue.valid()) return null; return fieldType.convert(fieldValue); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java index 28c63aa2d3a..1f60dd3d1cf 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; import com.yahoo.data.access.Type; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.search.result.FeatureData; /** @@ -13,7 +12,7 @@ import com.yahoo.search.result.FeatureData; */ public class FeatureDataField extends LongstringField { - public FeatureDataField (String name) { + public FeatureDataField(String name) { super(name); } @@ -25,23 +24,12 @@ public class FeatureDataField extends LongstringField { @Override public Object convert(Inspector value) { if (! value.valid()) { - if (getEmulConfig().stringBackedFeatureData()) { - return ""; - } else if (getEmulConfig().forceFillEmptyFields()) { - return new FeatureData(com.yahoo.data.access.simple.Value.empty()); - } else { - return null; - } + return null; } if (value.type() == Type.STRING) { return value.asString(); } - FeatureData obj = new FeatureData(value); - if (getEmulConfig().stringBackedFeatureData()) { - return obj.toJson(); - } else { - return obj; - } + return new FeatureData(value); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java index 9521854477a..18986667a6c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java @@ -4,7 +4,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.search.result.StructuredData; import com.yahoo.data.access.Inspector; import com.yahoo.data.access.Type; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.JSONString; /** @@ -23,7 +22,7 @@ public class StructDataField extends JSONField { @Override public Object convert(Inspector value) { - if (getEmulConfig().stringBackedStructuredData() || value.type() == Type.STRING) { + if (value.type() == Type.STRING) { return super.convert(value); } return new StructuredData(value); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index e8f5d7110f7..d6e87e58fd8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -177,7 +177,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (documentdbInfoConfig != null) { for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { - DocumentDatabase db = new DocumentDatabase(docDb, clusterParams.emulation); + DocumentDatabase db = new DocumentDatabase(docDb); if (documentDbs.isEmpty()) { defaultDocumentDb = db; } @@ -372,7 +372,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { s.append(" ranking.queryCache=true"); } if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) { - s.append(" sessionId=").append(query.getSessionId(false)); + s.append(" sessionId=").append(query.getSessionId()); } List<Grouping> grouping = GroupingExecutor.getGroupingList(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java index cb115502468..d768dda2657 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java @@ -31,10 +31,10 @@ public class XMLField extends DocsumField { @Override public Object convert(Inspector value) { /* In Vespa 6 the backend will send an XML-formatted string to represent - * positions data. This will change in next version to sending an array - * of objects instead, suitable for the PositionsData class. + * positions data. This will change in next version to sending an object + * or an array of objects instead, suitable for the PositionsData class. */ - if (value.type() == Type.ARRAY) { + if (value.type() == Type.OBJECT || value.type() == Type.ARRAY) { return new PositionsData(value); } return convert(value.asString("")); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java index c8dc87aeb52..3c61678bbde 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java @@ -7,7 +7,7 @@ import java.util.ListIterator; /** * A specialized list iterator to manipulate FieldParts in HitField objects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class FieldIterator implements ListIterator<FieldPart> { diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java index f6619a32a2b..2702da099be 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java @@ -24,7 +24,7 @@ public class HitField { private boolean xmlProperty; private List<FieldPart> tokenizedContent = null; - private String content = null; + private String content; private Object original; @@ -48,7 +48,7 @@ public class HitField { /** * @param f The field name * @param c The field content - * @param cjk true if this is a cjk-document + * @param cjk true if the content is CJK text */ public HitField(String f, String c, boolean cjk) { this(f, c, cjk, false); @@ -57,7 +57,7 @@ public class HitField { /** * @param f The field name * @param c The field content - * @param cjk true if this is a cjk-document + * @param cjk true if the content is CJK text */ public HitField(String f, XMLString c, boolean cjk) { this(f, c.toString(), cjk, true); @@ -66,7 +66,7 @@ public class HitField { /** * @param f The field name * @param c The field content - * @param cjk true if this is a cjk-document + * @param cjk true if the content is CJK text * @param xmlProperty true if this should not quote XML syntax */ public HitField(String f, String c, boolean cjk, boolean xmlProperty) { @@ -279,9 +279,8 @@ public class HitField { // Must null content reference _before_ calling getContent() content = null; } - /** - * @return the content of this field - */ + + /** Returns the content of this field */ public String getContent() { if (content == null) { StringBuilder buf = new StringBuilder(); @@ -294,13 +293,8 @@ public class HitField { return content; } - /** - * @return the content of this field, using the arguments as bolding - * tags - */ - public String getContent(String boldOpenTag, - String boldCloseTag, - String separatorTag) { + /** Returns the content of this field, using the arguments as bolding tags */ + public String getContent(String boldOpenTag, String boldCloseTag, String separatorTag) { StringBuilder buf = new StringBuilder(); Iterator<FieldPart> iter = ensureTokenized().iterator(); while(iter.hasNext()) { @@ -372,8 +366,9 @@ public class HitField { } return xml.toString(); } + /** - * @return the content of the field, stripped of markup + * Returns the content of the field, stripped of markup */ public String bareContent(boolean XMLQuote, boolean inAttribute) { StringBuilder bareContent = new StringBuilder(); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index 06db012309e..eee7b310d13 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -86,6 +86,7 @@ public class JSONString implements Inspectable { didInitContent = true; } + @Override public String toString() { if (value != null) { return renderFromInspector(); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java index 95100dd4d39..beda005152c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java @@ -8,7 +8,7 @@ import java.util.NoSuchElementException; /** * A specialized list iterator to manipulate tokens in HitField objects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class TokenFieldIterator implements ListIterator<FieldPart> { diff --git a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java b/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java deleted file mode 100644 index f9faf242350..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.logging; - -/** - * Hollow compatibility class for com.yahoo.container.logging.AccessLogEntry. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class AccessLogEntry extends com.yahoo.container.logging.AccessLogEntry { - - public AccessLogEntry() { - super(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/logging/package-info.java b/container-search/src/main/java/com/yahoo/prelude/logging/package-info.java deleted file mode 100644 index 01126ec3484..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/logging/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.prelude.logging; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index f6f96ae215c..244d895f357 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -13,7 +13,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER; * @author Steinar Knutsen * @deprecated since 5.11, YQL+ should be used for formal queries */ -@Deprecated // OK DO NOT REMOVE (we'll keep this around longer) +@Deprecated // DO NOT REMOVE (we'll keep this around longer) public class AdvancedParser extends StructuredParser { public AdvancedParser(ParserEnvironment environment) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java deleted file mode 100644 index ff603a64725..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.querytransform; - -import static com.yahoo.prelude.querytransform.PhrasingSearcher.PHRASE_REPLACEMENT; - -import com.yahoo.component.chain.dependencies.After; -import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.Index.Attribute; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchchain.PhaseNames; - -import java.util.*; - -/** - * Searcher to rewrite queries to achieve mixed recall between indices and - * memory attributes. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 (not necessary any more) -@After({PhaseNames.RAW_QUERY, PHRASE_REPLACEMENT}) -@Before(PhaseNames.TRANSFORMED_QUERY) -@Provides(IndexCombinatorSearcher.MIXED_RECALL_REWRITE) -@Deprecated // OK -public class IndexCombinatorSearcher extends Searcher { - - public static final String MIXED_RECALL_REWRITE = "MixedRecallRewrite"; - - private static class ArrayComparator implements Comparator<Attribute[]> { - /** - * Note, this ignores if there is a difference in whether to - * attributes have tokenized content. (If this is the case, - * we are having worse problems anyway.) - */ - public int compare(Attribute[] o1, Attribute[] o2 ) { - if (o1.length < o2.length) { - return -1; - } else if (o1.length > o2.length) { - return 1; - } - int limit = o1.length; - for (int i = 0; i < limit; ++i) { - int r = o1[i].name.compareTo(o2[i].name); - if (r != 0) { - return r; - } - } - return 0; - } - } - - private final ArrayComparator comparator = new ArrayComparator(); - - private enum RewriteStrategies { - NONE, CHEAP_AND, EXPENSIVE_AND, FLAT - } - - @Override - public Result search(Query query, Execution execution) { - Item root = query.getModel().getQueryTree().getRoot(); - IndexFacts.Session session = execution.context().getIndexFacts().newSession(query); - String oldQuery = (query.getTraceLevel() >= 2) ? root.toString() : ""; - - if (root instanceof BlockItem || root instanceof PhraseItem) { - root = convertSinglePhraseOrBlock(root, session); - } else if (root instanceof CompositeItem) { - root = rewrite((CompositeItem) root, session); - } - query.getModel().getQueryTree().setRoot(root); - - if ((query.getTraceLevel() >= 2) && !(oldQuery.equals(root.toString()))) { - query.trace("Rewriting for mixed recall between indices and attributes", true, 2); - } - return execution.search(query); - } - - private RewriteStrategies chooseRewriteStrategy(CompositeItem c, IndexFacts.Session session) { - if (c instanceof OrItem) { - return RewriteStrategies.FLAT; - } else if (!(c instanceof AndItem)) { - return RewriteStrategies.NONE; - } - Map<Attribute[], Integer> m = new TreeMap<>(comparator); - for (Iterator<Item> i = c.getItemIterator(); i.hasNext();) { - Item j = i.next(); - if (j instanceof BlockItem || j instanceof PhraseItem) { - Attribute[] attributes= getIndices((HasIndexItem) j, session); - if (attributes == null) { - continue; - } - Integer count = m.get(attributes); - if (count == null) { - count = 1; - } else { - count = count.intValue() + 1; - } - m.put(attributes, count); - } - } - - if (m.size() == 0) { - return RewriteStrategies.NONE; - } - - int singles = 0; - int pairs = 0; - int higher = 0; - // count the number of sets being associated with 1, 2 or more terms - for (Integer i : m.values()) { - switch (i.intValue()) { - case 1: - ++singles; - break; - case 2: - pairs += 2; - break; - default: - ++higher; - break; - } - } - if (higher == 0 && pairs + singles <= 2) { - return RewriteStrategies.EXPENSIVE_AND; - } else { - return RewriteStrategies.CHEAP_AND; - } - } - - private CompositeItem rewriteNot(NotItem not, IndexFacts.Session session) { - Item positive = not.getItem(0); - if (positive instanceof BlockItem || positive instanceof PhraseItem) { - positive = convertSinglePhraseOrBlock(positive, session); - not.setItem(0, positive); - } else if (positive instanceof CompositeItem) { - CompositeItem c = (CompositeItem) positive; - positive = rewrite(c, session); - not.setItem(0, positive); - } - - int length = not.getItemCount(); - // no need for keeping proximity in the negative branches, so we - // convert them one by one, _and_ always uses cheap transform - for (int i = 1; i < length; ++i) { - Item exclusion = not.getItem(i); - if (exclusion instanceof BlockItem || exclusion instanceof PhraseItem) { - exclusion = convertSinglePhraseOrBlock(exclusion, session); - not.setItem(i, exclusion); - } else if (exclusion instanceof CompositeItem) { - CompositeItem c = (CompositeItem) exclusion; - switch (chooseRewriteStrategy(c, session)) { - case NONE: - c = traverse(c, session); - break; - case CHEAP_AND: - case EXPENSIVE_AND: - c = cheapTransform(c, session); - break; - default: - c = flatTransform(c, session); - break; - } - not.setItem(i, c); - } - } - return not; - } - - private Item rewrite(CompositeItem c, IndexFacts.Session session) { - if (c instanceof NotItem) { - c = rewriteNot((NotItem) c, session); - return c; - } else { - switch (chooseRewriteStrategy(c, session)) { - case NONE: - c = traverse(c, session); - break; - case CHEAP_AND: - c = cheapTransform(c, session); - break; - case EXPENSIVE_AND: - c = expensiveTransform((AndItem) c, session); - break; - case FLAT: - c = flatTransform(c, session); - break; - default: - break; - } - } - return c; - } - - private CompositeItem traverse(CompositeItem c, IndexFacts.Session session) { - int length = c.getItemCount(); - for (int i = 0; i < length; ++i) { - Item word = c.getItem(i); - if (word instanceof CompositeItem && !(word instanceof PhraseItem) && !(word instanceof BlockItem)) { - c.setItem(i, rewrite((CompositeItem) word, session)); - } - } - return c; - } - - private CompositeItem expensiveTransform(AndItem c, IndexFacts.Session session) { - int[] indices = new int[2]; - int items = 0; - int length = c.getItemCount(); - Attribute[][] names = new Attribute[2][]; - CompositeItem result = null; - for (int i = 0; i < length; ++i) { - Item word = c.getItem(i); - if (word instanceof BlockItem || word instanceof PhraseItem) { - Attribute[] attributes = getIndices((HasIndexItem) word, session); - if (attributes == null) { - continue; - } - // this throwing an out of bounds if more than two candidates is intentional - names[items] = attributes; - indices[items++] = i; - } else if (word instanceof CompositeItem) { - c.setItem(i, rewrite((CompositeItem) word, session)); - } - } - switch (items) { - case 1: - result = linearAnd(c, names[0], indices[0]); - break; - case 2: - result = quadraticAnd(c, names[0], names[1], indices[0], indices[1]); - break; - default: - // should never happen - getLogger().log( - LogLevel.WARNING, - "Unexpected number of items for mixed recall, got " + items - + ", expected 1 or 2."); - break; - } - return result; - } - - private Attribute[] getIndices(HasIndexItem block, IndexFacts.Session session) { - return session.getIndex(block.getIndexName()).getMatchGroup(); - } - - private OrItem linearAnd(AndItem c, Attribute[] names, int brancherIndex) { - OrItem or = new OrItem(); - for (int i = 0; i < names.length; ++i) { - AndItem duck = (AndItem) c.clone(); - Item b = retarget(duck.getItem(brancherIndex), names[i]); - duck.setItem(brancherIndex, b); - or.addItem(duck); - } - return or; - } - - private OrItem quadraticAnd(AndItem c, Attribute[] firstNames, Attribute[] secondNames, int firstBrancher, int secondBrancher) { - OrItem or = new OrItem(); - for (int i = 0; i < firstNames.length; ++i) { - for (int j = 0; j < secondNames.length; ++j) { - AndItem duck = (AndItem) c.clone(); - Item b = retarget(duck.getItem(firstBrancher), firstNames[i]); - duck.setItem(firstBrancher, b); - b = retarget(duck.getItem(secondBrancher), secondNames[j]); - duck.setItem(secondBrancher, b); - or.addItem(duck); - } - } - return or; - } - - private CompositeItem flatTransform(CompositeItem c, IndexFacts.Session session) { - int maxIndex = c.getItemCount() - 1; - for (int i = maxIndex; i >= 0; --i) { - Item word = c.getItem(i); - if (word instanceof BlockItem || word instanceof PhraseItem) { - Attribute[] attributes = getIndices((HasIndexItem) word, session); - if (attributes == null) { - continue; - } - c.removeItem(i); - for (Attribute name : attributes) { - Item term = word.clone(); - Item forNewIndex = retarget(term, name); - c.addItem(forNewIndex); - } - } else if (word instanceof CompositeItem) { - c.setItem(i, rewrite((CompositeItem) word, session)); - } - } - return c; - } - - private CompositeItem cheapTransform(CompositeItem c, IndexFacts.Session session) { - if (c instanceof OrItem) { - return flatTransform(c, session); - } - int length = c.getItemCount(); - for (int i = 0; i < length; ++i) { - Item j = c.getItem(i); - if (j instanceof BlockItem || j instanceof PhraseItem) { - Attribute[] attributes = getIndices((HasIndexItem) j, session); - if (attributes == null) { - continue; - } - CompositeItem or = searchAllForItem(j, attributes); - c.setItem(i, or); - } else if (j instanceof CompositeItem) { - c.setItem(i, rewrite((CompositeItem) j, session)); - } - } - return c; - } - - private OrItem searchAllForItem(Item word, Attribute[] attributes) { - OrItem or = new OrItem(); - for (Attribute name : attributes) { - Item term = word.clone(); - term = retarget(term, name); - or.addItem(term); - } - return or; - } - - private Item retarget(Item word, Attribute newIndex) { - if (word instanceof PhraseItem && !newIndex.isTokenizedContent()) { - PhraseItem asPhrase = (PhraseItem) word; - WordItem newWord = new WordItem(asPhrase.getIndexedString(), newIndex.name, false); - return newWord; - } else if (word instanceof IndexedItem) { - word.setIndexName(newIndex.name); - } else if (word instanceof CompositeItem) { - CompositeItem asComposite = (CompositeItem) word; - for (Iterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { - Item segment = i.next(); - segment.setIndexName(newIndex.name); - } - } - return word; - } - - private Item convertSinglePhraseOrBlock(Item item, IndexFacts.Session session) { - Item newItem; - Attribute[] attributes = getIndices((HasIndexItem) item, session); - if (attributes == null) { - return item; - } - newItem = searchAllForItem(item, attributes); - return newItem; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 28d384b0f63..61ce9d98e69 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -38,21 +38,20 @@ public class BlendingSearcher extends Searcher { public static final String BLENDING = "Blending"; - private final String documentId; + private final String blendingField; @Inject public BlendingSearcher(ComponentId id, QrSearchersConfig cfg) { super(id); QrSearchersConfig.Com.Yahoo.Prelude.Searcher.BlendingSearcher s = cfg.com().yahoo().prelude().searcher().BlendingSearcher(); - documentId = s.docid().length() > 0 ? s.docid() : null; - + blendingField = s.docid().length() > 0 ? s.docid() : null; } /** * Only for legacy tests. */ - public BlendingSearcher(String blendingDocumentId) { - this.documentId = blendingDocumentId; + public BlendingSearcher(String blendingField) { + this.blendingField = blendingField; } @Override @@ -107,7 +106,7 @@ public class BlendingSearcher extends Searcher { result.hits().setOrderer(groups.get(0).getOrderer()); return result; } else { - if (documentId != null) { + if (blendingField != null) { return blendResultsUniquely(result, q, offset, hits, groups, execution); } else { return blendResultsDirectly(result, q, offset, hits, groups, execution); @@ -125,6 +124,7 @@ public class BlendingSearcher extends Searcher { } private abstract class DocumentMerger { + protected Set<String> documentsToStrip; protected Result result; protected HitGroup group; @@ -137,22 +137,22 @@ public class BlendingSearcher extends Searcher { return result; } - //Since we cannot use prelude.hit#getProperty, we'll have to improvise private String getProperty(Hit hit, String field) { + if ("[id]".equals(field)) return hit.getId().toString(); Object o = hit.getField(field); return o == null ? null : o.toString(); } protected void storeID(Hit hit, Execution execution) { - String id = getProperty(hit, documentId); + String id = getProperty(hit, blendingField); if (id != null) { documentsToStrip.add(id); } else { if (!result.isFilled(result.getQuery().getPresentation().getSummary())) { fill(result, result.getQuery().getPresentation().getSummary(), execution); - id = getProperty(hit, documentId); + id = getProperty(hit, blendingField); if (id != null) { documentsToStrip.add(id); } @@ -161,14 +161,14 @@ public class BlendingSearcher extends Searcher { } protected boolean known(HitGroup source, Hit hit, Execution execution) { - String stripID = getProperty(hit, documentId); + String stripID = getProperty(hit, blendingField); if (stripID == null) { if (!source.isFilled(result.getQuery().getPresentation().getSummary())) { Result nResult = new Result(result.getQuery()); nResult.hits().add(source); fill(nResult, nResult.getQuery().getPresentation().getSummary(), execution); - stripID = getProperty(hit, documentId); + stripID = getProperty(hit, blendingField); if (stripID == null) { return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java deleted file mode 100644 index bc3c6665ae2..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - - -/** - * <p>Implements a document source. You pass in a query and a Result - * set. When this Searcher is called with that query it will return - * that result set.</p> - * - * <p>This supports multi-phase search.</p> - * - * <p>To avoid having to add type information for the fields, a quck hack is used to - * support testing of attribute prefetching. - * Any field in the configured hits which has a name starting by attribute - * will be returned when attribute prefetch filling is requested.</p> - * - * @author bratseth - * @deprecated use {@link com.yahoo.search.searchchain.testutil.DocumentSourceSearcher} - */ -@SuppressWarnings({"rawtypes"}) -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class DocumentSourceSearcher extends Searcher { - - // as for the SuppressWarnings annotation above, we are inside - // com.yahoo.prelude, this is old stuff, really no point firing off those - // warnings here... - - private Result defaultFilledResult; - private Map<Query, Result> completelyFilledResults = new HashMap<>(); - private Map<Query, Result> attributeFilledResults = new HashMap<>(); - private Map<Query, Result> unFilledResults = new HashMap<>(); - - /** Time (in ms) at which the index of this searcher was last modified */ - long editionTimeStamp=0; - - private int queryCount; - - public DocumentSourceSearcher() { - addDefaultResults(); - } - - /** - * Adds a result which can be returned either as empty, - * filled or attribute only filled later. - * Summary fields starting by "a" are attributes, others are not. - * - * @return true when replacing an existing <query, result> pair. - */ - public boolean addResultSet(Query query, Result fullResult) { - Result emptyResult = new Result(query.clone()); - Result attributeResult = new Result(query.clone()); - emptyResult.setTotalHitCount(fullResult.getTotalHitCount()); - attributeResult.setTotalHitCount(fullResult.getTotalHitCount()); - int counter=0; - for (Iterator i = fullResult.hits().deepIterator();i.hasNext();) { - Hit fullHit = (Hit)i.next(); - - Hit emptyHit = fullHit.clone(); - emptyHit.clearFields(); - emptyHit.setFillable(); - emptyHit.setRelevance(fullHit.getRelevance()); - - Hit attributeHit = fullHit.clone(); - removePropertiesNotStartingByA(attributeHit); - attributeHit.setFillable(); - attributeHit.setRelevance(fullHit.getRelevance()); - for (Object propertyKeyObject : fullHit.fields().keySet()) { - String propertyKey=propertyKeyObject.toString(); - if (propertyKey.startsWith("attribute")) - attributeHit.setField(propertyKey, fullHit.getField(propertyKey)); - } - if (fullHit.getField(Hit.SDDOCNAME_FIELD)!=null) - attributeHit.setField(Hit.SDDOCNAME_FIELD, fullHit.getField(Hit.SDDOCNAME_FIELD)); - - // A simple summary lookup mechanism, similar to FastSearch's - emptyHit.setField("summaryid", String.valueOf(counter)); - attributeHit.setField("summaryid", String.valueOf(counter)); - fullHit.setField("summaryid", String.valueOf(counter)); - - counter++; - emptyResult.hits().add(emptyHit); - attributeResult.hits().add(attributeHit); - } - unFilledResults.put(getQueryKeyClone(query), emptyResult); - attributeFilledResults.put(getQueryKeyClone(query), attributeResult); - if (completelyFilledResults.put(getQueryKeyClone(query), fullResult.clone()) != null) { - setEditionTimeStamp(System.currentTimeMillis()); - return true; - } - return false; - } - - /** - * Returns a query clone which has source, offset and hits set to null. This is used by access to - * the maps using the query as key to achieve lookup independent of offset/hits value - */ - private Query getQueryKeyClone(Query query) { - Query key = query.clone(); - key.setWindow(0,0); - key.getModel().setSources(""); - return key; - } - - private void removePropertiesNotStartingByA(Hit hit) { - List<String> toRemove=new java.util.ArrayList<>(); - for (Iterator i= ((Set) hit.fields().keySet()).iterator(); i.hasNext(); ) { - String key=(String)i.next(); - if (!key.startsWith("a")) - toRemove.add(key); - } - for (Iterator<String> i=toRemove.iterator(); i.hasNext(); ) { - String propertyName=i.next(); - hit.removeField(propertyName); - } - } - - private void addDefaultResults() { - Query q = new Query("?query=default"); - Result r = new Result(q); - r.hits().add(new Hit("http://default-1.html")); - r.hits().add(new Hit("http://default-2.html")); - r.hits().add(new Hit("http://default-3.html")); - r.hits().add(new Hit("http://default-4.html")); - defaultFilledResult = r; - addResultSet(q, r); - } - - public long getEditionTimeStamp(){ - long myEditionTime; - synchronized(this){ - myEditionTime=this.editionTimeStamp; - } - return myEditionTime; - } - - public void setEditionTimeStamp(long editionTime) { - synchronized(this){ - this.editionTimeStamp=editionTime; - } - } - - public Result search(com.yahoo.search.Query query, Execution execution) { - queryCount++; - Result r; - r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) { - r = defaultFilledResult.clone(); - } else { - r = r.clone(); - } - r.setQuery(query); - r.hits().trim(query.getOffset(), query.getHits()); - return r; - } - - @Override - public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { - Result filledResult; - if ("attributeprefetch".equals(summaryClass)) - filledResult=attributeFilledResults.get(getQueryKeyClone(result.getQuery())); - else - filledResult = completelyFilledResults.get(getQueryKeyClone(result.getQuery())); - - if (filledResult == null) { - filledResult = defaultFilledResult; - } - fillHits(filledResult,result,summaryClass); - } - - private void fillHits(Result source,Result target,String summaryClass) { - for (Iterator hitsToFill= target.hits().deepIterator() ; hitsToFill.hasNext();) { - Hit hitToFill = (Hit) hitsToFill.next(); - String summaryId= (String) hitToFill.getField("summaryid"); - if (summaryId==null) continue; // Can not fill this - Hit filledHit = lookupBySummaryId(source,summaryId); - if (filledHit==null) - throw new RuntimeException("Can't fill hit with summaryid '" + summaryId + "', not present"); - - for (Iterator props= filledHit.fieldIterator();props.hasNext();) { - Map.Entry propertyEntry = (Map.Entry)props.next(); - hitToFill.setField(propertyEntry.getKey().toString(), - propertyEntry.getValue()); - } - hitToFill.setFilled(summaryClass); - } - target.analyzeHits(); - } - - private Hit lookupBySummaryId(Result result,String summaryId) { - for (Iterator i= result.hits().deepIterator(); i.hasNext(); ) { - Hit hit=(Hit)i.next(); - if (summaryId.equals(hit.getField("summaryid"))) { - return hit; - } - } - return null; - } - - /** - * Returns the number of queries made to this searcher since the last - * reset. For testing - not reliable if multiple threads makes - * queries simultaneously - */ - public int getQueryCount() { - return queryCount; - } - - public void resetQueryCount() { - queryCount=0; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index 5c56379efc0..d11b8d86d29 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -77,10 +77,9 @@ public class JuniperSearcher extends Searcher { @Override public void fill(Result result, String summaryClass, Execution execution) { - Result workResult = result; - int worstCase = workResult.getHitCount(); + int worstCase = result.getHitCount(); List<Hit> hits = new ArrayList<>(worstCase); - for (Iterator<Hit> i = workResult.hits().deepIterator(); i.hasNext();) { + for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext();) { Hit sniffHit = i.next(); if ( ! (sniffHit instanceof FastHit)) continue; @@ -89,26 +88,26 @@ public class JuniperSearcher extends Searcher { hits.add(hit); } - execution.fill(workResult, summaryClass); - highlight(workResult.getQuery().getPresentation().getBolding(), hits.iterator(), summaryClass, + execution.fill(result, summaryClass); + highlight(result.getQuery().getPresentation().getBolding(), hits.iterator(), summaryClass, execution.context().getIndexFacts().newSession(result.getQuery())); } private void highlight(boolean bolding, Iterator<Hit> hitsToHighlight, String summaryClass, IndexFacts.Session indexFacts) { while (hitsToHighlight.hasNext()) { - Hit sniffHit = hitsToHighlight.next(); - if ( ! (sniffHit instanceof FastHit)) continue; + Hit hit = hitsToHighlight.next(); + if ( ! (hit instanceof FastHit)) continue; - FastHit hit = (FastHit) sniffHit; - if (summaryClass != null && ! hit.isFilled(summaryClass)) continue; + FastHit fastHit = (FastHit) hit; + if (summaryClass != null && ! fastHit.isFilled(summaryClass)) continue; - Object searchDefinitionField = hit.getField(MAGIC_FIELD); + Object searchDefinitionField = fastHit.getField(MAGIC_FIELD); if (searchDefinitionField == null) continue; for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { if (index.getDynamicSummary() || index.getHighlightSummary()) { - HitField fieldValue = hit.buildHitField(index.getName(), true, true); + HitField fieldValue = fastHit.buildHitField(index.getName(), true); if (fieldValue != null) insertTags(fieldValue, bolding, index.getDynamicSummary()); } @@ -116,9 +115,9 @@ public class JuniperSearcher extends Searcher { } } - private void insertTags(HitField oldProperty, boolean bolding, boolean dynteaser) { + private void insertTags(HitField field, boolean bolding, boolean dynteaser) { boolean insideHighlight = false; - for (ListIterator<FieldPart> i = oldProperty.listIterator(); i.hasNext();) { + for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext();) { FieldPart f = i.next(); if (f instanceof SeparatorFieldPart) setSeparatorString(bolding, (SeparatorFieldPart) f); @@ -138,8 +137,7 @@ public class JuniperSearcher extends Searcher { break; case RAW_SEPARATOR_CHAR: newFieldParts = initFieldParts(newFieldParts); - addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, - previous, j); + addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, previous, j); previous = j + 1; break; default: diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java deleted file mode 100644 index 4edb907d337..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.Relevance; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; - -/** - * Save the query in the incoming state to a meta hit in the result. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QuerySnapshotSearcher extends Searcher { - - public Result search(Query query, Execution execution) { - Query q = query.clone(); - Result r = execution.search(query); - Hit h = new Hit("meta:querysnapshot", new Relevance(Double.POSITIVE_INFINITY)); - h.setMeta(true); - h.setField("query", q); - r.hits().add(h); - return r; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java deleted file mode 100644 index fd155354d1c..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; - -/** - * Ensures hits is 1000 or less and offset is 1000 or less. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryValidatingSearcher extends Searcher { - - public Result search(Query query, Execution execution) { - if (query.getHits() > 1000) { - Result result = new Result(query); - ErrorMessage error - = ErrorMessage.createInvalidQueryParameter("Too many hits (more than 1000) requested."); - result.hits().addError(error); - return result; - } - if (query.getOffset() > 1000) { - Result result = new Result(query); - ErrorMessage error - = ErrorMessage.createInvalidQueryParameter("Offset too high (above 1000)."); - result.hits().addError(error); - return result; - } - return execution.search(query); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index 89849f98461..4d46107726e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -15,7 +15,6 @@ import com.yahoo.search.searchchain.PhaseNames; import java.util.*; -import static com.yahoo.prelude.querytransform.IndexCombinatorSearcher.MIXED_RECALL_REWRITE; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; /** @@ -24,7 +23,7 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; * @author bratseth */ @After(PhaseNames.RAW_QUERY) -@Before({PhaseNames.TRANSFORMED_QUERY, STEMMING, MIXED_RECALL_REWRITE}) +@Before({PhaseNames.TRANSFORMED_QUERY, STEMMING}) public class SemanticSearcher extends Searcher { private static final CompoundName rulesRulebase=new CompoundName("rules.rulebase"); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index beb79acb893..3606e01ffe5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.statistics; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.container.Server; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; @@ -18,9 +17,7 @@ import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import com.yahoo.statistics.Callback; import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Handle; import com.yahoo.statistics.Value; import java.util.HashMap; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java b/container-search/src/main/java/com/yahoo/prelude/templates/Context.java deleted file mode 100644 index 7662ca832f9..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.util.Collection; - -import com.yahoo.text.XML; - -/** - * A set of variable bindings for template rendering - * - * @author bratseth - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public abstract class Context { - - private boolean xmlEscape = true; - - // These may be wrapped in an object if it gets unruly like this... - private String boldOpenTag; - private String boldCloseTag; - private String separatorTag; - - private boolean utf8Output = false; - - //prevent sub-classing outside of this package. - Context() {} - - // set|getXmlEscape no longer final on cause of HitContext subclassing _and_ wrapping Context - /** Sets whether this context should xml-escape returned values */ - public void setXmlEscape(boolean xmlEscape) { this.xmlEscape=xmlEscape; } - - /** Returns whether this context xml-escapes returned values. Default is true */ - public boolean getXmlEscape() { return xmlEscape; } - - /** - * Makes a <b>secondary</b> binding - * - * @return the old value bound to this key, or null it the key was previously unbound - */ - public abstract Object put(String key,Object value); - - /** - * <p>Returns a value by looking it up in the primary, - * and thereafter in secondary sources.</p> - * - * <p>If xml escaping is on and this is a string, xml attribute escaping is done - * </p> - */ - abstract public Object get(String key); - - /** - * Removes a <b>secondary</b> binding - * - * @return the removed value, or null if it wasn't bound - */ - public abstract Object remove(Object key); - - - // These three may be collapsed to one method - public void setBoldOpenTag(String boldOpenTag) { - this.boldOpenTag = boldOpenTag; - } - public void setBoldCloseTag(String boldCloseTag) { - this.boldCloseTag = boldCloseTag; - } - public void setSeparatorTag(String separatorTag) { - this.separatorTag = separatorTag; - } - - - protected Object normalizeValue(Object value) { - if (value == null) { - return ""; - } else if (xmlEscape && value instanceof String) { - return XML.xmlEscape((String) value, true, null); - } else { - return value; - } - } - - public String getBoldOpenTag() { - return boldOpenTag; - } - - public String getBoldCloseTag() { - return boldCloseTag; - } - - public String getSeparatorTag() { - return separatorTag; - } - - public abstract Collection<? extends Object> getKeys(); - - /** - * Used by the template to decide whether to use UTF-8 optimizations. - * - * @return whether the result encoding is UTF-8 - */ - public boolean isUtf8Output() { - return utf8Output; - } - - /** - * Used by the template to decide whether to use UTF-8 optimizations. - * TODO: TVT: Make this package private again - * @param utf8Output whether the output encoding is UTF-8 - */ - public void setUtf8Output(boolean utf8Output) { - this.utf8Output = utf8Output; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java deleted file mode 100644 index 4d6406fe00d..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.io.ByteWriter; -import com.yahoo.net.URI; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.Result; -import com.yahoo.search.grouping.result.HitRenderer; -import com.yahoo.search.result.*; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; -import java.util.Iterator; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * <p>A template set which provides XML rendering of results and hits.</p> - * - * <p>This can be extended to create custom programmatic templates. - * Create a subclass which has static inner classes extending DefaultTemplate for the templates - * you wish to override and call the set method for those templates in the subclass template set - * constructor. Some of the default templates contained utility functions, and can be overridden - * in place of DefaultTemplate to gain access to these. See TiledTemplateSet for an example.</p> - * - * @author bratseth - * @deprecated use JsonRenderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class DefaultTemplateSet extends UserTemplate<XMLWriter> { - - private static final Utf8String RESULT = new Utf8String("result"); - private static final Utf8String GROUP = new Utf8String("group"); - private static final Utf8String ID = new Utf8String("id"); - private static final Utf8String FIELD = new Utf8String("field"); - private static final Utf8String HIT = new Utf8String("hit"); - private static final Utf8String ERROR = new Utf8String("error"); - private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count"); - private static final Utf8String QUERY_TIME = new Utf8String("querytime"); - private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime"); - private static final Utf8String SEARCH_TIME = new Utf8String("searchtime"); - private static final Utf8String NAME = new Utf8String("name"); - private static final Utf8String CODE = new Utf8String("code"); - private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs"); - private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes"); - private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full"); - private static final Utf8String COVERAGE = new Utf8String("coverage"); - private static final Utf8String RESULTS_FULL = new Utf8String("results-full"); - private static final Utf8String RESULTS = new Utf8String("results"); - private static final Utf8String TYPE = new Utf8String("type"); - private static final Utf8String RELEVANCY = new Utf8String("relevancy"); - private static final Utf8String SOURCE = new Utf8String("source"); - - private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>(); - - - /** - * Create a template set with a name. This will be initialized with the default templates - - * use the set methods from the subclass constructor to override any of these with other template classes. - */ - protected DefaultTemplateSet(String name) { - super(name, - DEFAULT_MIMETYPE, - DEFAULT_ENCODING - ); - } - - public DefaultTemplateSet() { - this("default"); - } - - /** Uses an XML writer in this template */ - @Override - public XMLWriter wrapWriter(Writer writer) { - return XMLWriter.from(writer, 10, -1); - } - - @Override - public void header(Context context, XMLWriter writer) throws IOException { - Result result=(Result)context.get("result"); - // TODO: move setting this to Result - context.setUtf8Output("utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery()))); - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT,String.valueOf(result.getTotalHitCount())); - renderCoverageAttributes(result.getCoverage(false), writer); - renderTime(writer, result); - writer.closeStartTag(); - } - - private void renderTime(final XMLWriter writer, final Result result) { - if (!result.getQuery().getPresentation().getTiming()) { - return; - } - - final String threeDecimals = "%.3f"; - final double milli = .001d; - final long now = System.currentTimeMillis(); - final long searchTime = now - result.getElapsedTime().first(); - final double searchSeconds = ((double) searchTime) * milli; - - if (result.getElapsedTime().firstFill() != 0L) { - final long queryTime = result.getElapsedTime().weightedSearchTime(); - final long summaryFetchTime = result.getElapsedTime().weightedFillTime(); - final double querySeconds = ((double) queryTime) * milli; - final double summarySeconds = ((double) summaryFetchTime) * milli; - writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds)); - writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds)); - } - writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds)); - } - - @Override - public void footer(Context context, XMLWriter writer) throws IOException { - writer.closeTag(); - } - - @Override - /** - * Renders the header of a hit.<br/> - * Post-condition: The hit tag is open in this XML writer - */ - public void hit(Context context, XMLWriter writer) throws IOException { - Hit hit=(Hit)context.get("hit"); - - if (hit instanceof HitGroup) { - renderHitGroup((HitGroup) hit, context, writer); - } else { - writer.openTag(HIT); - renderHitAttributes(hit,writer); - writer.closeStartTag(); - renderHitFields(context, hit, writer); - } - } - - - @Override - /** - * Renders the footer of a hit. - * - * Pre-condition: The hit tag is open in this XML writer.<br/> - * Post-condition: The hit tag is closed - */ - public void hitFooter(Context context, XMLWriter writer) throws IOException { - writer.closeTag(); - } - - @Override - public void error(Context context, XMLWriter writer) { - ErrorMessage error=((Result)context.get("result")).hits().getError(); - writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag(); - } - - @Override - public void noHits(Context context, XMLWriter writer) { - // no hits, do nothing :) - } - - protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) { - if (coverage == null) return; - writer.attribute(COVERAGE_DOCS,coverage.getDocs()); - writer.attribute(COVERAGE_NODES,coverage.getNodes()); - writer.attribute(COVERAGE_FULL,coverage.getFull()); - writer.attribute(COVERAGE,coverage.getResultPercentage()); - writer.attribute(RESULTS_FULL,coverage.getFullResultSets()); - writer.attribute(RESULTS,coverage.getResultSets()); - } - - /** - * Writes a hit's default attributes like 'type', 'source', 'relevancy'. - */ - protected void renderHitAttributes(Hit hit, XMLWriter writer) throws IOException { - writer.attribute(TYPE, hit.types().stream().collect(Collectors.joining(" "))); - if (hit.getRelevance() != null) { - writer.attribute(RELEVANCY, hit.getRelevance().toString()); - } - writer.attribute(SOURCE, hit.getSource()); - } - - /** Opens (but does not close) the group hit tag */ - protected void renderHitGroup(HitGroup hit, Context context, XMLWriter writer) throws IOException { - if (HitRenderer.renderHeader(hit, writer)) { - // empty - } else if (hit.types().contains("grouphit")) { - // TODO Keep this? - renderHitGroupOfTypeGroupHit(context, hit, writer); - } else { - renderGroup(hit, writer); - } - } - - /** - * Renders a hit group. - */ - protected void renderGroup(HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag(GROUP); - renderHitAttributes(hit, writer); - writer.closeStartTag(); - } - - // Can't name this renderGroupHit as GroupHit is a class having nothing to do with HitGroup. - // Confused yet? Good! - protected void renderHitGroupOfTypeGroupHit(Context context, HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag(HIT); - renderHitAttributes(hit, writer); - renderId(hit.getId(), writer); - writer.closeStartTag(); - } - - - protected void renderId(URI uri, XMLWriter writer) { - if (uri != null) { - writer.openTag(ID).content(uri.stringValue(),false).closeTag(); - } - } - - /** - * Renders all fields of a hit. - * Simply calls {@link #renderField(Context, Hit, java.util.Map.Entry, XMLWriter)} for every field. - */ - protected void renderHitFields(Context context, Hit hit, XMLWriter writer) throws IOException { - renderSyntheticRelevancyField(hit, writer); - for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) { - renderField(context, hit, it.next(), writer); - } - } - - private void renderSyntheticRelevancyField(Hit hit, XMLWriter writer) { - final String relevancyFieldName = "relevancy"; - final Relevance relevance = hit.getRelevance(); - - if (shouldRenderField(hit, relevancyFieldName) && relevance != null) { - renderSimpleField(relevancyFieldName, relevance, writer); - } - } - - protected void renderField(Context context, Hit hit, Map.Entry<String, Object> entry, XMLWriter writer) throws IOException { - String fieldName = entry.getKey(); - - if (!shouldRenderField(hit, fieldName)) return; - if (fieldName.startsWith("$")) return; // Don't render fields that start with $ // TODO: Move to should render - - writeOpenFieldElement(fieldName, writer); - renderFieldContent(context, hit, fieldName, writer); - writeCloseFieldElement(writer); - } - - private void writeOpenFieldElement(String fieldName, XMLWriter writer) { - Utf8String utf8 = fieldNameMap.get(fieldName); - if (utf8 == null) { - utf8 = new Utf8String(fieldName); - fieldNameMap.put(fieldName, utf8); - } - writer.openTag(FIELD).attribute(NAME, utf8); - writer.closeStartTag(); - } - - private void writeCloseFieldElement(XMLWriter writer) { - writer.closeTag(); - } - - protected void renderFieldContent(Context context, Hit hit, String name, XMLWriter writer) { - writer.escapedContent(asXML(hit.getField(name)), false); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - private void renderSimpleField(String fieldName, Object fieldValue, XMLWriter writer) { - writeOpenFieldElement(fieldName, writer); - writer.content(fieldValue.toString(),false); - writeCloseFieldElement(writer); - } - - /** Returns whether a field should be rendered. This default implementation always returns true */ - protected boolean shouldRenderField(Hit hit, String fieldName) { - // skip depending on hit type - return true; - } - - - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java b/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java deleted file mode 100644 index 1f724aba112..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; - -/** - * Defines formatting options used with special kinds of hits. - * - * @author laboisse - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class FormattingOptions { - - public static final String DEFAULT_TYPE_ATTRIBUTE_NAME = "type"; - - /** - * A structure that defines the tag name and attribute name for a field - * that sould be formatted as a field with a subtype. - * @author laboisse - * - */ - static class SubtypeField { - String tagName; - String attributeName; - String attributeValue; - } - - static class SubtypeFieldWithPrefix extends SubtypeField { - - /* Note: attributeValue should always be null for instances of this class */ - - int prefixLength; - } - - private Map<String, String> fieldsAsAttributes = new LinkedHashMap<>(); - - private Map<String, SubtypeField> fieldsWithSubtypes = new LinkedHashMap<>(); - private Map<String, SubtypeFieldWithPrefix> prefixedFieldsWithSubtypes = new LinkedHashMap<>(); - - private Set<String> fieldsNotRendered = new LinkedHashSet<>(); - private Set<String> fieldsRendered = new LinkedHashSet<>(); - - /** - * Tells to format a field as an attribute of the hit's tag. - * - * For instance, field 'query-latency' could be rendered as an attribute 'latency' by - * invoking {@code formatFieldAsAttribute("query-latency", "latency")}. - * - * Output would be: - * <pre> - * <hit latency="100"></hit> - * </pre> - * instead of: - * <pre> - * <hit><latency>100</latency></hit> - * </pre> - */ - public void formatFieldAsAttribute(String fieldName, String attributeName) { - fieldsAsAttributes.put(fieldName, attributeName); - } - - public Set<Map.Entry<String, String>> fieldsAsAttributes() { - return Collections.unmodifiableSet(this.fieldsAsAttributes.entrySet()); - } - - public String getAttributeName(String fieldName) { - return this.fieldsAsAttributes.get(fieldName); - } - - /** - * Tells to format a field using a subtype. A subtype is used when there is kind of a grouping - * for a set of fields. - * - * For instance, fields 'latency-connect', 'latency-finish' all belong to the same 'latency' logical group. - * So invoking {@code formatFieldWithSubtype("latency-connect", "latency", "type", "connect"}, - * {@code formatFieldWithSubtype("latency-finish", "latency", "type", "connect"} and so on, - * allows to have a common 'latency' tag name for all fields of the same kind. - * Note that it does no collapsing on tags. - * - * Output would be: - * <pre> - * <latency type="connect">50</latency> - * <latency type="finish">250</latency> - * </pre> - * Instead of: - * <pre> - * <hit> - * <latency-connect>50</latency-connect> - * <latency-finish>50</latency-finish> - * </pre> - */ - public void formatFieldWithSubtype(String fieldName, String tagName, String typeAttributeName, String typeAttributeValue) { - SubtypeField names = new SubtypeField(); - names.attributeName = typeAttributeName; - names.attributeValue = typeAttributeValue; - names.tagName = tagName; - fieldsWithSubtypes.put(fieldName, names); - } - - public SubtypeField getSubtype(String fieldName) { - return this.fieldsWithSubtypes.get(fieldName); - } - - /** - * Same as {@link #formatFieldWithSubtype(String, String, String, String)} except that fields - * are selected based on the beginning of their name and the type attribute value is deduced - * from the rest of their name. So this may select many fields instead of only one. - * Invoking {@code formatFieldWithSubtype("latency-", "latency", "type")} only once allows to have a common 'latency' - * tag name for all fields that start with 'latency-'. Type attribute value will be 'start' for field 'latency-start'. - * Note that it does no collapsing on tags. - * - * This is mostly used when you don't know all field names ahead. - * - * Output would be: - * <pre> - * <latency type="connect">50</latency> - * <latency type="finish">250</latency> - * </pre> - * Instead of: - * <pre> - * <hit> - * <latency-connect>50</latency-connect> - * <latency-finish>50</latency-finish> - * </pre> - * - * Note: don't use this with prefixes that start with a common substring (e.g. 'http', 'http_proxy'), I can tell you it just won't work. - */ - public void formatFieldWithSubtype(String fieldNamePrefix, String tagName, String typeAttributeName) { - SubtypeFieldWithPrefix names = new SubtypeFieldWithPrefix(); - names.attributeName = typeAttributeName; - names.tagName = tagName; - names.prefixLength = fieldNamePrefix.length(); - prefixedFieldsWithSubtypes.put(fieldNamePrefix, names); - } - - public SubtypeFieldWithPrefix getSubtypeWithPrefix(String fieldName) { - for(Map.Entry<String, SubtypeFieldWithPrefix> e : this.prefixedFieldsWithSubtypes.entrySet()) { - if(fieldName.startsWith(e.getKey())) - return e.getValue(); - } - return null; - } - - /** - * Tells whether a field should be rendered. - * - * @see #setFieldNotToRender(String) - * @see #setFieldToRender(String) - */ - public boolean shouldRenderField(String fieldName) { - if(fieldName == null) - return false; - if (fieldName.startsWith("$")) { - return false; - } - if(!this.fieldsRendered.isEmpty()) - return this.fieldsRendered.contains(fieldName); - return !this.fieldsNotRendered.contains(fieldName); - } - - /** - * Tells a field should be rendered. - * - * <p> - * Note: if at least one field is set to render, then only - * these fields should be rendered. Use {@link #setFieldNotToRender(String)} - * to only exclude specific fields. - */ - public void setFieldToRender(String fieldName) { - this.fieldsRendered.add(fieldName); - } - - /** - * Tells a field should not be rendered. - * - * <p> - * Note: all other fields should be rendered. Use {@link #setFieldToRender(String)} - * to only include specific fields. - */ - public void setFieldNotToRender(String fieldName) { - this.fieldsNotRendered.add(fieldName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/GenericTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/GenericTemplateSet.java deleted file mode 100644 index 95875d0dd1f..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/GenericTemplateSet.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.protect.Validator; -import com.yahoo.search.Query; - -import java.io.Writer; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Properties; - -/** - * Superclass of a set of templates for rendering (serializing) results - * - * @deprecated use a renderer instead - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class GenericTemplateSet { - - public static final String DEFAULT_MIMETYPE = "text/xml"; - public static final String DEFAULT_ENCODING = "utf-8"; - - /** Templates */ - private HashMap<String, Template<? extends Writer>> templates; - - /** The text MIME subtype this template returns, xml, plain or html */ - private String mimeType; - - /** The charset encoding this template should have */ - private String encoding; - - private String boldOpenTag = null; - private String boldCloseTag = null; - private String separatorTag = null; - - /** - * Document summary class for this template - */ - private String summaryClass = null; - - /** - * The unique name of this template set - */ - private final String name; - - /** - * Creates a template set containing no templates - */ - public GenericTemplateSet(String name, String mimeType, String encoding) { - this.mimeType = mimeType; - this.encoding = encoding; - this.name = name; - - templates = new LinkedHashMap<>(); - } - - - public String getName() { - return name; - } - - /** - * Returns the text MIME - */ - public String getMimeType() { return mimeType; } - - /** - * Returns the text encoding - */ - public String getEncoding() { return encoding; } - - /** Returns the encoding of the query, or the encoding given by the template if none is set */ - public final String getRequestedEncoding(Query query) { - String encoding = query.getModel().getEncoding(); - if (encoding != null) return encoding; - return getEncoding(); - } - - /** - * Returns the selected template - * - * @return the template to use, never null - */ - public Template<? extends Writer> getTemplate(String templateName) { - return templates.get(templateName); - } - - /** - * Sets the selected template - * - * @throws NullPointerException if the given template is null - */ - public void setTemplate(String templateName, Template<? extends Writer> template) { - templates.put(templateName,template); - } - - /** - * Sets the selected template - * - * @throws NullPointerException if the given template is null - */ - public void setTemplateNotNull(String templateName, Template<? extends Writer> template) { - Validator.ensureNotNull("Template "+templateName,template); - templates.put(templateName,template); - } - - - /** - * Sets the highligting marks for this template - * - * @param start the highlingting start mark - * @param end the highlingting end mark - * @param sep the highlingting separator mark - */ - public void setHighlightTags(String start, String end, String sep) { - boldOpenTag = start; - boldCloseTag = end; - separatorTag = sep; - } - - // may return null - public String getBoldOpenTag() { - return boldOpenTag; - } - - // may return null - public String getBoldCloseTag() { - return boldCloseTag; - } - - // may return null - public String getSeparatorTag() { - return separatorTag; - } - - - /** - * Set the default summary class to use with this template. - */ - public void setSummaryClass(String summaryClass) { - this.summaryClass = summaryClass; - } - - /** - * Type safe accessor to get the default document summary class for this - * template set. This is also here to insulate the rest of the code - * against changes in the naming of the properties in the property file. - */ - public String getSummaryClass() { - if (summaryClass != null && ! summaryClass.isEmpty()) { - return summaryClass; - } else { - return null; - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java deleted file mode 100644 index 5ef28d6f6c6..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.StructuredData; -import com.yahoo.text.XML; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; - -/** - * A context providing all the fields of a hit, and falls back to MapContext behavior for all other keys. - * - * @author Tony Vaagenes - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class HitContext extends Context { - - private final Hit hit; - private final Context fallbackContext; - - public HitContext(Hit hit, Context fallbackContext) { - this.hit = hit; - this.fallbackContext = fallbackContext; - } - - @Override - public Object put(String key, Object value) { - return fallbackContext.put(key, value); - } - - @Override - public Object get(String key) { - Object value = normalizedHitProperty(key); - return value != null ? - value : - fallbackContext.get(key); - } - - @Override - public Object remove(Object key) { - return fallbackContext.remove(key); - } - - @Override - public Collection<? extends Object> getKeys() { - Set<Object> keys = new HashSet<>(fallbackContext.getKeys()); - keys.addAll(hit.fieldKeys()); - return keys; - } - - @Override - public void setBoldOpenTag(String boldOpenTag) { - fallbackContext.setBoldOpenTag(boldOpenTag); - } - - @Override - public void setBoldCloseTag(String boldCloseTag) { - fallbackContext.setBoldCloseTag(boldCloseTag); - } - - @Override - public void setSeparatorTag(String separatorTag) { - fallbackContext.setSeparatorTag(separatorTag); - } - - @Override - public String getBoldOpenTag() { - return fallbackContext.getBoldOpenTag(); - } - - @Override - public String getBoldCloseTag() { - return fallbackContext.getBoldCloseTag(); - } - - @Override - public String getSeparatorTag() { - return fallbackContext.getSeparatorTag(); - } - - @Override - //TVT: TODO: Make this package private again. - public boolean isUtf8Output() { - return fallbackContext.isUtf8Output(); - } - - @Override - //TODO: TVT: make this package private again - public void setUtf8Output(boolean utf8Output) { - fallbackContext.setUtf8Output(utf8Output); - } - - @Override - public void setXmlEscape(boolean xmlEscape) { - fallbackContext.setXmlEscape(xmlEscape); - } - - @Override - public boolean getXmlEscape() { - return fallbackContext.getXmlEscape(); - } - - @Override - protected Object normalizeValue(Object value) { - return fallbackContext.normalizeValue(value); - } - - private Object normalizedHitProperty(String key) { - Object value = hit.getField(key); - return value == null ? - null : - normalizeHitFieldValue(value); - } - - private Object normalizeHitFieldValue(Object value) { - if (value instanceof HitField) { - HitField hf = (HitField) value; - if (getXmlEscape()) { - return hf.quotedContent(getBoldOpenTag(), - getBoldCloseTag(), - getSeparatorTag(), - true); - } else { - return hf.getContent(getBoldOpenTag(), - getBoldCloseTag(), - getSeparatorTag()); - } - } else if (value instanceof StructuredData) { - return value.toString(); - } else if (value instanceof XMLString || value instanceof JSONString) { - return value.toString(); - } else if (getXmlEscape()) { - return XML.xmlEscape(value.toString(), true, null); - } else { - return value.toString(); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/LogExceptionUserTemplateDelegator.java b/container-search/src/main/java/com/yahoo/prelude/templates/LogExceptionUserTemplateDelegator.java deleted file mode 100644 index 7696790897e..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/LogExceptionUserTemplateDelegator.java +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.Writer; -import java.util.Properties; -import java.util.logging.Logger; - -/** - * Delegates to another UserTemplate, but handles any exceptions(except IOException) by logging them. - * - * @author Tony Vaagenes - * @deprecated use a renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class LogExceptionUserTemplateDelegator<T extends Writer> extends UserTemplate<T> { - - private static Logger log = Logger.getLogger(LogExceptionUserTemplateDelegator.class.getName()); - private final UserTemplate<T> delegate; - - public LogExceptionUserTemplateDelegator(UserTemplate<T> delegate) { - super(LogExceptionUserTemplateDelegator.class.getSimpleName()); - this.delegate = delegate; - } - - @Override - public Context createContext() { - return delegate.createContext(); - } - - @Override - public T wrapWriter(Writer writer) { - return delegate.wrapWriter(writer); - } - - @Override - public boolean isDefaultTemplateSet() { - return delegate.isDefaultTemplateSet(); - } - - @Override - public String getSummaryClass() { - return delegate.getSummaryClass(); - } - - @Override - public String getBoldOpenTag() { - return delegate.getBoldOpenTag(); - } - - @Override - public String getBoldCloseTag() { - return delegate.getBoldCloseTag(); - } - - @Override - public String getSeparatorTag() { - return delegate.getSeparatorTag(); - } - - @Override - public void setSummaryClass(String summaryClass) { - delegate.setSummaryClass(summaryClass); - } - - @Override - public void setHighlightTags(String start, String end, String sep) { - delegate.setHighlightTags(start, end, sep); - } - - @Override - public String getName() { - return delegate.getName(); - } - - @Override - public String getMimeType() { - return delegate.getMimeType(); - } - - @Override - public String getEncoding() { - return delegate.getEncoding(); - } - - @Override - public Template<T> getTemplate(String templateName) { - throw new UnsupportedOperationException(); - } - - @Override - public void setTemplate(String templateName, Template<? extends Writer> template) { - throw new UnsupportedOperationException(); - } - - @Override - public void setTemplateNotNull(String templateName, Template<? extends Writer> template) { - throw new UnsupportedOperationException(); - } - - /*** Template - - @Override - public void <methodName>(Context context, T writer) throws IOException { - try { - delegate.<methodName>(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - ***/ - - /*** Begin expanded template for - header, footer, hit, hitFooter, error, noHits, queryContext, - Thanks java, for giving me the opportunely to use copy-paste ***/ - - - @Override - public void header(Context context, T writer) throws IOException { - try { - delegate.header(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void footer(Context context, T writer) throws IOException { - try { - delegate.footer(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void hit(Context context, T writer) throws IOException { - try { - delegate.hit(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void hitFooter(Context context, T writer) throws IOException { - try { - delegate.hitFooter(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void error(Context context, T writer) throws IOException { - try { - delegate.error(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void noHits(Context context, T writer) throws IOException { - try { - delegate.noHits(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void queryContext(Context context, T writer) throws IOException { - try { - delegate.queryContext(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - /*** End expanded template. ***/ - - private void handleException(Exception e) throws IOException { - if (e instanceof IOException) { - throw (IOException) e; - } else { - log.log(LogLevel.WARNING, "Exception thrown in " + getName() - + ": " + Exceptions.toMessageString(e), e); - } - } - - UserTemplate<T> getDelegate() { - return delegate; - } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java deleted file mode 100644 index c08c9701741..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * A context having a map as secondary storage - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class MapContext extends Context { - - private Map<String, Object> map = new LinkedHashMap<>(); - - @Override - public Object get(String key) { - return normalizeValue(map.get(key)); - } - - public Object put(String name, Object value) { - return map.put(name, value); - } - - public Object remove(Object name) { - return map.remove(name); - } - - @Override - public Collection<? extends Object> getKeys() { - return map.keySet(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java deleted file mode 100644 index cac06011679..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; - -/** - * A template implementing the 'page' format. - * This is a variant of the tiled template set - see that class for details. - * - * @author bratseth - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class PageTemplateSet extends TiledTemplateSet { - - public PageTemplateSet() { - super("page"); - } - - @Override - /** Uses an XML writer in this */ - public XMLWriter wrapWriter(Writer writer) { return new XMLWriter(super.wrapWriter(writer)); } - - @Override - public void header(Context context,XMLWriter writer) throws IOException { - Result result=(Result)context.get("result"); - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag("page").attribute("version","1.0").attribute("layout",result.hits().getField("layout")); - renderCoverageAttributes(result.getCoverage(false), writer); - writer.closeStartTag(); - renderSectionContent(result.hits(),writer); - } - - @Override - public void footer(Context context,XMLWriter writer) throws IOException { - if (writer.isIn("content")) - writer.closeTag(); - super.footer(context,writer); - } - - @Override - protected void renderSection(HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag("section"); - writer.attribute("id",hit.getDisplayId()); - writer.attribute("layout",hit.getField("layout")); - writer.attribute("region",hit.getField("region")); - writer.closeStartTag(); - renderSectionContent(hit,writer); - } - - @Override - public void hit(Context context, XMLWriter writer) throws IOException { - Hit hit = (Hit) context.get("hit"); - if (!hit.isMeta() && !writer.isIn("content")) - writer.openTag("content"); - super.hit(context,writer); - } - - @Override - public void hitFooter(Context context, XMLWriter writer) throws IOException { - if (writer.isIn("content")) - writer.closeTag(); - super.hitFooter(context, writer); - } - - public String toString() { return "page template"; } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java b/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java deleted file mode 100644 index 467cd615ebd..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.search.Result; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.rendering.Renderer; -import com.yahoo.search.result.*; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.processing.request.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.text.XMLWriter; - -import java.io.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; - -/** - * Renders a search result using the old templates API. - * - * @author Tony Vaagenes - * @deprecated do not use - */ -@SuppressWarnings({ "rawtypes", "deprecation", "unchecked" }) -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public final class SearchRendererAdaptor extends Renderer { - - private final LogExceptionUserTemplateDelegator templates; - - //Per instance members, must be created at rendering time, not construction time due to cloning. - private Context context; - - public SearchRendererAdaptor(UserTemplate userTemplate) { - templates = new LogExceptionUserTemplateDelegator(userTemplate); - } - - @Override - public void init() { - super.init(); - context = templates.createContext(); - } - - /** A legacy test utility - do not use. */ - public static void callRender(OutputStream stream, Result result) throws IOException { - Renderer rendererAdaptor = new SearchRendererAdaptor(result.getTemplating().getTemplates()); - rendererAdaptor.init(); - result.getTemplating().setRenderer(rendererAdaptor); - rendererAdaptor.render(stream, result, result.getQuery().getModel().getExecution(), result.getQuery()); - } - - @Override - public String getEncoding() { - return templates.getEncoding(); - } - - @Override - public String getMimeType() { - return templates.getMimeType(); - } - - @Override - public String getDefaultSummaryClass() { - return templates.getSummaryClass(); - } - - /** - * Renders this result - */ - public void render(Writer writer, Result result) throws java.io.IOException { - Writer wrappedWriter = wrapWriter(writer); - - beginResult(wrappedWriter, result); - - if (result.hits().getError() != null || result.hits().getQuery().errors().size() > 0) { - error(wrappedWriter, Collections.unmodifiableCollection( - all(result.hits().getQuery().errors(), result.hits().getError()))); - } - - if (result.getConcreteHitCount() == 0) { - emptyResult(wrappedWriter, result); - } - - if (result.getContext(false) != null) { - queryContext(wrappedWriter, result.getContext(false)); - } - - renderHitGroup(wrappedWriter, result.hits(), result.hits().getQuery().getOffset() + 1); - - endResult(wrappedWriter, result); - } - - - private <T> Collection<T> all(Collection<T> collection, T extra) { - Collection<T> result = new ArrayList<>(collection); - result.add(extra); - return result; - } - - - public Writer wrapWriter(Writer writer) { - return templates.wrapWriter(writer); - } - - - public void beginResult(Writer writer, Result result) throws IOException { - context.put("context", context); - context.put("result", result); - context.setBoldOpenTag(templates.getBoldOpenTag()); - context.setBoldCloseTag(templates.getBoldCloseTag()); - context.setSeparatorTag(templates.getSeparatorTag()); - - templates.header(context, writer); - } - - public void endResult(Writer writer, Result result) throws IOException { - templates.footer(context, writer); - } - - public void error(Writer writer, Collection<ErrorMessage> errorMessages) throws IOException { - templates.error(context, writer); - } - - - public void emptyResult(Writer writer, Result result) throws IOException { - templates.noHits(context, writer); - } - - public void queryContext(Writer writer, QueryContext queryContext) throws IOException { - templates.queryContext(context, writer); - } - - private void renderHitGroup(Writer writer, HitGroup hitGroup, int hitnumber) - throws IOException { - boolean defaultTemplate = templates.isDefaultTemplateSet(); - for (Hit hit : hitGroup.asList()) { - if (!defaultTemplate && hit instanceof ErrorHit) continue; // TODO: Stop doing this - - renderHit(writer, hit, hitnumber); - if (!hit.isAuxiliary()) - hitnumber++; - } - } - - - /** - * Renders this hit as xml. The default implementation will call the simpleRender() - * hook. If it returns true, nothing more is done, otherwise the - * given template set will be used for rendering. - * - * - * @param writer the writer to append this hit to - * @throws java.io.IOException if rendering fails - */ - public void renderHit(Writer writer, Hit hit, int hitno) throws IOException { - renderRegularHit(writer, hit, hitno); - } - - private void renderRegularHit(Writer writer, Hit hit, int hitno) throws IOException { - boolean renderedSimple = simpleRenderHit(writer, hit); - - if (renderedSimple) { - return; - } - - HitContext hitContext = new HitContext(hit, context); - hitContext.put("hit", hit); - hitContext.put("hitno", Integer.valueOf(hitno)); - hitContext.put("relevancy",hit.getRelevance()); - templates.hit(hitContext, writer); - - if (hit instanceof HitGroup) - renderHitGroup(writer, (HitGroup) hit, hitno); - - // Put these back - may have been changed by nested rendering - hitContext.put("hit", hit); - hitContext.put("hitno", Integer.valueOf(hitno)); - templates.hitFooter(hitContext, writer); - - - hitContext.remove("hit"); - hitContext.remove("hitno"); - } - - private boolean simpleRenderHit(Writer writer, Hit hit) throws IOException { - if (hit instanceof DefaultErrorHit) { - return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit); - } else if (hit instanceof GroupingListHit) { - return true; - } else { - return false; - } - } - - public static boolean simpleRenderDefaultErrorHit(Writer writer, ErrorHit defaultErrorHit) throws IOException { - XMLWriter xmlWriter=(writer instanceof XMLWriter) ? (XMLWriter)writer : new XMLWriter(writer,10,-1); - xmlWriter.openTag("errordetails"); - for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - renderMessageDefaultErrorHit(xmlWriter, error); - } - xmlWriter.closeTag(); - return true; - } - - public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException { - writer.openTag("error"); - if (error instanceof com.yahoo.search.result.ErrorMessage) - writer.attribute("source",((com.yahoo.search.result.ErrorMessage)error).getSource()); - writer.attribute("error",error.getMessage()); - writer.attribute("code",Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(),false); - if (error.getCause()!=null) { - writer.openTag("cause"); - writer.content("\n",true); - StringWriter stackTrace=new StringWriter(); - error.getCause().printStackTrace(new PrintWriter(stackTrace)); - writer.content(stackTrace.toString(),true); - writer.closeTag(); - } - writer.closeTag(); - } - - /** - * Renders this hit as XML, disregarding the given template. - * The main error will be rendered first, the all the following errors. - */ - public boolean simpleRenderErrorHit(Writer writer, com.yahoo.search.result.ErrorHit errorHit) throws IOException { - XMLWriter xmlWriter=(writer instanceof XMLWriter) ? (XMLWriter)writer : new XMLWriter(writer,10,-1); - xmlWriter.openTag("errordetails"); - for (Iterator i = errorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - rendererErrorHitMessageMessage(xmlWriter, errorHit, error); - } - xmlWriter.closeTag(); - return true; - } - - public static void rendererErrorHitMessageMessage(XMLWriter writer, com.yahoo.search.result.ErrorHit errorHit, ErrorMessage error) throws IOException { - writer.openTag("error"); - if (errorHit instanceof Hit) { - writer.attribute("source", ((Hit) errorHit).getSource()); - } - writer.attribute("error",error.getMessage()); - writer.attribute("code",Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(),false); - writer.closeTag(); - } - - /** - * For internal use only - */ - public UserTemplate getAdaptee() { - return templates.getDelegate(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java b/container-search/src/main/java/com/yahoo/prelude/templates/Template.java deleted file mode 100644 index 761eb1db562..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.io.Writer; - - -/** - * A template turns a template string and some state into - * an instantiated string. Add support for a particular - * template mechanism by subclassing this. - * - * @author bratseth - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public abstract class Template<T extends Writer> { - - /** - * Renders this template - * - * @param context the context to evaluate in - * @param writer the writer to render to - */ - public abstract void render(Context context,T writer) throws java.io.IOException; - - - /** - * Get template name - * - * @return template name - */ - public abstract String getName(); - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/TemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/TemplateSet.java deleted file mode 100644 index 65a5b4cacfd..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/TemplateSet.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.text.GenericWriter; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; - -/** - * <p>A template set contains instances of the various templates - * required to render a result.</p> - * - * <p>Normal usage is to create an instance and populate it with templates, - * but this class also supports subclassing to refine the behaviour, - * like returning different templates for different hit types.</p> - * - * @author bratseth - * @deprecated use a renderer instead - */ -@SuppressWarnings("deprecation") -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public class TemplateSet<T extends Writer> extends UserTemplate<T> { - - private static final String queryContextTemplateName = "queryContext"; - - private static final DefaultTemplateSet defaultTemplateSet=new DefaultTemplateSet(); - - /** - * Creates a template set containing no templates - * - * @param name the unique name of this template set, used for - * refering to it by clients - */ - public TemplateSet(String name, - String mimeType, - String encoding) { - super(name, mimeType,encoding); - } - - /** - * Returns the default template set. This is a template set which renders in - * the default xml format - */ - public static UserTemplate<XMLWriter> getDefault() { - return defaultTemplateSet; - } - - /** - * Returns the result header template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getHeader(Result result) { return (Template<T>) getTemplate("header"); } - - /** - * Sets the header template - * - * @param header the template to use for rendering getHeaders - * @throws NullPointerException if the given template is null - */ - public void setHeader(Template<T> header) { - setTemplateNotNull("header",header); - } - - /** - * Returns the result footer template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getFooter(Result result) { return (Template<T>) getTemplate("footer"); } - - /** - * Sets the footer template - * - * @param footer the template to use for rendering footers - * @throws NullPointerException if the given template is null - */ - public void setFooter(Template<T> footer) { - setTemplateNotNull("footer",footer); - } - - /** - * Returns the empty body template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getNohits(Result result) { return (Template<T>) getTemplate("nohits"); } - - - /** - * @return the template for rendering the query context, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getQueryContext(Result result) { - return (Template<T>) getTemplate(queryContextTemplateName); - } - - /** - * @param template The template to be used for rendering query contexts, never null. - */ - public void setQueryContext(Template<T> template) { - setTemplateNotNull(queryContextTemplateName, template); - } - - /** - * Sets the nohits template - * - * @param nohits the template to use for rendering empty results - * @throws NullPointerException if the given template is null - */ - public void setNohits(Template<T> nohits) { - setTemplateNotNull("nohits",nohits); - } - - /** - * Returns the error body template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getError(Result result) { return (Template<T>) getTemplate("error"); } - - /** - * Sets the error template - * - * @param error the template to use for rendering errors - * @throws NullPointerException if the given template is null - */ - public void setError(Template<T> error) { - setTemplateNotNull("error",error); - } - - /** - * Returns the hit template - * - * @param resultHit the hit which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getHit(Hit resultHit) { return (Template<T>) getTemplate("hit"); } - - /** - * Sets the hit template - * - * @param hit the template to use for rendering hits - * @throws NullPointerException if the given template is null - */ - public void setHit(Template<T> hit) { - setTemplateNotNull("hit",hit); - } - - /** - * Returns the hit footer template - * - * @param hit the hit which will use the template - * @return the template to use, or null if no hit footer is used - */ - @SuppressWarnings("unchecked") - public Template<T> getHitFooter(Hit hit) { return (Template<T>) getTemplate("hitfooter"); } - - public String toString() { - return "template set " + getName() + " of type " + getMimeType() + - " [header=" + getTemplate("header") + - ",footer=" + getTemplate("footer") + - ",nohits=" + getTemplate("nohits") + - ",error=" + getTemplate("error") + - ",hit=" + getTemplate("hit") + "]"; - } - - @Override - public void header(Context context, T writer) throws IOException { - getHeader(null).render(context, writer); - } - - @Override - public void footer(Context context, T writer) throws IOException { - getFooter(null).render(context, writer); - } - - @Override - public void hit(Context context, T writer) throws IOException { - getHit(null).render(context, writer); - } - - @Override - public void error(Context context, T writer) throws IOException { - getError(null).render(context, writer); - } - - @Override - public void hitFooter(Context context, T writer) throws IOException { - Template<T> hitFooter = getHitFooter(null); - if (hitFooter != null) - hitFooter.render(context, writer); - } - - @Override - public void noHits(Context context, T writer) throws IOException { - getNohits(null).render(context, writer); - } - - @Override - public void queryContext(Context context, T writer) throws IOException { - getQueryContext(null).render(context, writer); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java deleted file mode 100644 index 2fa6e14019e..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.FormattingOptions.SubtypeFieldWithPrefix; -import com.yahoo.search.Result; -import com.yahoo.search.pagetemplates.model.Renderer; -import com.yahoo.search.pagetemplates.model.Source; -import com.yahoo.search.pagetemplates.result.SectionHitGroup; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.StructuredData; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; -import java.util.Iterator; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * A template set which implements the 'tiled' format. - * - * This template implementation requires a few rules to be observed for it to work properly: - * <ul> - * <li>As hit fields are rendered as XML tag names, their name must be compatible with XML tag names.</li> - * <li>Results sections, meta section, provider tags are rendered based on hits having specific types (as in {@link Hit#types()}, - * see table below for a list of hit types that are needed in order for hits to render properly.</li> - * <li>Some fields inside hits corresponding to provider tags (/result/meta/provider) are formatted in a specific way, see provider fields formatting options - * below. Other fields are rendered the usual way.</li> - * </ul> - * - * <p>Hit types required for proper rendering</p> - * <table summary="Hit types required for proper rendering"> - * <tr><td>XML tag path</td><td>Required hit type</td></tr> - * <tr><td>/result/section</td><td>A hit group and have a "section" type</td></tr> - * <tr><td>/result/meta</td><td>A hit group and have a "meta" type</td></tr> - * <tr><td>/result/meta/provider</td><td>A hit that has a "logging" type</td></tr> - * </table> - * - * <p>Provider fields formatting options</p> - * <table summary="Provider fields formatting options"> - * <tr><td>Field</td><td>Formatting</td><td>Field type</td></tr> - * <tr><td>provider</td><td>name attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>scheme</td><td>scheme attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>host</td><td>host attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>port</td><td>port attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>path</td><td>path attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>status</td><td>result attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>latency_connect</td><td><latency type="connect"> tag</td><td>Provided by container</td></tr> - * <tr><td>latency_start</td><td><latency type="start"> tag</td><td>Provided by container</td></tr> - * <tr><td>latency_finish</td><td><latency type="finish"> tag</td><td>Provided by container</td></tr> - * <tr><td>query_param_*</td><td><parameter name="..."> tag</td><td>Provided by container</td></tr> - * <tr><td>header_*</td><td><header name="..."> tag</td><td>Provided by container</td></tr> - * <tr><td>response_header_*</td><td><response-header name="..."> tag</td><td>Provided by container</td></tr> - * <tr><td>count_first</td><td><count type="first"> tag</td><td>Provided by container</td></tr> - * <tr><td>count_last</td><td><count type="last"> tag</td><td>Provided by container</td></tr> - * <tr><td>count_total</td><td><count type="total"> tag</td><td>Provided by container</td></tr> - * <tr><td>count_deep</td><td><count type="deep"> tag</td><td>Provided by container</td></tr> - * <tr><td>queryattrs_xorronum</td><td><queryattrs name="xorronum"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>queryattrs_RankFeaturesRewriterAttr</td><td><queryattrs name="RankFeaturesRewriterAttr"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>queryattrs_intlannotator</td><td><queryattrs name="intlannotator"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>queryattrs_category</td><td><queryattrs name="category"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>wordcounts_*</td><td><wordcounts word="..."> tag</td><td>Provided by YST searcher</td></tr> - * </table> - * - * @author bratseth - * @author laboisse - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TiledTemplateSet extends DefaultTemplateSet { - - private FormattingOptions hitOptionsForProvider; - private FormattingOptions hitOptions; - - public TiledTemplateSet() { - this("tiled"); - } - - public TiledTemplateSet(String templateName) { - super(templateName); - - // Define formatting options that will be used by various rendering methods - hitOptions = new FormattingOptions(); - // Render provider field as an attribute, not as a regular field - hitOptions.formatFieldAsAttribute("provider", "provider"); - hitOptions.setFieldNotToRender("provider"); - - - // Define formatting options that will be used by various rendering methods, for /result/meta/provider tags - hitOptionsForProvider = new FormattingOptions(); - hitOptionsForProvider.formatFieldAsAttribute("provider", "name"); // Provider name is rendered a provider/@name - // hitOptionsForProvider.formatFieldAsAttribute("uri", "query"); // FIXME Issue with attribute formatting, keeping as regular field for now - hitOptionsForProvider.formatFieldAsAttribute("scheme", "scheme"); - hitOptionsForProvider.formatFieldAsAttribute("host", "host"); - hitOptionsForProvider.formatFieldAsAttribute("port", "port"); - hitOptionsForProvider.formatFieldAsAttribute("path", "path"); - hitOptionsForProvider.formatFieldAsAttribute("status", "result"); - // Latency fields are not defined using prefixes as we know all the field names and prefixes are expensive - hitOptionsForProvider.formatFieldWithSubtype("latency_connect", "latency", "type", "connect"); - hitOptionsForProvider.formatFieldWithSubtype("latency_start", "latency", "type", "start"); - hitOptionsForProvider.formatFieldWithSubtype("latency_finish", "latency", "type", "finish"); - // Must use prefix for query parameters - hitOptionsForProvider.formatFieldWithSubtype("query_param_", "parameter", "name"); - // Must use prefix for getHeaders - hitOptionsForProvider.formatFieldWithSubtype("header_", "header", "name"); - // Must use prefix for response getHeaders - hitOptionsForProvider.formatFieldWithSubtype("response_header_", "response-header", "name"); - // Count fields are not defined using prefixes as we know all the field names and prefixes are expensive - hitOptionsForProvider.formatFieldWithSubtype("count_first", "count", "type", "first"); - hitOptionsForProvider.formatFieldWithSubtype("count_last", "count", "type", "last"); - hitOptionsForProvider.formatFieldWithSubtype("count_total", "count", "type", "total"); - hitOptionsForProvider.formatFieldWithSubtype("count_deep", "count", "type", "deep"); - - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_xorronum", "queryattrs", "name", "xorronum"); - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_RankFeaturesRewriterAttr", "queryattrs", "name", "RankFeaturesRewriterAttr"); - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_intlannotator", "queryattrs", "name", "intlannotator"); - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_category", "queryattrs", "name", "category"); - - hitOptionsForProvider.formatFieldWithSubtype("wordcounts_", "wordcounts", "word"); - // Provider field should not be rendered in logging hits as we already have <provider name="..."> - hitOptionsForProvider.setFieldNotToRender("provider"); - } - - @Override - /** Uses an XML writer in this template */ - public XMLWriter wrapWriter(Writer writer) { return new XMLWriter(super.wrapWriter(writer)); } - - @Override - public void header(Context context,XMLWriter writer) throws IOException { - Result result=(Result)context.get("result"); - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag("result").attribute("version","1.0"); - writer.attribute("layout", result.hits().getField("layout")); - renderCoverageAttributes(result.getCoverage(false), writer); - writer.closeStartTag(); - renderSectionContent(result.hits(),writer); - } - - /** - * Augments default hit attributes rendering with formatting options. - * There's also a hacky part: if hit is actually a hit group, tries to use - * the 'type' field in place of the hit's type, to avoid having the 'group' hit type. - */ - @Override - protected void renderHitAttributes(Hit hit, XMLWriter writer) throws IOException { - if (hit instanceof HitGroup) { - String type = hit.types().stream().collect(Collectors.joining(" ")); - if ("group".equals(type)) - type = String.valueOf(hit.getField("type")); - writer.attribute("type", type); - } - else { - writer.attribute("type", hit.types().stream().collect(Collectors.joining(" "))); - } - - if (hit.getRelevance() != null) - writer.attribute("relevance", hit.getRelevance()); - writer.attribute("source", hit.getSource()); - - for (Map.Entry<String, String> attr : hitOptions.fieldsAsAttributes()) { - Object val = hit.getField(attr.getKey()); - if (val != null) - writer.attribute(attr.getValue(), String.valueOf(val)); - } - } - - @Override - protected void renderField(Context context, Hit hit, Map.Entry<String, Object> entry, XMLWriter writer) throws IOException { - String fieldName = entry.getKey(); - - if ( !shouldRenderField(hit, fieldName)) return; - - writer.openTag(fieldName); - renderFieldContent(context, hit, fieldName, writer); - writer.closeTag(); - } - - /** Renders all fields of the hit */ - @Override - protected void renderHitFields(Context context, Hit hit, XMLWriter writer) throws IOException { - renderId(hit.getId(), writer); - for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) { - Map.Entry<String, Object> entry = it.next(); - // Exclude fields that should not be rendered - if (hitOptions.shouldRenderField(entry.getKey())) - renderField(context, hit, entry, writer); - } - } - - @Override - protected boolean shouldRenderField(Hit hit, String fieldName) { - if (fieldName.equals("relevancy")) return false; - if (fieldName.equals("collapseId")) return false; - return true; - } - - /** - * Overrides {@link DefaultTemplateSet#hit(Context, Writer)} - * to print 'logging' type meta hits as /result/meta/provider tags. - * Fails back to {@code super.hit(context, writer)} in other cases. - */ - @Override - public void hit(Context context, XMLWriter writer) throws IOException { - Hit hit = (Hit) context.get("hit"); - if (hit.isMeta() && hit.types().contains("logging")) - renderProvider(context, hit, writer); - else - super.hit(context, writer); - } - - /** - * Overrides {@link DefaultTemplateSet#renderHitGroup(HitGroup, Context, XMLWriter)} - * for /result/section and /result/meta hit groups. - * Fails back to {@code super.renderHitGroup(hit, context, writer)} otherwise. - */ - @Override - protected void renderHitGroup(HitGroup hit, Context context, XMLWriter writer) throws IOException { - if (hit.types().contains("section")) { - renderSection(hit, writer); // Renders /result/section - } - else if (hit.types().contains("meta")) { - writer.openTag("meta"); // renders /result/meta - writer.closeStartTag(); - } - else { - super.renderHitGroup(hit, context, writer); - } - } - - /** - * Renders /result/section. - * Doesn't use {@link #renderHitAttributes(Hit, XMLWriter)}. - */ - protected void renderSection(HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag("section"); - writer.attribute("id",hit.getDisplayId()); - writer.attribute("layout",hit.getField("layout")); - writer.attribute("region",hit.getField("region")); - writer.attribute("placement",hit.getField("placement")); // deprecated in 5.0 - writer.closeStartTag(); - renderSectionContent(hit,writer); - } - - protected void renderSectionContent(HitGroup hit,XMLWriter writer) throws IOException { - if (hit instanceof SectionHitGroup) { // render additional information - SectionHitGroup sectionGroup=(SectionHitGroup)hit; - for (Source source : sectionGroup.sources()) { - writer.openTag("source").attribute("url",source.getUrl()); - renderParameters(source.parameters(),writer); - writer.closeTag(); - } - for (Renderer renderer : sectionGroup.renderers()) { - writer.openTag("renderer").attribute("for",renderer.getRendererFor()).attribute("name",renderer.getName()); - renderParameters(renderer.parameters(),writer); - writer.closeTag(); - } - } - } - - private void renderParameters(Map<String,String> parameters,XMLWriter writer) throws IOException { - // Render content - for (Map.Entry<String, String> parameter : parameters.entrySet()) - writer.openTag("parameter").attribute("name",parameter.getKey()).content(parameter.getValue(),false).closeTag(); - } - - /** - * Renders /result/meta/provider. - * Uses {@link #renderProviderHitAttributes(Hit, XMLWriter)} instead of the default {@link #renderHitAttributes(Hit, XMLWriter)}. - * @see #renderProviderHitAttributes(Hit, XMLWriter) - * @see #renderProviderHitFields(Context, Hit, XMLWriter) - */ - protected void renderProvider(Context context, Hit hit, XMLWriter writer) - throws IOException { - writer.openTag("provider"); - renderProviderHitAttributes(hit, writer); - writer.closeStartTag(); - renderProviderHitFields(context, hit, writer); - } - - /** - * Specific hit attributes rendering for 'provider' meta hits under /result/meta. - */ - protected void renderProviderHitAttributes(Hit hit, XMLWriter writer) throws IOException { - // Browse through fields that should be rendered as attributes - for (Map.Entry<String, String> attr : hitOptionsForProvider.fieldsAsAttributes()) - writer.attribute(attr.getValue(),hit.getField(attr.getKey())); - } - - - /** - * Renders fields under /result/meta/provider. - * - * @see #renderProviderField(Context, Hit, java.util.Map.Entry, XMLWriter) - */ - protected void renderProviderHitFields(Context context, Hit hit, XMLWriter writer) - throws IOException { - renderId(hit.getId(), writer); - for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) { - Map.Entry<String, Object> entry = it.next(); - // Exclude fields that have already been rendered as attributes and - // fields that should not be rendered - if (hitOptionsForProvider.getAttributeName(entry.getKey()) == null - && hitOptionsForProvider.shouldRenderField(entry.getKey())) - renderProviderField(context, hit, entry, writer); - } - } - - /** - * Renders one field under /result/meta/provider. - */ - protected void renderProviderField(Context context, Hit hit, - Map.Entry<String, Object> entry, XMLWriter writer) throws IOException { - - String name = entry.getKey(); - FormattingOptions.SubtypeField subtypeField = hitOptionsForProvider.getSubtype(name); - if (subtypeField == null) - subtypeField = hitOptionsForProvider.getSubtypeWithPrefix(name); - - if (subtypeField != null) { - writer.openTag(subtypeField.tagName); - if (subtypeField.attributeValue != null) { - writer.attribute(subtypeField.attributeName,subtypeField.attributeValue); - } - else if (subtypeField instanceof SubtypeFieldWithPrefix) { - // This is a subtype field that was defined using a prefix - // get the remaining part of the field name - writer.attribute(subtypeField.attributeName, - name.substring(((SubtypeFieldWithPrefix)subtypeField).prefixLength)); - } - } else { - writer.openTag(name); - } - writer.escapedContent(asXML(hit.getField(name)),false).closeTag(); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - public String toString() { return "tiled result template"; } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/UserTemplate.java b/container-search/src/main/java/com/yahoo/prelude/templates/UserTemplate.java deleted file mode 100644 index bcdf3311c1f..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/UserTemplate.java +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.io.ByteWriter; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.XMLField; -import com.yahoo.search.Result; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; -import java.util.logging.Logger; - - -/** - * A wrapper for a template set, suitable for subclassing. - * - * <p> - * A subclass of UserTemplate must implement header(), footer(), hit(), - * hitFooter(), error() and noHits(). - * - * @deprecated use a renderer instead - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public abstract class UserTemplate<T extends Writer> extends GenericTemplateSet { - - // & - private static final byte[] ampersand = new byte[] { 38, 97, 109, 112, 59 }; - - // < - private static final byte[] lessThan = new byte[] { 38, 108, 116, 59 }; - // > - private static final byte[] greaterThan = new byte[] { 38, 103, 116, 59 }; - - // \\u00 - private static final byte[] quotePrefix = new byte[] { 92, 117, 48, 48 }; - - private static final Logger log = Logger.getLogger(UserTemplate.class.getName()); - - /** - * The signature of this constructor is the one which is invoked - * in a production setting. - */ - public UserTemplate(String name, String mimeType, - String encoding) { - super(name, mimeType, encoding); - } - - public UserTemplate(String name) { - this(name, - DEFAULT_MIMETYPE, - DEFAULT_ENCODING - ); - } - - /** - * This is called once before each result is rendered using this template. - * The returned writer is used in all subsequent calls. Use this if another (wrapper) - * writer of the raw incoming writer is desired in the implementation of this template. - * The class of the returned type must be given as a type argument to the template class, - * to be able to implement methods taking this wrapper writer as the argument type. - * This default implementation returns an XMLWriter. - */ - @SuppressWarnings("unchecked") - public T wrapWriter(Writer writer) { - //FIXME: Hack - return (T) XMLWriter.from(writer, 10, -1); - } - - /** - * Creates a new context suitable for this template. - * The context may be reused for several evaluations, but not multiple - * concurrent evaluations - */ - public Context createContext() { - return new MapContext(); - } - - - /** - * For internal use only - * TODO: get rid of this method * - */ - public boolean isDefaultTemplateSet() { - return getClass().equals(TemplateSet.getDefault().getClass()); - } - - /** - * Render the result set header. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, the result - * set instance - * @param writer - * the destination for rendering the result - * @throws IOException - * may be propagated from the writer - */ - public abstract void header(Context context, T writer) - throws IOException; - - /** - * Render the result set footer. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, the result - * set instance - * @param writer - * the destination for rendering the result - * @throws IOException - * may be propagated from the writer - */ - public abstract void footer(Context context, T writer) - throws IOException; - - /** - * Render a single top level hit. - * - * <p> - * The result set is available in the context object under the name - * "result". The hit itself as "hit", the index of the hit as "hitno", and - * all the fields under their normal names. - * - * @param context - * wrapper which will contain, among other thing, the hit - * instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void hit(Context context, T writer) throws IOException; - - /** - * Render a footer for a single top level hit. A typical implementation may - * do nothing. - * - * <p> - * The result set is available in the context object under the name - * "result". The hit itself as "hit", the index of the hit as "hitno", and - * all the fields under their normal names. - * - * @param context - * wrapper which will contain, among other thing, the hit - * instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void hitFooter(Context context, T writer) - throws IOException; - - /** - * Render the error message for a result set. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, main error - * and result set instances. - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void error(Context context, T writer) - throws IOException; - - /** - * Invoked when the result set has no hits. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, the result - * set instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void noHits(Context context, T writer) - throws IOException; - - /** - * Override this to add custom rendering for the query context of the result. - * Only called when the query context is present. - * - * <p> - * The result set is available in the context object under the name - * "result". The query context is retrieved from the result by calling - * result.getQuery.getContext(false) - * - * @param context - * wrapper which will contain, among other things, the result - * set instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public void queryContext(Context context, T writer) throws IOException { - Result result = (Result) context.get("result"); - result.getContext(false).render(writer); - } - - /** - * Dump UTF-8 byte array to writer, but escape low ASCII codes except - * TAB, NL and CR, and escape ampersand, less than and greater than. - * - * <p> - * It is presumed the writer is buffered (which is the case in normal - * result rendering), as the method may perform a large number of write - * operations. - * - * <p> - * public only for testing. - */ - public static void dumpAndXMLQuoteUTF8(ByteWriter writer, byte[] utf) throws java.io.IOException { - int startDump = 0; - - for (int i = 0; i < utf.length; ++i) { - byte b = utf[i]; - if (b < 0) { - // Not ASCII, above character 127 - // Don't try to do something smart with UNICODE characters, - // just pass them through. - } else if (b < 32) { - switch (b) { - case 9: - case 10: - case 13: - break; - default: - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - quoteByte(writer, b); - break; - } - } else { - // printable ASCII - // quote special characters, otherwise do nothing - switch (b) { - // case 34: // double quote - // writer.append(utf, startDump, i - startDump); - // startDump = i + 1; - // writer.append(doubleQuote); - // break; - case 38: // ampersand - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - writer.append(ampersand); - break; - case 60: // less than - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - writer.append(lessThan); - break; - case 62: // greater than - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - writer.append(greaterThan); - break; - } - } - } - if (startDump < utf.length) { - writer.append(utf, startDump, utf.length - startDump); - } - } - - /** - * If the field is available as a UTF-8 byte array, - * dump it to the writer. - */ - public static boolean dumpBytes(ByteWriter writer, - FastHit hit, - String fieldName) throws java.io.IOException { - return false; - } - - private static void quoteByte(ByteWriter writer, byte b) throws java.io.IOException { - byte[] quoted = new byte[2]; - writer.append(quotePrefix); - quoted[0] = (byte) ((b >>> 4) + 0x30); - if (quoted[0] > 0x39) { - quoted[0] = (byte) (quoted[0] + 7); - } - quoted[1] = (byte) ((b & 0x0f) + 0x30); - if (quoted[1] > 0x39) { - quoted[1] = (byte) (quoted[1] + 7); - } - writer.append(quoted); - } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/package-info.java b/container-search/src/main/java/com/yahoo/prelude/templates/package-info.java deleted file mode 100644 index 2363a12c78d..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -@PublicApi -package com.yahoo.prelude.templates; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index bf0920a2aa5..73335037b94 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -139,7 +139,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // The timeout to be used when dumping rank features private static final long dumpTimeout = (6 * 60 * 1000); // 6 minutes - private static final long defaultTimeout = 5000; + private static final long defaultTimeout = 500; /** The timeout of the query, in milliseconds */ private long timeout = defaultTimeout; @@ -527,17 +527,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public boolean requestHasProperty(String name) { - return httpRequest.hasProperty(name); - } - - /** * Returns the number of milliseconds to wait for a response from a search backend - * before timing it out. Default is 5000. + * before timing it out. Default is 500. * <p> * Note: If Ranking.RANKFEATURES is turned on, this is hardcoded to 6 minutes. * @@ -549,7 +540,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Sets the number of milliseconds to wait for a response from a search backend - * before time out. Default is 5000. + * before time out. Default is 500. */ public void setTimeout(long timeout) { if (timeout > 1000000000 || timeout < 0) @@ -639,24 +630,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setHits(hits); } - /** - * This is ignored - compression is controlled at the network level. - * - * @deprecated this is ignored - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setCompress(boolean ignored) { } - - /** - * Returns false. - * - * @deprecated this always returns false - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public boolean getCompress() { return false; } - /** Returns a string describing this query */ @Override public String toString() { @@ -885,7 +858,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { yql.append(" limit ").append(Integer.toString(getHits())); } } - if (getTimeout() != 5000L) { + if (getTimeout() != defaultTimeout) { yql.append(" timeout ").append(Long.toString(getTimeout())); } yql.append(';'); @@ -1009,22 +982,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public HttpRequest getHttpRequest() { return httpRequest; } - /** - * Returns the unique and stable session id of this query. - * - * @param create if true this is created if not already set - * @return the session id of this query, or null if not set and create is false - * @deprecated use getSessionId() or getSessionId(serverId) instead - */ - @Deprecated - public SessionId getSessionId(boolean create) { - if ( ! create) return getSessionId(); - - if (requestId == null) - requestId = UniqueRequestId.next(); - return new SessionId(requestId, getRanking().getProfile()); - } - /** Returns the session id of this query, or null if none is assigned */ public SessionId getSessionId() { if (requestId == null) return null; diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index e70916369a3..364e60e6263 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -48,11 +48,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ private ListMap<String,String> headers = null; - /** - * Result rendering infrastructure. - */ - private final Templating templating; - /** Creates a new Result where the top level hit group has id "toplevel" */ public Result(Query query) { this(query, new HitGroup("toplevel")); @@ -75,7 +70,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone if (query.getRanking().getSorting() != null) { setHitOrderer(new HitSortOrderer(query.getRanking().getSorting())); } - templating = new Templating(this); } /** Create a result containing an error */ @@ -92,8 +86,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ @SuppressWarnings("deprecation") public void mergeWith(Result result) { - if (templating.usesDefaultTemplate()) - templating.setRenderer(result.templating.getRenderer()); totalHitCount += result.getTotalHitCount(); deepHitCount += result.getDeepHitCount(); timeAccountant.merge(result.getElapsedTime()); @@ -228,7 +220,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone resultClone.hits = hits.clone(); - resultClone.getTemplating().setRenderer(null); // TODO: Remove on Vespa 7 resultClone.setElapsedTime(new ElapsedTime()); return resultClone; } @@ -299,9 +290,7 @@ public final class Result extends com.yahoo.processing.Response implements Clone * @param hit * the hit to be analyzed */ - protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) { - return; - } + protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) { } /** Returns the context of this result - this is equal to getQuery().getContext(create) */ public QueryContext getContext(boolean create) { return getQuery().getContext(create); } @@ -337,17 +326,4 @@ public final class Result extends com.yahoo.processing.Response implements Clone return headers; } - /** - * The Templating object contains helper methods and data containers for - * result rendering. - * - * @return helper object for result rendering - * @deprecated use renderers - */ - // TODO: Remove on Vespa 7 - @Deprecated// OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) - public Templating getTemplating() { - return templating; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 5de0c5eff74..ac0c8375f04 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -35,13 +35,6 @@ public class ClusterMonitor<T> { /** A map from Node to corresponding MonitoredNode */ private final Map<T, BaseNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); - /** @deprecated use the constructor with just the first argument instead */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public ClusterMonitor(NodeManager<T> manager, String ignored) { - this(manager); - } - public ClusterMonitor(NodeManager<T> manager) { nodeManager = manager; monitorThread = new MonitorThread("search.clustermonitor"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java index fb72e8c77c9..578c447dfbe 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java @@ -110,7 +110,7 @@ public class RpcFillInvoker extends FillInvoker { Query query = result.getQuery(); String rankProfile = query.getRanking().getProfile(); byte[] serializedSlime = BinaryFormat - .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(false), hits)); + .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(), hits)); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(compression, serializedSlime); resourcePool.client().getDocsums(hits, node, compressionResult.type(), serializedSlime.length, compressionResult.data(), diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java index 8ab80ec17dd..bf626312b13 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java @@ -3,7 +3,6 @@ package com.yahoo.search.dispatch; import com.google.common.collect.ImmutableMap; import com.yahoo.compress.Compressor; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; @@ -52,9 +51,8 @@ public class RpcResourcePool { if (query.properties().getBoolean(dispatchSummaries, true) && ! searcher.summaryNeedsQuery(query) && query.getRanking().getLocation() == null - && ! searcher.getCacheControl().useCache(query) - && ! legacyEmulationConfigIsSet(documentDb)) { - + && ! searcher.getCacheControl().useCache(query)) + { return Optional.of(new RpcFillInvoker(this, documentDb)); } else { return Optional.empty(); @@ -66,17 +64,6 @@ public class RpcResourcePool { return new RpcFillInvoker(this, documentDb); } - private boolean legacyEmulationConfigIsSet(DocumentDatabase db) { - LegacyEmulationConfig config = db.getDocsumDefinitionSet().legacyEmulationConfig(); - if (config.forceFillEmptyFields()) - return true; - if (config.stringBackedFeatureData()) - return true; - if (config.stringBackedStructuredData()) - return true; - return false; - } - public Compressor compressor() { return compressor; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java deleted file mode 100644 index 1607d108722..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.Collections; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - - -/** - * Superclass for http client searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Forwards to the next in chain fill(result,summaryName) */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - execution.fill(result,summaryName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java deleted file mode 100644 index 9f99a790df0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import java.util.Collections; - - -/** - * Superclass for http provider searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - } - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager); - } - - /** - * Override this to provider multi-phase result filling towards a backend. - * This default implementation does nothing. - */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java deleted file mode 100644 index 0fa235709f9..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * Some static helper classes for configured*Searcher classes - * - * @author bratseth - */ -// TODO: Remove on Vespa 7 -class ConfiguredSearcherHelper { - - /** No instantiation */ - private ConfiguredSearcherHelper() { } - - public static List<Connection> toConnectionList(ProviderConfig providerConfig) { - List<Connection> connections=new ArrayList<>(); - for(ProviderConfig.Node node : providerConfig.node()) { - connections.add(new Connection(node.host(), node.port())); - } - return connections; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java deleted file mode 100644 index 4f624a12547..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Represents a connection to a particular node (host/port). - * Right now this is just a container of connection parameters, but might be extended to - * contain an open connection later. - * The host and port state is immutable. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class Connection { - - private String host; - private int port; - - public Connection(String host,int port) { - this.host=host; - this.port=port; - } - - public String getHost() { return host; } - - public int getPort() { return port; } - - public String toString() { - return "http connection '" + host + ":" + port + "'"; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java deleted file mode 100644 index 0407ddeca1b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import org.apache.http.HttpEntity; -import org.apache.http.entity.HttpEntityWrapper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.GZIPInputStream; - -/** - * Used by HTTPSearcher when talking to services returning compressed content. - * - * @author Mainak Mandal - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GzipDecompressingEntity extends HttpEntityWrapper { - - private static class Resources { - - byte [] buffer; - int total; - - Resources() { - total = 0; - buffer = new byte[65536]; - } - void drain(InputStream zipStream) throws IOException { - int numRead = zipStream.read(buffer, total, buffer.length); - while (numRead != -1) { - total += numRead; - if ((total + 65536) > buffer.length) { - buffer = Arrays.copyOf(buffer, buffer.length + numRead); - } - numRead = zipStream.read(buffer, total, buffer.length - total); - } - } - - } - - private final Resources resources = new Resources(); - - public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException { - super(entity); - GZIPInputStream gz = new GZIPInputStream(entity.getContent()); - InputStream zipStream = new BufferedInputStream(gz); - try { - resources.drain(zipStream); - } catch (IOException e) { - throw e; - } finally { - zipStream.close(); - } - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - - final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total); - return new InputStream() { - - @Override - public int available() throws IOException { - return buff.remaining(); - } - - @Override - public int read() throws IOException { - if (buff.hasRemaining()) - return buff.get() & 0xFF; - - return -1; - } - - @Override - public int read(byte[] b) throws IOException { - if (!buff.hasRemaining()) - return -1; - - int len = b.length; - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, 0, len); - return len; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, off, len); - return len; - } - - @Override - public long skip(long n) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (n > buff.remaining()) - n = buff.remaining(); - - buff.position(buff.position() + (int) n); - return n; - } - }; - } - - @Override - public long getContentLength() { - return resources.total; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - outstream.write(resources.buffer, 0, resources.total); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java deleted file mode 100644 index ee093adc6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * A utility parent for searchers which gets data from web services which is incorporated into the query. - * This searcher will take care of implementing the search method while the extending class implements - * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, is this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPClientSearcher extends HTTPSearcher { - - static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta"); - - protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName()); - - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param path the path portion of the url to be used - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) { - super(id, connections, path, statistics); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics, - CertificateStore certificateStore) { - super(id, connections, path, statistics, certificateStore); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) { - super(id, connections, parameters, statistics); - } - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - * @param certificateStore the certificate store to use to pass certificates in requests - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - } - - /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */ - @Override - public Result robustSearch(Query query, Execution execution, Connection connection) { - return search(query,execution,connection); - } - - /** Implements a search towards the connection chosen by the cluster searcher for this query */ - @Override - public Result search(Query query, Execution execution, Connection connection) { - Hit requestMeta = doHttpRequest(query, connection); - Result result = execution.search(query); - result.hits().add(requestMeta); - return result; - } - - private Hit doHttpRequest(Query query, Connection connection) { - URI uri; - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - query.properties().set(REQUEST_META_CARRIER, requestMeta); - - query.trace("Created request information hit",false,9); - try { - uri = getURI(query, connection); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } - - HttpEntity entity; - try { - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - entity = getEntity(uri, requestMeta, query); - } catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " + - query + ": " + Exceptions.toMessageString(e))); - return requestMeta; - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in " - + this + " for " + query + ": " + e.getMessage())); - return requestMeta; - } - if (entity==null) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "No result from connecting to HTTP backend in " + this + " using " + connection + " for " + query)); - return requestMeta; - } - - try { - query = handleResponse(entity,query); - } - catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return requestMeta; - } - - /** Overrides to pass the query on to the next searcher */ - @Override - public Result search(Query query, Execution execution, ErrorMessage error) { - query.errors().add(error); - return execution.search(query); - } - - /** Do nothing on fill in client searchers */ - @Override - public void fill(Result result,String summaryClass,Execution execution,Connection connection) { - } - - /** - * Convenience hook for unmarshalling the response and adding the information to the query. - * Implement this or <code>handleResponse(entity,query)</code> in any subclass. - * This default implementation throws an exception. - * - * @param inputStream the stream containing the data from the http service - * @param contentLength the length of the content in the stream in bytes, or a negative number if not known - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - throw new UnsupportedOperationException("handleResponse must be implemented by " + this); - } - - /** - * Unmarshals the response and adds the resulting data to the given query. - * This default implementation calls - * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code> - * (and does some detailed query tracing). - * - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(HttpEntity entity, Query query) throws IOException { - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query = handleResponse(entity.getContent(), len, query); - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - return query; - } - - /** Never retry individual queries to clients for now */ - @Override - protected boolean shouldRetry(Query query,Result result) { return false; } - - /** - * numHits and offset should not be part of the cache key as cache supports - * partial read/write that is only one cache entry is maintained per query - * irrespective of the offset and numhits. - */ - public abstract Map<String, String> getCacheKey(Query q); - - /** - * Adds all key-values starting by "service." + getClientName() in query.properties(). - * Returns the empty map if {@link #getServiceName} is not overridden. - */ - @Override - public Map<String,String> getQueryMap(Query query) { - LinkedHashMap<String, String> queryMap=new LinkedHashMap<>(); - if (getServiceName().isEmpty()) return queryMap; - - for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName - queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString()); - return queryMap; - } - - /** - * Override this to return the name of the service this is a client of. - * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty. - * This default implementation returns "", which means service specific parameters will not be used. - */ - protected String getServiceName() { return ""; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java deleted file mode 100644 index a7b32678c1a..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.base.Preconditions; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnPerRouteBean; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * A set of parameters for talking to an http backend - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public final class HTTPParameters { - - public static final String RETRIES = "com.yahoo.search.federation.http.retries"; - - private boolean frozen=false; - - // All timing parameters below are in milliseconds - /** The url request path portion */ - private String path="/"; - private int connectionTimeout=2000; - private int readTimeout=5000; - private boolean persistentConnections=true; - private boolean enableProxy = false; - private String proxyHost = "localhost"; - private int proxyPort = 1080; - private String method = "GET"; - private String schema = "http"; - private String inputEncoding = "utf-8"; - private String outputEncoding = "utf-8"; - private int maxTotalConnections=10000; - private int maxConnectionsPerRoute=10000; - private int socketBufferSizeBytes=-1; - private int retries = 1; - private int configuredReadTimeout = -1; - private int configuredConnectionTimeout = -1; - private int connectionPoolTimeout = -1; - private String certificateProxy = null; - private int certificatePort = 0; - private String certificateApplicationId = null; - private boolean certificateUseProxy = false; - private long certificateTtl = 0L; - private long certificateRetry = 0L; - - private PingOption.Enum pingOption = PingOption.NORMAL; - - - private boolean followRedirects = true; - - public HTTPParameters() {} - - public HTTPParameters(String path) { - setPath(path); - } - - public HTTPParameters(ProviderConfig providerConfig) { - configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d); - configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d); - connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d); - retries = providerConfig.retries(); - setPath(providerConfig.path()); - certificateUseProxy = providerConfig.yca().useProxy(); - if (certificateUseProxy) { - certificateProxy = providerConfig.yca().host(); - certificatePort = providerConfig.yca().port(); - } - certificateApplicationId = providerConfig.yca().applicationId(); - certificateTtl = providerConfig.yca().ttl() * 1000L; - certificateRetry = providerConfig.yca().retry() * 1000L; - followRedirects = providerConfig.followRedirects(); - pingOption = providerConfig.pingOption(); - } - - /** - * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/". - * If a leading "/" is missing, it is added automatically. - */ - public final void setPath(String path) { - if (path==null || path.isEmpty()) path="/"; - - if (! path.startsWith("/")) - path="/" + path; - this.path = path; - } - - public PingOption.Enum getPingOption() { - return pingOption; - } - - public void setPingOption(PingOption.Enum pingOption) { - Preconditions.checkNotNull(pingOption); - ensureNotFrozen(); - this.pingOption = pingOption; - } - - /** Returns the url path. Default is "/". */ - public String getPath() { return path; } - - public boolean getFollowRedirects() { - return followRedirects; - } - - public void setFollowRedirects(boolean followRedirects) { - ensureNotFrozen(); - this.followRedirects = followRedirects; - } - - - public void setConnectionTimeout(int connectionTimeout) { - ensureNotFrozen(); - this.connectionTimeout=connectionTimeout; - } - - /** Returns the connection timeout in milliseconds. Default is 2000. */ - public int getConnectionTimeout() { return connectionTimeout; } - - public void setReadTimeout(int readTimeout) { - ensureNotFrozen(); - this.readTimeout=readTimeout; - } - - /** Returns the read timeout in milliseconds. Default is 5000. */ - public int getReadTimeout() { return readTimeout; } - - /** - * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true. - * The setting will change from sharing connections between threads to only reusing it within a thread - * but it is still reused.</b> - */ - public void setPersistentConnections(boolean persistentConnections) { - ensureNotFrozen(); - this.persistentConnections=persistentConnections; - } - - /** Returns whether this should use persistent connections. Default is true. */ - public boolean getPersistentConnections() { return persistentConnections; } - - /** Returns whether proxying should be enabled. Default is false. */ - public boolean getEnableProxy() { return enableProxy; } - - public void setEnableProxy(boolean enableProxy ) { - ensureNotFrozen(); - this.enableProxy=enableProxy; - } - - /** Returns the proxy type to use (if enabled). Default is "http". */ - public String getProxyType() { - return "http"; - } - - public void setProxyHost(String proxyHost) { - ensureNotFrozen(); - this.proxyHost=proxyHost; - } - - /** Returns the proxy host to use (if enabled). Default is "localhost". */ - public String getProxyHost() { return proxyHost; } - - public void setProxyPort(int proxyPort) { - ensureNotFrozen(); - this.proxyPort=proxyPort; - } - - /** Returns the proxy port to use (if enabled). Default is 1080. */ - public int getProxyPort() { return proxyPort; } - - public void setMethod(String method) { - ensureNotFrozen(); - this.method=method; - } - - /** Returns the http method to use. Default is "GET". */ - public String getMethod() { return method; } - - public void setSchema(String schema) { - ensureNotFrozen(); - this.schema=schema; - } - - /** Returns the schema to use. Default is "http". */ - public String getSchema() { return schema; } - - public void setInputEncoding(String inputEncoding) { - ensureNotFrozen(); - this.inputEncoding=inputEncoding; - } - - /** Returns the input encoding. Default is "utf-8". */ - public String getInputEncoding() { return inputEncoding; } - - public void setOutputEncoding(String outputEncoding) { - ensureNotFrozen(); - this.outputEncoding=outputEncoding; - } - - /** Returns the output encoding. Default is "utf-8". */ - public String getOutputEncoding() { return outputEncoding; } - - /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */ - public void freeze() { - frozen=true; - } - - private void ensureNotFrozen() { - if (frozen) throw new IllegalStateException("Cannot modify frozen " + this); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams() { - return toHttpParams(connectionTimeout, readTimeout); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams(int connectionTimeout, int readTimeout) { - HttpParams params = new BasicHttpParams(); - // force use of configured value if available - if (configuredConnectionTimeout > 0) { - HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout); - } else { - HttpConnectionParams.setConnectionTimeout(params, connectionTimeout); - } - if (configuredReadTimeout > 0) { - HttpConnectionParams.setSoTimeout(params, configuredReadTimeout); - } else { - HttpConnectionParams.setSoTimeout(params, readTimeout); - } - if (socketBufferSizeBytes > 0) { - HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes); - } - if (connectionPoolTimeout > 0) { - ConnManagerParams.setTimeout(params, connectionPoolTimeout); - } - ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections); - ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute)); - if (retries >= 0) { - params.setIntParameter(RETRIES, retries); - } - params.setParameter("http.protocol.handle-redirects", followRedirects); - return params; - } - - public int getMaxTotalConnections() { - return maxTotalConnections; - } - - public void setMaxTotalConnections(int maxTotalConnections) { - ensureNotFrozen(); - this.maxTotalConnections = maxTotalConnections; - } - - public int getMaxConnectionsPerRoute() { - return maxConnectionsPerRoute; - } - - public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { - ensureNotFrozen(); - this.maxConnectionsPerRoute = maxConnectionsPerRoute; - } - - public int getSocketBufferSizeBytes() { - return socketBufferSizeBytes; - } - - public void setSocketBufferSizeBytes(int socketBufferSizeBytes) { - ensureNotFrozen(); - this.socketBufferSizeBytes = socketBufferSizeBytes; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - ensureNotFrozen(); - this.retries = retries; - } - - public String getYcaProxy() { - return certificateProxy; - } - - public int getYcaPort() { - return certificatePort; - } - - public String getYcaApplicationId() { - return certificateApplicationId; - } - - public boolean getYcaUseProxy() { - return certificateUseProxy; - } - - public long getYcaTtl() { - return certificateTtl; - } - - public long getYcaRetry() { - return certificateRetry; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java deleted file mode 100644 index c72c2f26a1c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.collect.ImmutableList; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend - * over HTTP which is not supported by the platform out of the box. - * <p> - * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response. - * </p> - * - * @author Arne Bergene Fossaa - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPProviderSearcher extends HTTPSearcher { - - private final Counter emptyResults; - private final Value hitsPerQuery; - private final Value responseLatency; - private final Counter readTimeouts; - - private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider"); - - protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName()); - - /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */ - protected String cacheName=null; - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) { - this(id,connections,new HTTPParameters(path), statistics); - } - - /** Creates a http provider searcher using id.getName as provider name */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path, - Statistics statistics, CertificateStore certificateStore) { - this(id, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(id, connections, parameters, statistics, new ThrowingCertificateStore()); - } - - /** - * Creates a provider searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - String suffix = "_" + getId().getName().replace('.', '_'); - hitsPerQuery = new Value("hits_per_query" + suffix, statistics, - new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true)); - responseLatency = new Value(LOG_LATENCY_START + suffix, statistics, - new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); - emptyResults = new Counter("empty_results" + suffix, statistics, false); - readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false); - } - - /** @deprecated this method does nothing */ - @Deprecated // OK - protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) { - } - - /** - * Unmarshal the stream by converting it to hits and adding the hits to the given result. - * A convenience hook called by the default <code>unmarshal(entity,result).</code> - * Override this in subclasses which does not override <code>unmarshal(entity,result).</code> - * <p> - * This default implementation throws an exception. - * - * @param stream the stream of data returned - * @param contentLength the length of the content in bytes if known, or a negative number if unknown - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - throw new UnsupportedOperationException("Unmarshal must be implemented by " + this); - } - - /** - * Unmarshal the result from an http entity. This default implementation calls - * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code> - * (and does some detailed query tracing). - * - * @param entity the entity containing the data to unmarshal - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(HttpEntity entity,Result result) throws IOException { - Query query=result.getQuery(); - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query.trace("Unmarshaling result.", false, 6); - unmarshal(entity.getContent(), len, result); - - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - - } - - protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) { - Properties sourceProperties = FederationSearcher.getSourceProperties(query); - if (sourceProperties != null) { - for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) { - if (!excludedSourceProperties.contains(entry.getKey())) { - queryMap.put(entry.getKey(), entry.getValue().toString()); - } - } - } - } - - /** - * Hook called at the moment the result is returned from this searcher. This default implementation - * does <code>return result</code>. - * - * @param result the result which is to be returned - * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup) - * @param e the exception caused during execution of this query, or null if none - * @return the result which is returned upwards - */ - protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) { - return result; - } - - private Result statisticsBeforeInspection(Result result, Hit requestMeta, Exception e) { - int hitCount = result.getConcreteHitCount(); - if (hitCount == 0) { - emptyResults.increment(); - } - hitsPerQuery.put((double) hitCount); - - if (requestMeta != null) { - requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount)); - } - - return inspectAndReturnFinalResult(result, requestMeta, e); - } - - - @Override - protected void logResponseLatency(long latency) { - responseLatency.put((double) latency); - } - - @Override - public Result search(Query query, Execution execution,Connection connection) { - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - Result result = new Result(query); - result.hits().add(requestMeta); - query.trace("Created request information hit", false, 9); - - try { - URI uri = getURI(query, requestMeta, connection); - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - long requestStartTime = System.currentTimeMillis(); - - HttpEntity entity = getEntity(uri, requestMeta, query); - - // Why should consumeEntity call inspectAndReturnFinalResult itself? - // Seems confusing to me. - return entity == null - ? statisticsBeforeInspection(result, requestMeta, null) - : consumeEntity(entity, query, result, requestMeta, requestStartTime); - - } catch (MalformedURLException|URISyntaxException e) { - result.hits().addError(createMalformedUrlError(query,e)); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (TimeoutException e) { - result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this - + " for " + query + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, requestMeta, e); - } - } - - private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) { - - try { - // remove some time from timeout to allow for close calls with return result - unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result); - logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime); - return statisticsBeforeInspection(result, logHit, null); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, logHit, e); - } catch (TimeoutException e) { - readTimeouts.increment(); - result.hits().addError(ErrorMessage - .createTimeout("Timed out while reading/unmarshaling from backend in " - + this + " for " + query - + ": " + e.getMessage())); - return statisticsBeforeInspection(result, logHit, e); - } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors - cleanupHttpEntity(entity); - } - } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns the query.properties() prefixed by - * "source.[sourceName]" or "property.[propertyName]" - * (by calling {@link #addNonExcludedSourceProperties}). - */ - @Override - public Map<String,String> getQueryMap(Query query) { - Map<String,String> queryMap = super.getQueryMap(query); - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - /** - * @deprecated the cache key is ignored as there is no built-in caching support - */ - @Deprecated // OK - public abstract Map<String, String> getCacheKey(Query q); - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java deleted file mode 100644 index edf347bd84e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java +++ /dev/null @@ -1,961 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.inject.Inject; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.cluster.ClusterSearcher; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; - -import org.apache.http.*; -import org.apache.http.client.HttpClient; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnRoutePNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.DefaultHttpRoutePlanner; -import org.apache.http.impl.conn.SingleClientConnManager; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.HttpParams; -import org.apache.http.params.HttpProtocolParams; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.ExecutionContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpRequestExecutor; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLHandshakeException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; -import java.net.*; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Generic superclass of searchers making connections to some HTTP service. This - * supports clustered connections - a list of alternative servers may be given, - * requests will be hashed across these and failed over in case some are down. - * <p> - * This simply provides some utility methods for working with http connections - * and implements ping against the service. - * - * <p>This searcher contains code from the Apache httpcomponents client library, - * licensed to the Apache Software Foundation under the Apache License, Version - * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, as this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_HITCOUNT - * <dd> - * Number of concrete hits in the result returned by this provider. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - * - * @author Arne Bergene Fossaa - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPSearcher extends ClusterSearcher<Connection> { - - protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth"; - - private static final Charset iso8859Charset = Charset.forName("ISO-8859-1"); - - // Logging field name constants - public static final String LOG_PATH = "path"; - public static final String LOG_PORT = "port"; - public static final String LOG_HOST = "host"; - public static final String LOG_IP_ADDRESS = "ip_address"; - public static final String IP_ADDRESS_UNKNOWN = "unknown"; - - public static final String LOG_SCHEME = "scheme"; - public static final String LOG_URI = "uri"; - public static final String LOG_PROXY_PORT = "proxy_port"; - public static final String LOG_PROXY_HOST = "proxy_host"; - public static final String LOG_PROXY_TYPE = "proxy_type"; - public static final String LOG_STATUS = "status"; - public static final String LOG_LATENCY_FINISH = "latency_finish"; - public static final String LOG_LATENCY_START = "latency_start"; - public static final String LOG_LATENCY_CONNECT = "latency_connect"; - public static final String LOG_QUERY_PARAM_PREFIX = "query_param_"; - public static final String LOG_HEADER_PREFIX = "header_"; - public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_"; - public static final String LOG_HITCOUNT = "hit_count"; - public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_"; - public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_"; - - protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName()); - - /** The HTTP parameters to use. Assigned in the constructor */ - private HTTPParameters httpParameters; - - private final Counter connectTimeouts; - - /** Whether to use certificates */ - protected boolean useCertificate = false; - - private final CertificateStore certificateStore; - - /** The (optional) certificate application ID. */ - private String certificateApplicationId = null; - - /** The (optional) certificate server proxy */ - protected HttpHost certificateProxy = null; - - /** Certificate cache TTL in ms */ - private long certificateTtl = 0L; - - /** Certificate server retry rate in the cache if no cert is found, in ms */ - private long certificateRetry = 0L; - - /** Set at construction if this is using persistent connections */ - private ClientConnectionManager sharedConnectionManager = null; - - /** Set at construction if using non-persistent connections */ - private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null; - - private static final SchemeRegistry schemeRegistry = new SchemeRegistry(); - - static { - schemeRegistry.register(new Scheme("http", PlainSocketFactory - .getSocketFactory(), 80)); - schemeRegistry.register(new Scheme("https", SSLSocketFactory - .getSocketFactory(), 443)); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) { - this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore()); - } - - /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */ - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics, - CertificateStore certificateStore) { - this(componentId, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(componentId, connections, parameters, statistics, new ThrowingCertificateStore()); - } - /** - * Creates a http searcher - * - * @param componentId the id of this instance - * @param connections the connections to establish to the backend nodes - * @param parameters the http parameters to use. This object will be frozen if it isn't already - */ - @Inject - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(componentId,connections,false); - String suffix = "_" + getId().getName().replace('.', '_'); - - connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false); - - parameters.freeze(); - this.httpParameters = parameters; - this.certificateStore = certificateStore; - - if (parameters.getPersistentConnections()) { - HttpParams params=parameters.toHttpParams(); - HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); - ConnManagerParams.setTimeout(params, 10); - sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry); - Thread connectionPurgerThread = new Thread(() -> { - //this is the default value in yahoo jvm installations - long DNSTTLSec = 120; - while (true) { - try { - Thread.sleep(DNSTTLSec * 1000); - if (sharedConnectionManager == null) - continue; - - sharedConnectionManager.closeExpiredConnections(); - DNSTTLSec = Long.valueOf(java.security.Security - .getProperty("networkaddress.cache.ttl")); - //No DNS TTL, no need to close idle connections - if (DNSTTLSec <= 0) { - DNSTTLSec = 120; - continue; - } - sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS); - } catch (InterruptedException e) { - return; - } catch (NumberFormatException e) { - continue; - } - } - }); - connectionPurgerThread.setDaemon(true); - connectionPurgerThread.start(); - - } - else { - singleClientConnManagerThreadLocal =new ThreadLocal<>(); - } - - initializeCertificate(httpParameters, certificateStore); - } - - /** - * Initialize certificate store and proxy if they have been set to non-null, - * non-empty values. It will wrap thrown exceptions from the certificate store into - * RuntimeException and propagate them. - */ - private void initializeCertificate(HTTPParameters parameters, CertificateStore certificateStore) { - String applicationId = parameters.getYcaApplicationId(); - String proxy = parameters.getYcaProxy(); - int port = parameters.getYcaPort(); - long ttl = parameters.getYcaTtl(); - long retry = parameters.getYcaRetry(); - - if (applicationId != null && !applicationId.trim().isEmpty()) { - initializeCertificate(applicationId, ttl, retry, certificateStore); - } - - if (parameters.getYcaUseProxy()) { - initializeProxy(proxy, port); - } - } - - /** Returns the HTTP parameters used in this. This is always frozen */ - public HTTPParameters getParameters() { return httpParameters; } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns an empty LinkedHashMap. - */ - public Map<String,String> getQueryMap(Query query) { - return new LinkedHashMap<>(); - } - - /** - * Initialize the certificate. - * This will warn but not throw if certificates could not be loaded, as the certificates - * are external state which can fail independently. - */ - private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) { - try { - // get the certificate, i.e. init the cache and check integrity - String certificate = certificateStore.getCertificate(applicationId, ttl, retry); - if (certificate == null) { - getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'"); - return; - } - - this.useCertificate = true; - this.certificateApplicationId = applicationId; - this.certificateTtl = ttl; - this.certificateRetry = retry; - getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate); - } - catch (Exception e) { - getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" + - applicationId + "' in " + this, e); - } - } - - /** - * Initialize the certificate proxy setting. - */ - private void initializeProxy(String host, int port) { - certificateProxy = new HttpHost(host, port); - getLogger().log(LogLevel.CONFIG, "Proxy is configured; will use proxy: " + certificateProxy); - } - - /** - * Same a {@code getURI(query, offset, hits, null)}. - * @see #getURI(Query, Hit, Connection) - */ - protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException { - Hit requestMeta; - try { - requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER); - } catch (ClassCastException e) { - requestMeta = null; - } - return getURI(query, requestMeta, connection); - } - - /** - * Creates the URI for a query. - * Populates the {@code requestMeta} meta hit with the created URI HTTP properties. - * - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - */ - protected URI getURI(Query query, Hit requestMeta, Connection connection) - throws MalformedURLException, URISyntaxException { - StringBuilder parameters = new StringBuilder(); - - Map<String, String> queries = getQueryMap(query); - if (queries.size() > 0) { - Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator(); - parameters.append("?"); - try { - Map.Entry<String, String> entry; - while (mapIterator.hasNext()) { - entry = mapIterator.next(); - - if (requestMeta != null) - requestMeta.setField(LOG_QUERY_PARAM_PREFIX - + entry.getKey(), entry.getValue()); - - parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(), - httpParameters.getInputEncoding())); - if (mapIterator.hasNext()) { - parameters.append("&"); - } - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Unknown input encoding set in " + this, e); - } - } - - URI uri = new URL(httpParameters.getSchema(), connection.getHost(), - connection.getPort(), getPath() + parameters.toString()).toURI(); - if (requestMeta != null) { - requestMeta.setField(LOG_URI, uri.toString()); - requestMeta.setField(LOG_SCHEME, uri.getScheme()); - requestMeta.setField(LOG_HOST, uri.getHost()); - requestMeta.setField(LOG_PORT, uri.getPort()); - requestMeta.setField(LOG_PATH, uri.getPath()); - } - return uri; - } - - /** - * Called by getURI() to get the path of the URI for the external service. - * The default implementation returns httpParameters.getPath(); subclasses - * which only wants to override the path from httpParameters may use this - * method instead of overriding all of getURI(). - * - * @return the path to use for getURI - */ - protected String getPath() { - return httpParameters.getPath(); - } - - /** - * The URI that is used to check if the provider is up or down. This will again be used in the - * checkPing method by checking that we get a response that has a good status code (below 300). If better - * validation than just status code checking is needed, override the checkPing method. - */ - protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI(); - } - - /** - * Called by getPingURI() to get the path of the URI for pinging the - * external service. The default implementation returns - * httpParameters.getPath(); subclasses which only wants to override the - * path from httpParameters may use this method instead of overriding all of - * getPingURI(). - * - * @return the path to use for getPingURI - */ - protected String getPingPath() { - return httpParameters.getPath(); - } - - /** - * Checks if the response is valid. - * @param response The response from the ping request - * @param pong The pong result to return back to the calling method. This method - * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above. - */ - protected void checkPing(HttpResponse response, Pong pong) { - if (response.getStatusLine().getStatusCode() >= 300) { - pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError( - "Got error " + response.getStatusLine().getStatusCode() - + " when contacting backend") - ); - } - } - - /** - * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path. - * If this returns a status that is below 300, the ping is considered good. - * - * If another uri is needed for pinging, reimplement getPingURI. - * - * Override either this method to change how ping - */ - @Override - public Pong ping(Ping ping, Connection connection) { - URI uri = null; - Pong pong = new Pong(); - HttpResponse response = null; - - if (httpParameters.getPingOption() == PingOption.DISABLE) - return pong; - - try { - uri = getPingURI(connection); - if (uri == null) - pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null")); - if (uri.getHost()==null) { - pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host")); - uri=null; - } - } catch (MalformedURLException | URISyntaxException e) { - pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + - " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } - - if (uri == null) return pong; - pong.setPingInfo("using uri '" + uri + "'"); - - try { - response = getPingResponse(uri, ping); - checkPing(response, pong); - } catch (IOException e) { - // We do not have a valid ping - pong.addError(ErrorMessage.createBackendCommunicationError( - "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e))); - } catch (TimeoutException e) { - pong.addError(ErrorMessage.createTimeout("Timeout for ping " + uri + " in " + this + ": " + e.getMessage())); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } finally { - if (response != null) { - cleanupHttpEntity(response.getEntity()); - } - } - - return pong; - } - - private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException { - long timeLeft = ping.getTimeout(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - - Map<String, String> requestHeaders = null; - if (httpParameters.getPingOption() == PingOption.YCA) - requestHeaders = generateYCAHeaders(); - - return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout); - } - - /** - * Same a {@code getEntity(uri, null)}. - * @param uri resource to fetch - * @param query the originating query - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Query query) throws IOException{ - return getEntity(uri, null, query); - } - - - /** - * Gets the HTTP entity that holds the response contents. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query - * @return the http entity, or null if none - * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range. - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - if (query.getTimeLeft() <= 0) { - throw new TimeoutException("No time left for querying external backend."); - } - HttpResponse response = getResponse(uri, requestMeta, query); - StatusLine statusLine = response.getStatusLine(); - - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_STATUS, statusLine.getStatusCode()); - for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue()); - } - } - - if (statusLine.getStatusCode() >= 300) { - HttpEntity entity = response.getEntity(); - String message = createServerReporterErrorMessage(statusLine, entity); - cleanupHttpEntity(response.getEntity()); - throw new IOException(message); - } - - return response.getEntity(); - } - - private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) { - String message = "Error when trying to connect to HTTP backend: " - + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase(); - - try { - if (entity != null) { - message += "(Message = " + EntityUtils.toString(entity) + ")"; - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get message.", e); - } - - return message; - } - - /** - * Creates a meta hit dedicated to holding logging information. This hit has - * the 'logging:[searcher's ID]' type. - */ - protected Hit createRequestMeta() { - Hit requestMeta = new Hit("logging:" + getId().toString()); - requestMeta.setMeta(true); - requestMeta.types().add("logging"); - return requestMeta; - } - - protected void cleanupHttpEntity(HttpEntity entity) { - if (entity == null) return; - - try { - entity.consumeContent(); - } catch (IOException e) { - // It is ok if do not consume it, the resource will be freed after - // timeout. - // But log it just in case. - log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG), - "Not able to consume after processing: " + Exceptions.toMessageString(e)); - } - } - - /** - * Same as {@code getResponse(uri, null)}. - */ - protected HttpResponse getResponse(URI uri, Query query) throws IOException{ - return getResponse(uri, null, query); - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query, used to calculate timeouts - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException { - long timeLeft = query.getTimeLeft(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout; - readTimeout = readTimeout <= 0 ? 1 : readTimeout; - HttpEntity reqEntity = getRequestEntity(query, requestMeta); - Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta); - if ((reqEntity == null) && (reqHeaders == null)) { - return getResponse(uri, requestMeta, connectionTimeout, readTimeout); - } else { - return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout); - } - } - - /** - * Returns the set of headers to be passed in the http request to provider backend. The default - * implementation returns null, unless certificates are in use. If certificates are used, it will return a map - * only containing the needed certificate headers. - */ - protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) { - if (useCertificate) { - return generateYCAHeaders(); - } - return null; - } - - /** - * Returns the HTTP request entity to use when making the request for this query. - * This default implementation returns null. - * - * <p> Do return a repeatable entity if HTTP retry is active. - * - * @return the http request entity to use, or null to use the default entity - */ - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - return null; - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout) - throws IOException { - return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout); - } - - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, HttpEntity reqEntity, - Map<String, String> reqHeaders, Hit requestMeta, - int connectionTimeout, int readTimeout) throws IOException { - - HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout); - HttpClient httpClient = createClient(httpParams); - long start = 0L; - HttpUriRequest request; - if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) { - HttpHost proxy = new HttpHost(httpParameters.getProxyHost(), - httpParameters.getProxyPort(), httpParameters.getProxyType()); - httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType()); - requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost()); - requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort()); - } - } - if (reqEntity == null) { - request = createRequest(httpParameters.getMethod(), uri); - } else { - request = createRequest(httpParameters.getMethod(), uri, reqEntity); - } - - if (reqHeaders != null) { - for (Entry<String, String> entry : reqHeaders.entrySet()) { - if (entry.getValue() == null || isAscii(entry.getValue())) { - request.addHeader(entry.getKey(), entry.getValue()); - } else { - byte[] asBytes = Utf8.toBytes(entry.getValue()); - String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset); - request.addHeader(entry.getKey(), asLyingString); - } - } - } - - // Logging - if (requestMeta != null) { - for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue()); - } - start = System.currentTimeMillis(); - } - - HttpResponse response; - - try { - HttpContext context = new BasicHttpContext(); - response = httpClient.execute(request, context); - - if (requestMeta != null) { - requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context)); - } - } catch (ConnectTimeoutException e) { - connectTimeouts.increment(); - throw e; - } - - // Logging - long latencyStart = System.currentTimeMillis() - start; - if (requestMeta != null) { - requestMeta.setField(LOG_LATENCY_START, latencyStart); - } - logResponseLatency(latencyStart); - return response; - } - - private String getIpAddress(HttpContext context) { - HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION); - if (connection instanceof HttpInetConnection) { - InetAddress address = ((HttpInetConnection) connection).getRemoteAddress(); - String hostAddress = address.getHostAddress(); - return hostAddress == null ? - IP_ADDRESS_UNKNOWN: - hostAddress; - } else { - getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName()); - return IP_ADDRESS_UNKNOWN; - } - } - - private boolean isAscii(String value) { - char[] scanBuffer = new char[value.length()]; - value.getChars(0, value.length(), scanBuffer, 0); - for (char c: scanBuffer) - if (c > 127) return false; - return true; - } - - protected void logResponseLatency(long latency) { } - - /** - * Creates a http client for one request. Override to customize the client - * to use, e.g for testing. This default implementation will add a certificate store - * proxy to params if is necessary, and then do - * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code> - */ - protected HttpClient createClient(HttpParams params) { - if (certificateProxy != null) { - params.setParameter(ConnRoutePNames.DEFAULT_PROXY, certificateProxy); - } - return new SearcherHttpClient(getConnectionManager(params), params); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri) { - return createRequest(method, uri, null); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) { - return new SearcherHttpRequest(method,uri); - } - - /** Get a connection manager which may be used safely from this thread */ - protected ClientConnectionManager getConnectionManager(HttpParams params) { - if (sharedConnectionManager != null) {// We are using shared connections - return sharedConnectionManager; - } else { - SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get(); - if (singleClientConnManager == null) { - singleClientConnManager = new SingleClientConnManager(params, schemeRegistry); - singleClientConnManagerThreadLocal.set(singleClientConnManager); - } - return singleClientConnManager; - } - } - - /** Utility method for creating error messages when a url is incorrect */ - protected ErrorMessage createMalformedUrlError(Query query,Exception e) { - return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query + - ": " + Exceptions.toMessageString(e)); - } - - private Map<String, String> generateYCAHeaders() { - Map<String, String> headers = new HashMap<>(); - String certificate = certificateStore.getCertificate(certificateApplicationId, certificateTtl, certificateRetry); - headers.put(YCA_HTTP_HEADER, certificate); - return headers; - } - - protected static class SearcherHttpClient extends DefaultHttpClient { - - private final int retries; - - public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) { - super(conman, params); - retries = params.getIntParameter(HTTPParameters.RETRIES, 1); - addRequestInterceptor((request, context) -> { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - }); - addResponseInterceptor((response, context) -> { - HttpEntity entity = response.getEntity(); - if (entity == null) return; - Header ceheader = entity.getContentEncoding(); - if (ceheader == null) return; - for (HeaderElement codec : ceheader.getElements()) { - if (codec.getName().equalsIgnoreCase("gzip")) { - response.setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - } - }); - } - - @Override - protected HttpRequestExecutor createRequestExecutor() { - return new HttpRequestExecutor(); - } - - @Override - protected HttpRoutePlanner createHttpRoutePlanner() { - return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry()); - } - - @Override - protected HttpRequestRetryHandler createHttpRequestRetryHandler() { - return new SearcherHttpRequestRetryHandler(retries); - } - } - - /** A retry handler which avoids retrying forever on errors misclassified as transient */ - private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler { - private final int retries; - - public SearcherHttpRequestRetryHandler(int retries) { - this.retries = retries; - } - - @Override - public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) { - if (e == null) { - throw new IllegalArgumentException("Exception parameter may not be null"); - } - if (executionCount > retries) { - return false; - } - if (e instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (e instanceof InterruptedIOException) { - // Timeout from federation layer - return false; - } - if (e instanceof UnknownHostException) { - // Unknown host - return false; - } - if (e instanceof SSLHandshakeException) { - // SSL handshake exception - return false; - } - return true; - } - - - } - - private static class SearcherHttpRequest extends HttpRequestBase { - String method; - - public SearcherHttpRequest(String method, final URI uri) { - super(); - this.method = method; - setURI(uri); - } - - @Override - public String getMethod() { - return method; - } - } - - /** - * Only for testing. - */ - public void shutdownConnectionManagers() { - ClientConnectionManager manager; - if (sharedConnectionManager != null) { - manager = sharedConnectionManager; - } else { - manager = singleClientConnManagerThreadLocal.get(); - } - if (manager != null) { - manager.shutdown(); - } - } - - protected static final class ThrowingCertificateStore implements CertificateStore { - - @Override - public String getCertificate(String key, long ttl, long retry) { - throw new UnsupportedOperationException("A certificate store is not available"); - } - - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java deleted file mode 100644 index 03ffe2b8a9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; - -/** - * Wrapper for adding timeout to an HttpEntity instance. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedHttpEntity implements HttpEntity { - /** - * The wrapped entity. Never null. - */ - private final HttpEntity entity; - private final long startTime; - private final long timeout; - - public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) { - if (entity == null) { - throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity."); - } - this.entity = entity; - this.startTime = startTime; - this.timeout = timeout; - } - - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - InputStream content = entity.getContent(); - if (content == null) { - return null; - } else { - return new TimedStream(content, startTime, timeout); - } - } - - - // START OF PURE FORWARDING METHODS - @Override - public void consumeContent() throws IOException { - entity.consumeContent(); - } - - - @Override - public Header getContentEncoding() { - return entity.getContentEncoding(); - } - - @Override - public long getContentLength() { - return entity.getContentLength(); - } - - @Override - public Header getContentType() { - return entity.getContentType(); - } - - @Override - public boolean isChunked() { - return entity.isChunked(); - } - - @Override - public boolean isRepeatable() { - return entity.isRepeatable(); - } - - @Override - public boolean isStreaming() { - return entity.isStreaming(); - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - entity.writeTo(outstream); - } - // END OF PURE FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java deleted file mode 100644 index 77a42ee0a34..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; - -/** - * A stream which throws a TimeoutException if query timeout has been reached. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedStream extends InputStream { - - /** - * A time barrier value, the point in time from which on read operations will cause an exception. - */ - private final long limit; - - /** - * A wrapped InputStream instance. - */ - private final InputStream content; - - /** - * Wrap an InputStream to make read operations potentially fire off - * TimeoutException. - * - * <p>Typical use would be<br> - * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code> - * - * @param content - * the InputStream to wrap - * @param startTime - * start time of query - * @param timeout - * how long the query is allowed to run - */ - public TimedStream(InputStream content, long startTime, long timeout) { - if (content == null) { - throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream"); - } - this.content = content; - // The reasion for doing it in here instead of outside the constructor - // is this makes the usage of the class more intuitive IMHO - this.limit = startTime + timeout; - } - - private void checkTime(String message) { - if (System.currentTimeMillis() >= limit) { - throw new TimeoutException(message); - } - } - - // START FORWARDING METHODS: - // All methods below are forwarding methods to the contained stream, where - // some do a timeout check. - @Override - public int read() throws IOException { - int data = content.read(); - checkTime("Timed out during read()."); - return data; - } - - @Override - public int available() throws IOException { - return content.available(); - } - - @Override - public void close() throws IOException { - content.close(); - } - - @Override - public synchronized void mark(int readlimit) { - content.mark(readlimit); - } - - @Override - public boolean markSupported() { - return content.markSupported(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int length = content.read(b, off, len); - checkTime("Timed out during read(byte[], int, int)"); - return length; - } - - @Override - public int read(byte[] b) throws IOException { - int length = content.read(b); - checkTime("Timed out during read(byte[])"); - return length; - } - - @Override - public synchronized void reset() throws IOException { - content.reset(); - } - - @Override - public long skip(long n) throws IOException { - long skipped = content.skip(n); - checkTime("Timed out during skip(long)"); - return skipped; - } - // END FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java deleted file mode 100644 index 18dd59ec37b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Timeout marker for slow HTTP connections. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimeoutException extends RuntimeException { - - /** - * Auto-generated version ID. - */ - private static final long serialVersionUID = 7084147598258586559L; - - public TimeoutException(String message) { - super(message); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java deleted file mode 100644 index af334b35221..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.http; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java deleted file mode 100644 index 3a0db9b76de..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.util.Iterator; - -import com.yahoo.prelude.query.*; - -/** - * Marshal a query stack into an advanced query string suitable for - * passing to another QRS. - * - * @author Steinar Knutsen - * @author Rong-En Fan - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryMarshaller { - - private boolean atRoot = true; - - public String marshal(Item root) { - if (root == null || root instanceof NullItem) { - return null; - } - StringBuilder s = new StringBuilder(); - marshal(root, s); - atRoot = true; - return s.toString(); - } - - /** - * We do not yet care about exact match indices - */ - private void marshal(Item root, StringBuilder s) { - switch (root.getItemType()) { - case OR: - marshalOr((OrItem) root, s); - break; - case AND: - marshalAnd((CompositeItem) root, s); - break; - case NOT: - marshalNot((NotItem) root, s); - break; - case RANK: - marshalRank((RankItem) root, s); - break; - case WORD: - case INT: - case PREFIX: - case SUBSTRING: - case SUFFIX: - marshalWord((TermItem) root, s); - break; - case PHRASE: - // PhraseItem and PhraseSegmentItem don't add quotes for segmented - // termse - if (root instanceof PhraseSegmentItem) { - marshalPhrase((PhraseSegmentItem) root, s); - } else { - marshalPhrase((PhraseItem) root, s); - } - break; - case NEAR: - marshalNear((NearItem) root, s); - break; - case ONEAR: - marshalNear((ONearItem) root, s); - break; - case WEAK_AND: - marshalWeakAnd((WeakAndItem)root, s); - break; - default: - break; - } - } - - - private void marshalWord(TermItem item, StringBuilder s) { - String index = item.getIndexName(); - if (index.length() != 0) { - s.append(item.getIndexName()).append(':'); - } - s.append(item.stringValue()); - if (item.getWeight() != Item.DEFAULT_WEIGHT) - s.append("!").append(item.getWeight()); - } - - private void marshalRank(RankItem root, StringBuilder s) { - marshalComposite("RANK", root, s); - } - - private void marshalNot(NotItem root, StringBuilder s) { - marshalComposite("ANDNOT", root, s); - } - - private void marshalOr(OrItem root, StringBuilder s) { - marshalComposite("OR", root, s); - } - - /** - * Dump WORD items, and add space between each of them unless those - * words came from segmentation. - * - * @param root CompositeItem - * @param s current marshaled query - */ - private void dumpWords(CompositeItem root, StringBuilder s) { - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item word = i.next(); - boolean useSeparator = true; - if (word instanceof TermItem) { - s.append(((TermItem) word).stringValue()); - if (word instanceof WordItem) { - useSeparator = !((WordItem) word).isFromSegmented(); - } - } else { - dumpWords((CompositeItem) word, s); - } - if (useSeparator && i.hasNext()) { - s.append(' '); - } - } - } - - private void marshalPhrase(PhraseItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), false); - } - - private void marshalPhrase(PhraseSegmentItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), true); - } - - private void marshalPhrase(IndexedItem root, StringBuilder s, boolean isExplicit, boolean isSegmented) { - String index = root.getIndexName(); - if (index.length() != 0) { - s.append(root.getIndexName()).append(':'); - } - if (isExplicit || !isSegmented) s.append('"'); - dumpWords((CompositeItem) root, s); - if (isExplicit || !isSegmented) s.append('"'); - } - - private void marshalNear(NearItem root, StringBuilder s) { - marshalComposite(root.getName() + "(" + root.getDistance() + ")", root, s); - } - - // Not only AndItem returns ItemType.AND - private void marshalAnd(CompositeItem root, StringBuilder s) { - marshalComposite("AND", root, s); - } - - private void marshalWeakAnd(WeakAndItem root, StringBuilder s) { - marshalComposite("WAND(" + root.getN() + ")", root, s); - } - - private void marshalComposite(String operator, CompositeItem root, StringBuilder s) { - boolean useParen = !atRoot; - if (useParen) { - s.append("( "); - } else { - atRoot = false; - } - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item item = i.next(); - marshal(item, s); - if (i.hasNext()) - s.append(' ').append(operator).append(' '); - } - if (useParen) { - s.append(" )"); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java deleted file mode 100644 index a6aec30b496..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java +++ /dev/null @@ -1,642 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.Relevance; -import com.yahoo.text.XML; -import org.xml.sax.*; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import static com.yahoo.text.Lowercase.toLowerCase; - -/** - * Parse Vespa XML results and create Result instances. - * - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -@SuppressWarnings("deprecation") -public class ResultBuilder extends DefaultHandler { - private static final String ERROR = "error"; - - private static final String FIELD = "field"; - - private static Logger log = Logger.getLogger(ResultBuilder.class.getName()); - - /** Namespaces feature id (http://xml.org/sax/features/namespaces). */ - protected static final String NAMESPACES_FEATURE_ID = "http://xml.org/sax/features/namespaces"; - - /** - * Namespace prefixes feature id - * (http://xml.org/sax/features/namespace-prefixes). - */ - protected static final String NAMESPACE_PREFIXES_FEATURE_ID = "http://xml.org/sax/features/namespace-prefixes"; - - /** Validation feature id (http://xml.org/sax/features/validation). */ - protected static final String VALIDATION_FEATURE_ID = "http://xml.org/sax/features/validation"; - - /** - * Schema validation feature id - * (http://apache.org/xml/features/validation/schema). - */ - protected static final String SCHEMA_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/schema"; - - /** - * Dynamic validation feature id - * (http://apache.org/xml/features/validation/dynamic). - */ - protected static final String DYNAMIC_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/dynamic"; - - // default settings - - /** Default parser name. */ - protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser"; - - /** Default namespaces support (false). */ - protected static final boolean DEFAULT_NAMESPACES = false; - - /** Default namespace prefixes (false). */ - protected static final boolean DEFAULT_NAMESPACE_PREFIXES = false; - - /** Default validation support (false). */ - protected static final boolean DEFAULT_VALIDATION = false; - - /** Default Schema validation support (false). */ - protected static final boolean DEFAULT_SCHEMA_VALIDATION = false; - - /** Default dynamic validation support (false). */ - protected static final boolean DEFAULT_DYNAMIC_VALIDATION = false; - - private StringBuilder fieldContent; - - private String fieldName; - - private int fieldLevel = 0; - - private boolean hasLiteralTags = false; - - private Map<String, Object> hitFields = new HashMap<>(); - private String hitType; - private String hitRelevance; - private String hitSource; - - private int offset = 0; - - private List<Tag> tagStack = new ArrayList<>(); - - private final XMLReader parser; - - private Query query; - - private Result result; - - private static enum ResultPart { - ROOT, ERRORDETAILS, HIT, HITGROUP; - } - - Deque<ResultPart> location = new ArrayDeque<>(10); - - private String currentErrorCode; - - private String currentError; - - private Deque<HitGroup> hitGroups = new ArrayDeque<>(5); - - private static class Tag { - public final String name; - - /** - * Offset is a number which is generated for all data and tags inside - * fields, used to determine whether a tag was closed without enclosing - * any characters or other tags. - */ - public final int offset; - - public Tag(final String name, final int offset) { - this.name = name; - this.offset = offset; - } - - @Override - public String toString() { - return name + '(' + Integer.valueOf(offset) + ')'; - } - } - - /** Default constructor. */ - public ResultBuilder() throws RuntimeException { - this(createParser()); - } - - public ResultBuilder(XMLReader parser) { - this.parser = parser; - this.parser.setContentHandler(this); - this.parser.setErrorHandler(this); - } - - public static XMLReader createParser() { - ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(org.apache.xerces.parsers.SAXParser.class.getClassLoader()); - - try { - XMLReader reader = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME); - setParserFeatures(reader); - return reader; - } catch (Exception e) { - throw new RuntimeException("error: Unable to instantiate parser (" - + DEFAULT_PARSER_NAME + ")", e); - } finally { - Thread.currentThread().setContextClassLoader(savedContextClassLoader); - } - } - - private static void setParserFeatures(XMLReader reader) { - try { - reader.setFeature(NAMESPACES_FEATURE_ID, DEFAULT_NAMESPACES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACES_FEATURE_ID + ")"); - } - try { - reader.setFeature(NAMESPACE_PREFIXES_FEATURE_ID, - DEFAULT_NAMESPACE_PREFIXES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACE_PREFIXES_FEATURE_ID + ")"); - } - try { - reader.setFeature(VALIDATION_FEATURE_ID, DEFAULT_VALIDATION); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + VALIDATION_FEATURE_ID + ")"); - } - try { - reader.setFeature(SCHEMA_VALIDATION_FEATURE_ID, - DEFAULT_SCHEMA_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - } - - try { - reader.setFeature(DYNAMIC_VALIDATION_FEATURE_ID, - DEFAULT_DYNAMIC_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - } - } - - @Override - public void startDocument() throws SAXException { - reset(); - result = new Result(query); - hitGroups.addFirst(result.hits()); - location.addFirst(ResultPart.ROOT); - return; - } - - private void reset() { - result = null; - fieldLevel = 0; - hasLiteralTags = false; - tagStack = null; - fieldContent = null; - offset = 0; - currentError = null; - currentErrorCode = null; - hitGroups.clear(); - location.clear(); - } - - @Override - public void startElement(String uri, String local, String raw, - Attributes attrs) throws SAXException { - // "Everybody" wants this switch to be moved into the - // enum class instead, but in this case, I find the classic - // approach more readable. - switch (location.peekFirst()) { - case HIT: - if (fieldLevel > 0) { - tagInField(raw, attrs, FIELD); - ++offset; - return; - } - if (FIELD.equals(raw)) { - ++fieldLevel; - fieldName = attrs.getValue("name"); - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case ERRORDETAILS: - if (fieldLevel > 0) { - tagInField(raw, attrs, ERROR); - ++offset; - return; - } - if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - currentError = attrs.getValue("error"); - } - ++fieldLevel; - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case HITGROUP: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } - break; - case ROOT: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("errordetails".equals(raw)) { - location.addFirst(ResultPart.ERRORDETAILS); - } else if ("result".equals(raw)) { - if (attrs != null) { - String total = attrs.getValue("total-hit-count"); - if (total != null) { - result.setTotalHitCount(Long.valueOf(total)); - } - } - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } else if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - fieldContent = new StringBuilder(); - } - } - break; - } - ++offset; - } - - private void startHitGroup(Attributes attrs) { - HitGroup g = new HitGroup(); - Set<String> types = g.types(); - - final String source; - if (attrs != null) { - String groupType = attrs.getValue("type"); - if (groupType != null) { - for (String s : groupType.split(" ")) { - if (s.length() > 0) { - types.add(s); - } - } - } - - source = attrs.getValue("source"); - } else { - source = null; - } - - g.setId((source != null) ? source : "dummy"); - - hitGroups.peekFirst().add(g); - hitGroups.addFirst(g); - location.addFirst(ResultPart.HITGROUP); - } - - private void startHit(Attributes attrs) { - hitFields.clear(); - location.addFirst(ResultPart.HIT); - if (attrs != null) { - hitRelevance = attrs.getValue("relevancy"); - hitSource = attrs.getValue("source"); - hitType = attrs.getValue("type"); - } else { - hitRelevance = null; - hitSource = null; - hitType = null; - } - } - - private void tagInField(String tag, Attributes attrs, String enclosingTag) { - if (!hasLiteralTags) { - hasLiteralTags = true; - String fieldTillNow = XML.xmlEscape(fieldContent.toString(), false); - fieldContent = new StringBuilder(fieldTillNow); - tagStack = new ArrayList<>(); - } - if (enclosingTag.equals(tag)) { - ++fieldLevel; - } - if (tagStack.size() > 0) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (prevTag != null && (prevTag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append("<").append(tag); - if (attrs != null) { - int attrCount = attrs.getLength(); - for (int i = 0; i < attrCount; i++) { - fieldContent.append(" ").append(attrs.getQName(i)) - .append("=\"").append( - XML.xmlEscape(attrs.getValue(i), true)).append( - "\""); - } - } - tagStack.add(new Tag(tag, offset)); - } - - private void endElementInField(String qName, String enclosingTag) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - - private void endElementInHitField(String qName) { - if (FIELD.equals(qName) && --fieldLevel == 0) { - Object content; - if (hasLiteralTags) { - content = new XMLString(fieldContent.toString()); - } else { - content = fieldContent.toString(); - } - hitFields.put(fieldName, content); - if ("collapseId".equals(fieldName)) { - hitFields.put(fieldName, Integer.valueOf(content.toString())); - } - fieldName = null; - fieldContent = null; - tagStack = null; - } else { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - } - @Override - public void characters(char ch[], int start, int length) - throws SAXException { - - switch (location.peekFirst()) { - case ERRORDETAILS: - case HIT: - if (fieldLevel > 0) { - if (hasLiteralTags) { - if (tagStack.size() > 0) { - Tag tag = tagStack.get(tagStack.size() - 1); - if (tag != null && (tag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append( - XML.xmlEscape(new String(ch, start, length), false)); - } else { - fieldContent.append(ch, start, length); - } - } - break; - default: - if (fieldContent != null) { - fieldContent.append(ch, start, length); - } - break; - } - ++offset; - } - - @Override - public void ignorableWhitespace(char ch[], int start, int length) - throws SAXException { - return; - } - - @Override - public void processingInstruction(String target, String data) - throws SAXException { - return; - } - - @Override - public void endElement(String namespaceURI, String localName, String qName) - throws SAXException { - switch (location.peekFirst()) { - case HITGROUP: - if ("group".equals(qName)) { - hitGroups.removeFirst(); - location.removeFirst(); - } - break; - case HIT: - if (fieldLevel > 0) { - endElementInHitField(qName); - } else if ("hit".equals(qName)) { - //assert(hitKeys.size() == hitValues.size()); - //We try to get either uri or documentID and use that as id - Object docId = extractDocumentID(); - Hit newHit = new Hit(docId.toString()); - if (hitRelevance != null) newHit.setRelevance(new Relevance(Double.parseDouble(hitRelevance))); - if(hitSource != null) newHit.setSource(hitSource); - if(hitType != null) { - for(String type: hitType.split(" ")) { - newHit.types().add(type); - } - } - for(Map.Entry<String, Object> field : hitFields.entrySet()) { - newHit.setField(field.getKey(), field.getValue()); - } - - hitGroups.peekFirst().add(newHit); - location.removeFirst(); - } - break; - case ERRORDETAILS: - if (fieldLevel == 1 && ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - currentError, - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentError = null; - currentErrorCode = null; - fieldContent = null; - tagStack = null; - fieldLevel = 0; - } else if (fieldLevel > 0) { - endElementInField(qName, ERROR); - } else if ("errordetails".equals(qName)) { - location.removeFirst(); - } - break; - case ROOT: - if (ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentErrorCode = null; - fieldContent = null; - } - break; - default: - break; - } - ++offset; - } - - private Object extractDocumentID() { - Object docId = null; - if (hitFields.containsKey("uri")) { - docId = hitFields.get("uri"); - } else { - final String documentId = "documentId"; - if (hitFields.containsKey(documentId)) { - docId = hitFields.get(documentId); - } else { - final String lcDocumentId = toLowerCase(documentId); - for (Map.Entry<String, Object> e : hitFields.entrySet()) { - String key = e.getKey(); - // case insensitive matching, checking length first hoping to avoid some lowercasing - if (documentId.length() == key.length() && lcDocumentId.equals(toLowerCase(key))) { - docId = e.getValue(); - break; - } - } - } - } - if (docId == null) { - docId = "dummy"; - log.info("Results from vespa backend did not contain either uri or documentId"); - } - return docId; - } - - @Override - public void warning(SAXParseException ex) throws SAXException { - printError("Warning", ex); - } - - @Override - public void error(SAXParseException ex) throws SAXException { - printError("Error", ex); - } - - @Override - public void fatalError(SAXParseException ex) throws SAXException { - printError("Fatal Error", ex); - // throw ex; - } - - /** Prints the error message. */ - protected void printError(String type, SAXParseException ex) { - StringBuilder errorMessage = new StringBuilder(); - - errorMessage.append(type); - if (ex != null) { - String systemId = ex.getSystemId(); - if (systemId != null) { - int index = systemId.lastIndexOf('/'); - if (index != -1) - systemId = systemId.substring(index + 1); - errorMessage.append(' ').append(systemId); - } - } - errorMessage.append(':') - .append(ex.getLineNumber()) - .append(':') - .append(ex.getColumnNumber()) - .append(": ") - .append(ex.getMessage()); - log.log(LogLevel.WARNING, errorMessage.toString()); - - } - - public Result parse(String identifier, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(identifier); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - public Result parse(InputSource input, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(input); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - - private void setQuery(Query query) { - this.query = query; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java deleted file mode 100644 index 246732d0970..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.Set; - -import com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher; -import org.xml.sax.InputSource; -import org.xml.sax.XMLReader; - -import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.ComponentId; -import com.yahoo.component.Version; -import com.yahoo.component.chain.dependencies.After; -import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.textserialize.TextSerialize; -import com.yahoo.search.yql.MinimalQueryInserter; -import com.yahoo.statistics.Statistics; - -import edu.umd.cs.findbugs.annotations.Nullable; - -/** - * Backend searcher for external Vespa clusters (queried over http). - * - * <p>If the "sources" argument should be honored on an external cluster - * when using YQL+, override {@link #chooseYqlSources(Set)}.</p> - * - * @author Arne Bergene Fossaa - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated -@Provides("Vespa") -@After("*") -public class VespaSearcher extends ConfiguredHTTPProviderSearcher { - - private final ThreadLocal<XMLReader> readerHolder = new ThreadLocal<>(); - private final Query.Type queryType; - private final Tuple2<String, Version> segmenterVersion; - - private static final CompoundName select = new CompoundName("select"); - private static final CompoundName streamingUserid = new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection = new CompoundName("streaming.selection"); - - /** Create an instance from configuration */ - public VespaSearcher(ComponentId id, ProviderConfig config, QrBinaryCacheConfig c, - QrBinaryCacheRegionConfig r, Statistics statistics) { - this(id, config, c, r, statistics, null); - } - - /** - * Create an instance from configuration - * - * @param linguistics used for generating meta info for YQL+ - */ - @Inject - public VespaSearcher(ComponentId id, ProviderConfig config, - QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r, - Statistics statistics, @Nullable Linguistics linguistics) { - super(id, config, c, r, statistics); - queryType = toQueryType(config.queryType()); - if (linguistics == null) { - segmenterVersion = null; - } else { - segmenterVersion = linguistics.getVersion(Linguistics.Component.SEGMENTER); - } - } - - /** - * Create an instance from direct parameters having a single connection. - * Useful for testing - */ - public VespaSearcher(String idString, String host, int port, String path) { - super(idString, host, port, path, Statistics.nullImplementation); - queryType = toQueryType(ProviderConfig.QueryType.LEGACY); - segmenterVersion = null; - } - - void addProperty(Map<String, String> queryMap, Query query, CompoundName property) { - Object o = query.properties().get(property); - if (o != null) { - queryMap.put(property.toString(), o.toString()); - } - } - - @Override - public Map<String, String> getQueryMap(Query query) { - Map<String, String> queryMap = getQueryMapWithoutHitsOffset(query); - queryMap.put("offset", Integer.toString(query.getOffset())); - queryMap.put("hits", Integer.toString(query.getHits())); - queryMap.put("presentation.format", "xml"); - - addProperty(queryMap, query, select); - addProperty(queryMap, query, streamingUserid); - addProperty(queryMap, query, streamingGroupname); - addProperty(queryMap, query, streamingSelection); - return queryMap; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return getQueryMapWithoutHitsOffset(q); - } - - private Map<String, String> getQueryMapWithoutHitsOffset(Query query) { - Map<String, String> queryMap = super.getQueryMap(query); - if (queryType == Query.Type.YQL) { - queryMap.put(MinimalQueryInserter.YQL.toString(), marshalQuery(query)); - } else { - queryMap.put("query", marshalQuery(query.getModel().getQueryTree())); - queryMap.put("type", queryType.toString()); - } - - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - Query.Type toQueryType(ProviderConfig.QueryType.Enum providerQueryType) { - if (providerQueryType == ProviderConfig.QueryType.LEGACY) { - return Query.Type.ADVANCED; - } else if (providerQueryType == ProviderConfig.QueryType.PROGRAMMATIC) { - return Query.Type.PROGRAMMATIC; - } else if (providerQueryType == ProviderConfig.QueryType.YQL) { - return Query.Type.YQL; - } else if (providerQueryType == ProviderConfig.QueryType.SELECT) { - return Query.Type.SELECT; - } else { - throw new RuntimeException("Query type " + providerQueryType + " unsupported."); - } - } - - /** - * Serialize the query parameter for outgoing queries. For YQL+ queries, - * sources and fields will be set to all sources and all fields, to follow - * the behavior of other query types. - * - * @param query - * the current, outgoing query - * @return a string to include in an HTTP request - */ - public String marshalQuery(Query query) { - if (queryType != Query.Type.YQL) { - return marshalQuery(query.getModel().getQueryTree()); - } - - query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once - Query workQuery = query.clone(); - String error = QueryCanonicalizer.canonicalize(workQuery); - if (error != null) { - getLogger().log(LogLevel.WARNING,"Could not normalize [" + query.toString() + "]: " + error); - // Just returning null here is the pattern from existing code... - return null; - } - chooseYqlSources(workQuery.getModel().getSources()); - chooseYqlSummaryFields(workQuery.getPresentation().getSummaryFields()); - return workQuery.yqlRepresentation(getSegmenterVersion(), false); - } - - public String marshalQuery(QueryTree root) { - QueryTree rootClone = root.clone(); // TODO: Why? - String error = QueryCanonicalizer.canonicalize(rootClone); - if (error != null) return null; - - return marshalRoot(rootClone.getRoot()); - } - - private String marshalRoot(Item root) { - switch (queryType) { - case ADVANCED: return new QueryMarshaller().marshal(root); - case PROGRAMMATIC: return TextSerialize.serialize(root); - default: throw new RuntimeException("Unsupported query type."); - } - } - - private XMLReader getReader() { - XMLReader reader = readerHolder.get(); - if (reader == null) { - reader = ResultBuilder.createParser(); - readerHolder.set(reader); - } - return reader; - } - - @Override - public void unmarshal(InputStream stream, long contentLength, Result result) { - ResultBuilder parser = new ResultBuilder(getReader()); - Result mResult = parser.parse(new InputSource(stream), - result.getQuery()); - result.mergeWith(mResult); - result.hits().addAll(mResult.hits().asUnorderedHits()); - } - - /** Returns the canonical Vespa ping URI, http://host:port/status.html */ - @Override - public URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(getParameters().getSchema(), connection.getHost(), - connection.getPort(), "/status.html").toURI(); - } - - /** - * Get the segmenter version data used when creating YQL queries. Useful if - * overriding {@link #marshalQuery(Query)}. - * - * @return a tuple with the name of the segmenting engine in use, and its - * version - */ - protected Tuple2<String, Version> getSegmenterVersion() { - return segmenterVersion; - } - - /** - * Choose which source arguments to use for the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all sources. Other implementations may modify the source - * set as they see fit, or simply do nothing. - * - * @param sources - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSources(Set<String> sources) { - sources.clear(); - } - - /** - * Choose which summary fields to request from the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all fields. Other implementations may modify the summary - * field set as they see fit, or simply do nothing. - * - * @param summaryFields - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSummaryFields(Set<String> summaryFields) { - summaryFields.clear(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java deleted file mode 100644 index a1c9236ff9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.vespa; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java index 83eca89dffa..13c23234910 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java @@ -121,20 +121,6 @@ public class GroupingRequest { } /** - * Sets the result {@link RootGroup} of this request. This is used by the executing grouping searcher, and should - * not be called by a requesting searcher. - * - * @param group the result to set. - * @return this, to allow chaining. - * @deprecated this is a noop - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public GroupingRequest setResultGroup(RootGroup group) { - return this; - } - - /** * Returns the list of {@link Continuation}s of this request. This is used by the executing grouping searcher to * allow pagination of grouping results. * @@ -156,21 +142,6 @@ public class GroupingRequest { return newRequest; } - /** - * Returns all instances of this class that have been attached to the given {@link Query}. If no requests have been - * attached to the {@link Query}, this method returns an empty list. - * - * @param query the query whose requests to return. - * @return the list of grouping requests. - * @deprecated use query.getSelect().getGrouping() - */ - @SuppressWarnings({ "unchecked" }) - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static List<GroupingRequest> getRequests(Query query) { - return query.getSelect().getGrouping(); - } - @Override public String toString() { return root == null ? "(empty)" : root.toString(); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java deleted file mode 100644 index 8ca772b9188..00000000000 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.grouping.request; - -/** - * This class represents a document checksum in a {@link GroupingExpression}. It evaluates to the YMUM checksum of the - * input {@link com.yahoo.search.result.Hit}. - * - * @author Simon Thoresen Hult - * @author bratseth - */ -public class YmumValue extends DocumentValue { - - /** - * Constructs a new instance of this class. - */ - public YmumValue() { - this(null, null); - } - - private YmumValue(String label, Integer level) { - super("ymum()", label, level); - } - - @Override - public YmumValue copy() { - return new YmumValue(getLabel(), getLevelOrNull()); - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java index 07729b4524a..e1f74df7172 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java @@ -30,9 +30,9 @@ public abstract class BucketGroupId<T> extends GroupId { * * @param type The type of this id's value. * @param from The inclusive-from of the range. - * @param fromImage The String representation of the <tt>from</tt> argument. + * @param fromImage The String representation of the <code>from</code> argument. * @param to The exclusive-to of the range. - * @param toImage The String representation of the <tt>to</tt> argument. + * @param toImage The String representation of the <code>to</code> argument. */ public BucketGroupId(String type, T from, String fromImage, T to, String toImage) { super(type, fromImage, toImage); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java index bebd3fd85b0..a76af885180 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java @@ -27,7 +27,7 @@ public abstract class ValueGroupId<T> extends GroupId { * * @param type The type of this id's value. * @param value The identifying value. - * @param valueImage The String representation of the <tt>value</tt> argument. + * @param valueImage The String representation of the <code>value</code> argument. */ public ValueGroupId(String type, T value, String valueImage) { super(type, valueImage); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java index d017fe2edb3..d5d1bb5b18e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java @@ -85,7 +85,6 @@ import com.yahoo.search.grouping.request.XorAggregator; import com.yahoo.search.grouping.request.XorBitFunction; import com.yahoo.search.grouping.request.XorFunction; import com.yahoo.search.grouping.request.YearFunction; -import com.yahoo.search.grouping.request.YmumValue; import com.yahoo.search.grouping.request.ZCurveXFunction; import com.yahoo.search.grouping.request.ZCurveYFunction; import com.yahoo.searchlib.aggregation.AggregationResult; @@ -115,7 +114,6 @@ import com.yahoo.searchlib.expression.FloatBucketResultNode; import com.yahoo.searchlib.expression.FloatBucketResultNodeVector; import com.yahoo.searchlib.expression.FloatResultNode; import com.yahoo.searchlib.expression.GetDocIdNamespaceSpecificFunctionNode; -import com.yahoo.searchlib.expression.GetYMUMChecksumFunctionNode; import com.yahoo.searchlib.expression.IntegerBucketResultNode; import com.yahoo.searchlib.expression.IntegerBucketResultNodeVector; import com.yahoo.searchlib.expression.IntegerResultNode; @@ -522,9 +520,6 @@ class ExpressionConverter { return new XorBitFunctionNode().setNumBits(((XorBitFunction)exp).getNumBits()) .addArg(toExpressionNode(((XorBitFunction)exp).getArg(0))); } - if (exp instanceof YmumValue) { - return new GetYMUMChecksumFunctionNode(); - } throw new UnsupportedOperationException("Can not convert '" + exp + "' of class " + exp.getClass().getName() + " to an expression."); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index b28e648be78..7c2e774f68b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -217,7 +217,7 @@ public class GroupingExecutor extends Searcher { baseRoot = origRoot.clone(); } if (query.isTraceable(3) && query.getGroupingSessionCache()) { - query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId(false) + "'.", 3); + query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId() + "'.", 3); } for (int pass = 0; pass <= lastPass; ++pass) { boolean firstPass = (pass == 0); diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 6f965944bdf..d2490ec9532 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -13,13 +13,11 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; -import com.yahoo.container.core.QrTemplatesConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.container.logging.AccessLog; -import com.yahoo.container.protect.FreezeDetector; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.language.Linguistics; @@ -33,6 +31,7 @@ import com.yahoo.prelude.query.parser.ParseException; import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.vespa.config.SlimeUtils; @@ -164,48 +163,6 @@ public class SearchHandler extends LoggingRequestHandler { Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey()); } - /** @deprecated use the constructor with ContainerHttpConfig */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public SearchHandler( - final ChainsConfig chainsConfig, - final IndexInfoConfig indexInfo, - final QrSearchersConfig clusters, - final SpecialtokensConfig specialtokens, - final Statistics statistics, - final Linguistics linguistics, - final Metric metric, - final ComponentRegistry<Renderer> renderers, - final Executor executor, - final AccessLog accessLog, - final QueryProfilesConfig queryProfileConfig, - final ComponentRegistry<Searcher> searchers) { - this (chainsConfig, indexInfo, clusters, specialtokens, statistics, linguistics, metric, renderers, executor, - accessLog, queryProfileConfig, searchers, new ContainerHttpConfig(new ContainerHttpConfig.Builder())); - } - - /** @deprecated use the constructor without deprecated parameters */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public SearchHandler( - final ChainsConfig chainsConfig, - final IndexInfoConfig indexInfo, - final QrSearchersConfig clusters, - final SpecialtokensConfig specialTokens, - final QrTemplatesConfig ignored, - final FreezeDetector ignored2, - final Statistics statistics, - final Linguistics linguistics, - final Metric metric, - final ComponentRegistry<Renderer> renderers, - final Executor executor, - final AccessLog accessLog, - final QueryProfilesConfig queryProfileConfig, - final ComponentRegistry<Searcher> searchers) { - this(chainsConfig, indexInfo, clusters, specialTokens, statistics, linguistics, metric, renderers, - executor, accessLog, queryProfileConfig, searchers); - } - @Override protected void destroy() { super.destroy(); @@ -263,8 +220,6 @@ public class SearchHandler extends LoggingRequestHandler { Result result = new Result(query, errorMessage); Renderer renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); - result.getTemplating().setRenderer(renderer); // Pre-Vespa 6 Result.getEncoding() expects this TODO: Remove opn Vespa 7 - return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); } @@ -291,8 +246,12 @@ public class SearchHandler extends LoggingRequestHandler { Query query = new Query(request, requestMap, queryProfile); - boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); - boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); + boolean benchmarking = VespaHeaders.benchmarkOutput(request); + boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarking, request.getJDiscRequest().headers()); + + // Don't use soft timeout by default when benchmarking to avoid wrong conclusions by excluding nodes + if (benchmarking && ! request.hasProperty(SoftTimeout.enableProperty.toString())) + query.properties().set(SoftTimeout.enableProperty, false); // Find and execute search chain if we have a valid query String invalidReason = query.validate(); @@ -321,22 +280,14 @@ public class SearchHandler extends LoggingRequestHandler { result = search(pathAndQuery, query, searchChain, searchChainRegistry); } - Renderer renderer; - if (result.getTemplating().usesDefaultTemplate()) { // TODO: Remove on Vespa 7 - renderer = toRendererCopy(query.getPresentation().getRenderer()); - result.getTemplating().setRenderer(renderer); // pre-Vespa 6 Result.getEncoding() expects this to be set. - } - else { // somebody explicitly assigned a old style template // TODO: Remove on Vespa 7 - renderer = perRenderingCopy(result.getTemplating().getRenderer()); - } - // Transform result to response - HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), + Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); + HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); if (hostResponseHeaderKey.isPresent()) response.headers().add(hostResponseHeaderKey.get(), selfHostname); - if (benchmarkOutput) + if (benchmarking) VespaHeaders.benchmarkOutput(response.headers(), benchmarkCoverage, response.getTiming(), response.getHitCounts(), getErrors(result), response.getCoverage()); @@ -390,9 +341,6 @@ public class SearchHandler extends LoggingRequestHandler { } Result result = execution.search(query); - if (result.getTemplating() == null) // TODO: Remove on Vespa 7 - result.getTemplating().setRenderer(renderer); - ensureQuerySet(result, query); execution.fill(result, result.getQuery().getPresentation().getSummary()); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index 92e4bb7e5b8..0741e8effac 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -13,6 +13,7 @@ import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.model.Renderer; import com.yahoo.search.pagetemplates.model.Source; import com.yahoo.search.query.context.QueryContext; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.DefaultErrorHit; import com.yahoo.search.result.ErrorHit; @@ -124,7 +125,7 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type", QueryContext.ID); TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new com.yahoo.search.rendering.DefaultRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); + traceRoot.accept(new XmlRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); xmlWriter.closeTag(); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index fd52618ad85..28b605fa095 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -59,7 +59,7 @@ public class Model implements Cloneable { argumentType.addField(new FieldDescription(QUERY_STRING, "string", "query")); argumentType.addField(new FieldDescription(TYPE, "string", "type")); argumentType.addField(new FieldDescription(FILTER, "string","filter")); - argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index def-idx defidx")); + argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index")); argumentType.addField(new FieldDescription(LANGUAGE, "string", "language lang")); argumentType.addField(new FieldDescription(ENCODING, "string", "encoding")); argumentType.addField(new FieldDescription(SOURCES, "string", "sources search")); @@ -93,34 +93,6 @@ public class Model implements Cloneable { setParent(query); } - /** - * Creates trace a message of language detection results into this Model - * instance's parent query. Do note this will give bogus results if the - * Execution instance is not set correctly. This is done automatically - * inside {@link Execution#search(Query)}. If tracing the same place as - * creating the query instance, {@link #setExecution(Execution)} has to be - * invoked first with the same Execution instance the query is intended to - * be run by. - * - * @deprecated do not use; language can now be assigned later and for parts of the query tree, making this quite useless - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void traceLanguage() { - if (getParent().getTraceLevel() < 2) return; - if (language != null) { - getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2); - } - else { - Language l = getParsingLanguage(); - // Don't include the query, it will trigger query parsing - getParent().trace("Detected language: " + l, false, 2); - getParent().trace("Language " + l + " determined by " + - (Language.fromEncoding(encoding) != Language.UNKNOWN ? "query encoding" : - "the characters in the terms") + ".", false, 2); - } - } - public Language getParsingLanguage() { return getParsingLanguage(queryString); } @@ -535,28 +507,4 @@ public class Model implements Cloneable { return false; } - /** - * Set the YTrace header value to use when transmitting this model to a - * search backend (of some kind). - * - * @param next string representation of header value - * @deprecated not used, ytrace has been discontinued - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setYTraceHeaderToNext(String next) { } - - /** - * Get the YTrace header value to use when transmitting this model to a - * search backend (of some kind). Returns null if no ytrace data is not - * turned on. - * - * @deprecated not used, ytrace has been discontinued - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getYTraceHeaderToNext() { - return null; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index ae179a2ba07..6b10fd0847c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -32,7 +32,6 @@ public class Presentation implements Cloneable { public static final String BOLDING = "bolding"; public static final String TIMING = "timing"; public static final String SUMMARY = "summary"; - public static final String REPORT_COVERAGE = "reportCoverage"; public static final String SUMMARY_FIELDS = "summaryFields"; /** The (short) name of the parameter holding the name of the return format to use */ @@ -45,7 +44,6 @@ public class Presentation implements Cloneable { argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding")); argumentType.addField(new FieldDescription(TIMING, "boolean", "timing")); argumentType.addField(new FieldDescription(SUMMARY, "string", "summary")); - argumentType.addField(new FieldDescription(REPORT_COVERAGE, "string", "reportcoverage")); argumentType.addField(new FieldDescription(FORMAT, "string", "format template")); argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields")); argumentType.freeze(); @@ -96,16 +94,6 @@ public class Presentation implements Cloneable { /** Sets whether matching query terms should be bolded in the result */ public void setBolding(boolean bolding) { this.bolding = bolding; } - /** @deprecated coverage information is always returned */ - @Deprecated // OK - // TODO: Remove on Vespa 7 - public boolean getReportCoverage() { return true; } - - /** @deprecated coverage information is always returned */ - @Deprecated // OK - // TODO: Remove on Vespa 7 - public void setReportCoverage(boolean ignored) { } - /** Get the name of the format desired for result rendering. */ @NonNull public ComponentSpecification getRenderer() { return format; } diff --git a/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java b/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java index 49529936901..f57a5f4ab1e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java +++ b/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import com.yahoo.container.Server; - import java.util.concurrent.atomic.AtomicLong; /** @@ -27,17 +25,6 @@ public class UniqueRequestId { /** * Creates a session id which is unique across the cluster this runtime is a member of each time this is called. * Calling this causes synchronization. - * - * @deprecated use nextId(serverId) instead - */ - @Deprecated - public static UniqueRequestId next() { - return new UniqueRequestId(Server.get().getServerDiscriminator(), System.currentTimeMillis(), sequenceCounter.getAndIncrement()); - } - - /** - * Creates a session id which is unique across the cluster this runtime is a member of each time this is called. - * Calling this causes synchronization. */ public static UniqueRequestId next(String serverId) { return new UniqueRequestId(serverId, System.currentTimeMillis(), sequenceCounter.getAndIncrement()); diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java index e2a025c8fae..6b28b166150 100644 --- a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java @@ -3,7 +3,7 @@ package com.yahoo.search.query.context; import com.yahoo.processing.execution.Execution; import com.yahoo.search.Query; -import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.text.XMLWriter; import com.yahoo.yolean.trace.TraceNode; @@ -88,7 +88,7 @@ public class QueryContext implements Cloneable { XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type",ID); TraceNode traceRoot=owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new DefaultRenderer.RenderingVisitor(xmlWriter,owner.getStartTime())); + traceRoot.accept(new XmlRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); xmlWriter.closeTag(); } return true; diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index 7aefa6f0cf2..ca437fb9def 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -18,9 +18,7 @@ import com.yahoo.search.searchchain.Execution; public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); - - @SuppressWarnings("deprecation") - private Linguistics linguistics = new SimpleLinguistics(false); + private Linguistics linguistics = new SimpleLinguistics(); private SpecialTokens specialTokens = new SpecialTokens(); public IndexFacts getIndexFacts() { diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index 69d46527255..a57c0f98b45 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -18,7 +18,7 @@ public final class ParserFactory { } /** - * Creates a {@link Parser} appropriate for the given <tt>Query.Type</tt>, providing the Parser with access to + * Creates a {@link Parser} appropriate for the given <code>Query.Type</code>, providing the Parser with access to * the {@link ParserEnvironment} given. * * @param type the query type for which to create a Parser diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 60427aeb0af..9e043bc3dc9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -29,12 +29,6 @@ import java.util.Map; */ public class QueryProperties extends Properties { - /** @deprecated use Query.nativeProperties */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static final CompoundName[] PER_SOURCE_QUERY_PROPERTIES = - Query.nativeProperties.toArray(new CompoundName[] {}); - private Query query; private final CompiledQueryProfileRegistry profileRegistry; @@ -119,7 +113,6 @@ public class QueryProperties extends Properties { else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) { if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.REPORT_COVERAGE)) return true; // TODO: Remove this line on Vespa 7 if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); @@ -217,9 +210,9 @@ public class QueryProperties extends Properties { } else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, false)); - if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, 0.50)); - if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, 0.10)); + if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, true)); + if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, null)); + if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, null)); } else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { Matching matching = ranking.getMatching(); @@ -249,7 +242,7 @@ public class QueryProperties extends Properties { query.getPresentation().setTiming(asBoolean(value, true)); else if (key.last().equals(Presentation.SUMMARY_FIELDS)) query.getPresentation().setSummaryFields(asString(value,"")); - else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7 + else throwIllegalParameter(key.last(), Presentation.PRESENTATION); } else if (key.size()==2 && key.first().equals(Select.SELECT)) { diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java index ca6fd44af50..a05ecee8a1e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -21,6 +22,10 @@ public class SoftTimeout implements Cloneable { public static final String FACTOR = "factor"; public static final String TAILCOST = "tailcost"; + /** The full property name for turning softtimeout on or off */ + public static final CompoundName enableProperty = + CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, ENABLE); + static { argumentType = new QueryProfileType(Ranking.SOFTTIMEOUT); argumentType.setStrict(true); @@ -31,14 +36,15 @@ public class SoftTimeout implements Cloneable { } public static QueryProfileType getArgumentType() { return argumentType; } - public Boolean enable = null; + private boolean enabled = true; private Double factor = null; private Double tailcost = null; - public void setEnable(boolean enable) { this.enable = enable; } + public void setEnable(boolean enable) { this.enabled = enable; } - public Boolean getEnable() { return enable; } + public Boolean getEnable() { return enabled; } + /** Override the adaptive factor determined on the content nodes */ public void setFactor(double factor) { if ((factor < 0.0) || (factor > 1.0)) { throw new IllegalArgumentException("factor must be in the range [0.0, 1.0], got " + factor); @@ -48,6 +54,7 @@ public class SoftTimeout implements Cloneable { public Double getFactor() { return factor; } + /** Override the tail cost factor determined on the content nodes */ public void setTailcost(double tailcost) { if ((tailcost < 0.0) || (tailcost > 1.0)) { throw new IllegalArgumentException("tailcost must be in the range [0.0, 1.0], got " + tailcost); @@ -59,15 +66,13 @@ public class SoftTimeout implements Cloneable { /** Internal operation - DO NOT USE */ public void prepare(RankProperties rankProperties) { - if (enable != null) { - rankProperties.put("vespa.softtimeout.enable", String.valueOf(enable)); - } - if (factor != null) { + if ( !enabled) return; + + rankProperties.put("vespa.softtimeout.enable", "true"); + if (factor != null) rankProperties.put("vespa.softtimeout.factor", String.valueOf(factor)); - } - if (tailcost != null) { + if (tailcost != null) rankProperties.put("vespa.softtimeout.tailcost", String.valueOf(tailcost)); - } } @Override @@ -83,7 +88,7 @@ public class SoftTimeout implements Cloneable { @Override public int hashCode() { int hash = 0; - if (enable != null) hash += 11 * enable.hashCode(); + if (enabled) hash += 11; if (factor != null) hash += 13 * factor.hashCode(); if (tailcost != null) hash += 17 * tailcost.hashCode(); return hash; @@ -95,7 +100,7 @@ public class SoftTimeout implements Cloneable { if ( ! (o instanceof SoftTimeout)) return false; SoftTimeout other = (SoftTimeout)o; - if ( ! Objects.equals(this.enable, other.enable)) return false; + if ( ! Objects.equals(this.enabled, other.enabled)) return false; if ( ! Objects.equals(this.factor, other.factor)) return false; if ( ! Objects.equals(this.tailcost, other.tailcost)) return false; return true; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java deleted file mode 100644 index d6a88a8f4e7..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.language.Language; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NotItem; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.RankItem; -import com.yahoo.prelude.query.parser.CustomParser; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; - -/** - * Compatibility layer to implement the old multi part query syntax, along with - * the features of QueryCombinator. Do <b>not</b> use both QueryCombinator and - * LegacyCombinator in a single search. - * - * <p> - * A searcher which grabs query parameters of the form - * "defidx.(identifier)=(index name)" and "query.(identifier)=(user query)", - * parses them and adds them as AND items to the query root. - * - * <p> - * If the given default index does not exist in the search definition, the query - * part will be parsed with the settings of the default index set to "". - * - * <p> - * If any of the following arguments exist, they will be used: - * - * <p> - * query.(identifier)=query string<br> - * query.(identifier).operator={"req", "rank", "not"}, where "req" is default<br> - * query.(identifier).defidx=default index<br> - * query.(identifier).type={"all", "any", "phrase", "adv", "web"} where "all" is - * default - * - * <p> - * If both defidx.(identifier) and any of - * query.(identifier).{operator,defidx,type} is present in the query, an - * InvalidQueryParameter error will be added, and the query will be passed - * through untransformed. - * - * @author Steinar Knutsen - */ -@Before({"transformedQuery", "com.yahoo.prelude.querytransform.StemmingSearcher"}) -public class LegacyCombinator extends Searcher { - - private static final String TYPESUFFIX = ".type"; - private static final String OPERATORSUFFIX = ".operator"; - private static final String DEFIDXSUFFIX = ".defidx"; - private static final String DEFIDXPREFIX = "defidx."; - private static final String QUERYPREFIX = "query."; - - private enum Combinator { - REQUIRED("req"), PREFERRED("rank"), EXCLUDED("not"); - - String parameterValue; - - private Combinator(String parameterValue) { - this.parameterValue = parameterValue; - } - - static Combinator getCombinator(String name) { - for (Combinator c : Combinator.values()) { - if (c.parameterValue.equals(name)) { - return c; - } - } - return REQUIRED; - } - } - - private static class QueryPart { - final String query; - final String defaultIndex; - final Combinator operator; - final String identifier; - final Query.Type syntax; - - QueryPart(String identifier, String defaultIndex, String oldIndex, - String operator, String query, String syntax) { - validateArguments(identifier, defaultIndex, oldIndex, - operator,syntax); - this.query = query; - if (defaultIndex != null) { - this.defaultIndex = defaultIndex; - } else { - this.defaultIndex = oldIndex; - } - this.operator = Combinator.getCombinator(operator); - this.identifier = identifier; - this.syntax = Query.Type.getType(syntax); - } - - private static void validateArguments(String identifier, String defaultIndex, - String oldIndex, String operator, String syntax) { - if (defaultIndex == null) { - return; - } - if (oldIndex != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, DEFIDXSUFFIX)); - } - if (operator != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, OPERATORSUFFIX)); - } - if (syntax != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, TYPESUFFIX)); - } - } - - private static String createErrorMessage(String identifier, String legacyArgument) { - return "Cannot set both " + DEFIDXPREFIX + identifier + " and " - + QUERYPREFIX + identifier + legacyArgument + "."; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((identifier == null) ? 0 : identifier.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - QueryPart other = (QueryPart) obj; - if (identifier == null) { - if (other.identifier != null) - return false; - } else if (!identifier.equals(other.identifier)) - return false; - return true; - } - - @Override - public String toString() { - return "QueryPart(" + identifier + ", " + defaultIndex + ", " - + operator + ", " + syntax + ")"; - } - } - - @Override - public Result search(Query query, Execution execution) { - Set<QueryPart> pieces; - Set<String> usedSources; - IndexFacts indexFacts = execution.context().getIndexFacts(); - try { - pieces = findQuerySnippets(query.properties()); - } catch (IllegalArgumentException e) { - query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator got invalid parameters: " - + e.getMessage())); - return execution.search(query); - } - if (pieces.size() == 0) { - return execution.search(query); - } - IndexFacts.Session session = indexFacts.newSession(query); - Language language = query.getModel().getParsingLanguage(); - addAndItems(language, query, pieces, session, execution.context()); - addRankItems(language, query, pieces, session, execution.context()); - try { - addNotItems(language, query, pieces, session, execution.context()); - } catch (IllegalArgumentException e) { - query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator found only excluding terms, no including.")); - return execution.search(query); - } - query.trace("Adding extra query parts.", true, 2); - return execution.search(query); - } - - private void addNotItems(Language language, Query query, Set<QueryPart> pieces, - IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.EXCLUDED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addNotItem(query.getModel().getQueryTree(), item); - } - - } - - private void addNotItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - // errr... no positive branch at all? - throw new IllegalArgumentException("No positive terms for query."); - } else if (root.getClass() == NotItem.class) { - ((NotItem) root).addNegativeItem(item); - } else { - NotItem newRoot = new NotItem(); - newRoot.addPositiveItem(root); - newRoot.addNegativeItem(item); - queryTree.setRoot(newRoot); - } - } - - private void addRankItems(Language language, Query query, Set<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.PREFERRED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addRankItem(query.getModel().getQueryTree(), item); - } - } - - private void addRankItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == RankItem.class) { - // if no clear recall terms, just set the rank term as recall - ((RankItem) root).addItem(item); - } else { - RankItem newRoot = new RankItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private void addAndItems(Language language, Query query, Iterable<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.REQUIRED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addAndItem(query.getModel().getQueryTree(), item); - } - } - - private void setDefaultIndex(QueryPart part, String defaultIndex, Item item) { - if (defaultIndex == null) { - assignDefaultIndex(item, part.defaultIndex); - } - } - - private Item parse(Language language, Query query, QueryPart part, String defaultIndex, Execution.Context context) { - Item item = null; - try { - CustomParser parser = (CustomParser)ParserFactory.newInstance( - part.syntax, ParserEnvironment.fromExecutionContext(context)); - item = parser.parse(part.query, null, language, query.getModel().getSources(), - context.getIndexFacts(), defaultIndex); - } catch (RuntimeException e) { - String err = Exceptions.toMessageString(e); - query.trace("Query parser threw an exception: " + err, true, 1); - getLogger().log(LogLevel.WARNING, - "Query parser threw exception in searcher LegacyCombinator for " - + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err); - } - return item; - } - - private String defaultIndex(IndexFacts.Session indexFacts, QueryPart part) { - String defaultIndex; - if (indexFacts.getIndex(part.defaultIndex) == Index.nullIndex) { - defaultIndex = null; - } else { - defaultIndex = part.defaultIndex; - } - return defaultIndex; - } - - private static void addAndItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == AndItem.class) { - ((AndItem) root).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private static void assignDefaultIndex(Item item, String defaultIndex) { - if (item instanceof IndexedItem) { - IndexedItem indexName = (IndexedItem) item; - - if ("".equals(indexName.getIndexName())) { - indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - Iterator<Item> items = ((CompositeItem) item).getItemIterator(); - while (items.hasNext()) { - Item i = items.next(); - assignDefaultIndex(i, defaultIndex); - } - } - - } - - private static Set<QueryPart> findQuerySnippets(Properties properties) { - Set<QueryPart> pieces = new HashSet<>(); - for (Map.Entry<String, Object> k : properties.listProperties() - .entrySet()) { - String key = k.getKey(); - if (!key.startsWith(QUERYPREFIX)) { - continue; - } - String name = key.substring(QUERYPREFIX.length()); - if (hasDots(name)) { - continue; - } - String index = properties.getString(DEFIDXPREFIX + name); - String oldIndex = properties.getString(QUERYPREFIX + name - + DEFIDXSUFFIX); - String operator = properties.getString(QUERYPREFIX + name - + OPERATORSUFFIX); - String type = properties.getString(QUERYPREFIX + name + TYPESUFFIX); - pieces.add(new QueryPart(name, index, oldIndex, operator, k - .getValue().toString(), type)); - } - return pieces; - } - - private static boolean hasDots(String name) { - int index = name.indexOf('.', 0); - return index != -1; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java deleted file mode 100644 index 67cb291a760..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import com.yahoo.component.ComponentId; -import com.yahoo.language.Language; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.parser.CustomParser; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.yolean.Exceptions; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -/** - * <p>A searcher which grabs query parameters of the form "defidx.(identifier)=(index name)" and - * "query.(identifier)=(user query)", * parses them and adds them as AND items to the query root.</p> - * - * <p>If the given default index does not exist in the search definition, the query part will be parsed with the - * settings of the default index set to the "".</p> - * - * @author Steinar Knutsen - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryCombinator extends Searcher { - - private static final String QUERYPREFIX = "query."; - - private static class QueryPart { - final String query; - final String defaultIndex; - - QueryPart(String query, String defaultIndex) { - this.query = query; - this.defaultIndex = defaultIndex; - } - } - - public QueryCombinator(ComponentId id) { - super(id); - } - - @Override - public Result search(Query query, Execution execution) { - Set<QueryPart> pieces = findQuerySnippets(query.properties()); - if (pieces.size() == 0) { - return execution.search(query); - } - addAndItems(query, pieces, execution.context()); - query.trace("Adding extra query parts.", true, 2); - return execution.search(query); - } - - private void addAndItems(Query query, Iterable<QueryPart> pieces, Execution.Context context) { - IndexFacts indexFacts = context.getIndexFacts(); - IndexFacts.Session session = indexFacts.newSession(query); - Set<String> usedSources = new HashSet<>(session.documentTypes()); - Language language = query.getModel().getParsingLanguage(); - for (QueryPart part : pieces) { - String defaultIndex; - Item item = null; - Index index = session.getIndex(part.defaultIndex); - if (index == Index.nullIndex) { - defaultIndex = null; - } else { - defaultIndex = part.defaultIndex; - } - try { - CustomParser parser = (CustomParser)ParserFactory.newInstance(query.getModel().getType(), - ParserEnvironment.fromExecutionContext(context)); - item = parser.parse(part.query, null, language, usedSources, indexFacts, defaultIndex); - } catch (RuntimeException e) { - String err = Exceptions.toMessageString(e); - query.trace("Query parser threw an exception: " + err, true, 1); - getLogger().log(LogLevel.WARNING, - "Query parser threw exception searcher QueryCombinator for " - + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err); - } - if (item == null) { - continue; - } - if (defaultIndex == null) { - assignDefaultIndex(item, part.defaultIndex); - } - addAndItem(query.getModel().getQueryTree(), item); - } - } - - private static void addAndItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == AndItem.class) { - ((AndItem) root).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private static void assignDefaultIndex(Item item, String defaultIndex) { - if (item instanceof IndexedItem) { - IndexedItem indexName = (IndexedItem) item; - - if ("".equals(indexName.getIndexName())) { - indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - Iterator<Item> items = ((CompositeItem) item).getItemIterator(); - while (items.hasNext()) { - Item i = items.next(); - assignDefaultIndex(i, defaultIndex); - } - } - } - - private static Set<QueryPart> findQuerySnippets(Properties properties) { - Set<QueryPart> pieces = new HashSet<>(); - for (Map.Entry<String, Object> k : properties.listProperties().entrySet()) { - String key = k.getKey(); - if (!key.startsWith(QUERYPREFIX)) { - continue; - } - String name = key.substring(QUERYPREFIX.length()); - if (hasDots(name)) { - continue; - } - String index = properties.getString("defidx." + name); - pieces.add(new QueryPart(k.getValue().toString(), index)); - } - return pieces; - } - - private static boolean hasDots(String name) { - int index = name.indexOf('.', 0); - return index != -1; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java deleted file mode 100644 index 30695338741..00000000000 --- a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.rendering; - -import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.data.XmlProducer; -import com.yahoo.io.ByteWriter; -import com.yahoo.net.URI; -import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; -import com.yahoo.processing.response.Data; -import com.yahoo.processing.response.DataList; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.grouping.result.HitRenderer; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.result.*; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.io.Writer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import java.util.Iterator; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -/** - * XML rendering of search results. This is NOT the default (but it once was). - * - * @author Tony Vaagenes - * @deprecated use JsonRenderer instead - */ -@SuppressWarnings({ "rawtypes", "deprecation" }) -@Deprecated // OK -// TODO: Rename to XmlRenderer on Vespa 7 -public final class DefaultRenderer extends AsynchronousSectionedRenderer<Result> { - - public static final String DEFAULT_MIMETYPE = "text/xml"; - public static final String DEFAULT_ENCODING = "utf-8"; - - private static final Utf8String RESULT = new Utf8String("result"); - private static final Utf8String GROUP = new Utf8String("group"); - private static final Utf8String ID = new Utf8String("id"); - private static final Utf8String FIELD = new Utf8String("field"); - private static final Utf8String HIT = new Utf8String("hit"); - private static final Utf8String ERROR = new Utf8String("error"); - private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count"); - private static final Utf8String QUERY_TIME = new Utf8String("querytime"); - private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime"); - private static final Utf8String SEARCH_TIME = new Utf8String("searchtime"); - private static final Utf8String NAME = new Utf8String("name"); - private static final Utf8String CODE = new Utf8String("code"); - private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs"); - private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes"); - private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full"); - private static final Utf8String COVERAGE = new Utf8String("coverage"); - private static final Utf8String RESULTS_FULL = new Utf8String("results-full"); - private static final Utf8String RESULTS = new Utf8String("results"); - private static final Utf8String TYPE = new Utf8String("type"); - private static final Utf8String RELEVANCY = new Utf8String("relevancy"); - private static final Utf8String SOURCE = new Utf8String("source"); - - - // this is shared between umpteen threads by design - private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>(); - - private XMLWriter writer; - - public DefaultRenderer() { - this(null); - } - - /** - * Creates an XML renderer using a custom executor. - * Using a custom executor is useful for tests to avoid creating new threads for each renderer registry. - */ - public DefaultRenderer(Executor executor) { - super(executor); - } - - @Override - public void init() { - super.init(); - writer = null; - } - - @Override - public String getEncoding() { - if (getResult() == null - || getResult().getQuery() == null - || getResult().getQuery().getModel().getEncoding() == null) { - return DEFAULT_ENCODING; - } else { - return getResult().getQuery().getModel().getEncoding(); - } - } - - @Override - public String getMimeType() { - return DEFAULT_MIMETYPE; - } - - private XMLWriter wrapWriter(Writer writer) { - return XMLWriter.from(writer, 10, -1); - } - - private void header(XMLWriter writer, Result result) throws IOException { - // TODO: move setting this to Result - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT, String.valueOf(result.getTotalHitCount())); - renderCoverageAttributes(result.getCoverage(false), writer); - renderTime(writer, result); - writer.closeStartTag(); - } - - private void renderTime(XMLWriter writer, Result result) { - if ( ! result.getQuery().getPresentation().getTiming()) return; - - final String threeDecimals = "%.3f"; - final double milli = .001d; - final long now = System.currentTimeMillis(); - final long searchTime = now - result.getElapsedTime().first(); - final double searchSeconds = ((double) searchTime) * milli; - - if (result.getElapsedTime().firstFill() != 0L) { - final long queryTime = result.getElapsedTime().weightedSearchTime(); - final long summaryFetchTime = result.getElapsedTime().weightedFillTime(); - final double querySeconds = ((double) queryTime) * milli; - final double summarySeconds = ((double) summaryFetchTime) * milli; - writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds)); - writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds)); - } - writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds)); - } - - protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) throws IOException { - if (coverage == null) return; - writer.attribute(COVERAGE_DOCS,coverage.getDocs()); - writer.attribute(COVERAGE_NODES,coverage.getNodes()); - writer.attribute(COVERAGE_FULL,coverage.getFull()); - writer.attribute(COVERAGE,coverage.getResultPercentage()); - writer.attribute(RESULTS_FULL,coverage.getFullResultSets()); - writer.attribute(RESULTS,coverage.getResultSets()); - } - - public void error(XMLWriter writer, Result result) throws IOException { - ErrorMessage error = result.hits().getError(); - writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag(); - } - - @SuppressWarnings("UnusedParameters") - protected void emptyResult(XMLWriter writer, Result result) throws IOException {} - - @SuppressWarnings("UnusedParameters") - public void queryContext(XMLWriter writer, QueryContext queryContext, Query owner) throws IOException { - if (owner.getTraceLevel()!=0) { - XMLWriter xmlWriter=XMLWriter.from(writer); - xmlWriter.openTag("meta").attribute("type", QueryContext.ID); - TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new RenderingVisitor(xmlWriter, owner.getStartTime())); - xmlWriter.closeTag(); - } - } - - private void renderSingularHit(XMLWriter writer, Hit hit) { - writer.openTag(HIT); - renderHitAttributes(writer, hit); - writer.closeStartTag(); - renderHitFields(writer, hit); - } - - private void renderHitFields(XMLWriter writer, Hit hit) { - renderSyntheticRelevanceField(writer, hit); - hit.forEachField((name, value) -> renderField(writer, name, value)); - } - - private void renderField(XMLWriter writer, String name, Object value) { - if (name.startsWith("$")) return; - - writeOpenFieldElement(writer, name); - renderFieldContent(writer, value); - writeCloseFieldElement(writer); - } - - private void renderFieldContent(XMLWriter writer, Object value) { - writer.escapedContent(asXML(value), false); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof XmlProducer) - return ((XmlProducer)value).toXML(); - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - private void renderSyntheticRelevanceField(XMLWriter writer, Hit hit) { - String relevancyFieldName = "relevancy"; - Relevance relevance = hit.getRelevance(); - - // skip depending on hit type - if (relevance != null) { - renderSimpleField(writer, relevancyFieldName, relevance); - } - } - - private void renderSimpleField(XMLWriter writer, String relevancyFieldName, Relevance relevance) { - writeOpenFieldElement(writer, relevancyFieldName); - writer.content(relevance.toString(), false); - writeCloseFieldElement(writer); - } - - private void writeCloseFieldElement(XMLWriter writer) { - writer.closeTag(); - } - - private void writeOpenFieldElement(XMLWriter writer, String relevancyFieldName) { - Utf8String utf8 = fieldNameMap.get(relevancyFieldName); - if (utf8 == null) { - utf8 = new Utf8String(relevancyFieldName); - fieldNameMap.put(relevancyFieldName, utf8); - } - writer.openTag(FIELD).attribute(NAME, utf8); - writer.closeStartTag(); - } - - private void renderHitAttributes(XMLWriter writer, Hit hit) { - writer.attribute(TYPE, hit.types().stream().collect(Collectors.joining(" "))); - if (hit.getRelevance() != null) - writer.attribute(RELEVANCY, hit.getRelevance().toString()); - writer.attribute(SOURCE, hit.getSource()); - } - - private void renderHitGroup(XMLWriter writer, HitGroup hit) throws IOException { - if (HitRenderer.renderHeader(hit, writer)) { - // empty - } else if (hit.types().contains("grouphit")) { - // TODO Keep this? - renderHitGroupOfTypeGroupHit(writer, hit); - } else { - renderGroup(writer, hit); - } - } - - private void renderGroup(XMLWriter writer, HitGroup hit) { - writer.openTag(GROUP); - renderHitAttributes(writer, hit); - writer.closeStartTag(); - } - - private void renderHitGroupOfTypeGroupHit(XMLWriter writer, HitGroup hit) { - writer.openTag(HIT); - renderHitAttributes(writer, hit); - renderId(writer, hit); - writer.closeStartTag(); - } - - private void renderId(XMLWriter writer, HitGroup hit) { - URI uri = hit.getId(); - if (uri != null) { - writer.openTag(ID).content(uri.stringValue(),false).closeTag(); - } - } - - private boolean simpleRenderHit(XMLWriter writer, Hit hit) throws IOException { - if (hit instanceof DefaultErrorHit) { - return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit); - } else if (hit instanceof GroupingListHit) { - return true; - } else { - return false; - } - } - - public static boolean simpleRenderDefaultErrorHit(XMLWriter writer, ErrorHit defaultErrorHit) throws IOException { - writer.openTag("errordetails"); - for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - renderMessageDefaultErrorHit(writer, error); - } - writer.closeTag(); - return true; - } - - public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException { - writer.openTag("error"); - writer.attribute("source", error.getSource()); - writer.attribute("error", error.getMessage()); - writer.attribute("code", Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(), false); - if (error.getCause()!=null) { - writer.openTag("cause"); - writer.content("\n", true); - StringWriter stackTrace=new StringWriter(); - error.getCause().printStackTrace(new PrintWriter(stackTrace)); - writer.content(stackTrace.toString(), true); - writer.closeTag(); - } - writer.closeTag(); - } - - public static final class RenderingVisitor extends TraceVisitor { - - private static final String tag = "p"; - private final XMLWriter writer; - private long baseTime; - - public RenderingVisitor(XMLWriter writer,long baseTime) { - this.writer=writer; - this.baseTime=baseTime; - } - - @Override - public void entering(TraceNode node) { - if (node.isRoot()) return; - writer.openTag(tag); - } - - @Override - public void leaving(TraceNode node) { - if (node.isRoot()) return; - writer.closeTag(); - } - - @Override - public void visit(TraceNode node) { - if (node.isRoot()) return; - if (node.payload()==null) return; - - writer.openTag(tag); - if (node.timestamp()!=0) - writer.content(node.timestamp()-baseTime,false).content(" ms: ", false); - writer.content(node.payload().toString(),false); - writer.closeTag(); - } - - } - - private Result getResult() { - Result r; - try { - r = (Result) getResponse(); - } catch (ClassCastException e) { - throw new IllegalArgumentException( - "DefaultRenderer attempted used outside a search context, got a " - + getResponse().getClass().getName()); - } - return r; - } - - @Override - public void beginResponse(OutputStream stream) throws IOException { - Charset cs = Charset.forName(getRequestedEncoding(getResult().getQuery())); - CharsetEncoder encoder = cs.newEncoder(); - writer = wrapWriter(new ByteWriter(stream, encoder)); - - header(writer, getResult()); - if (getResult().hits().getError() != null || getResult().hits().getQuery().errors().size() > 0) { - error(writer, getResult()); - } - - if (getResult().getConcreteHitCount() == 0) { - emptyResult(writer, getResult()); - } - - if (getResult().getContext(false) != null) { - queryContext(writer, getResult().getContext(false), getResult().getQuery()); - } - - } - - /** Returns the encoding of the query, or the encoding given by the template if none is set */ - public final String getRequestedEncoding(Query query) { - String encoding = query.getModel().getEncoding(); - if (encoding != null) return encoding; - return getEncoding(); - } - - @Override - public void beginList(DataList<?> list) throws IOException { - if (getRecursionLevel() == 1) return; - - HitGroup hit = (HitGroup) list; - boolean renderedSimple = simpleRenderHit(writer, hit); - if (renderedSimple) return; - - renderHitGroup(writer, hit); - } - - @Override - public void data(Data data) throws IOException { - Hit hit = (Hit) data; - boolean renderedSimple = simpleRenderHit(writer, hit); - if (renderedSimple) return; - - renderSingularHit(writer, hit); - writer.closeTag(); - } - - @Override - public void endList(DataList<?> list) { - if (getRecursionLevel() > 1) - writer.closeTag(); - } - - @Override - public void endResponse() { - writer.closeTag(); - writer.close(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 7d60d7cf9ee..e8af150ce25 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -18,6 +18,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.JsonWriter; import com.yahoo.lang.MutableBoolean; +import com.yahoo.prelude.hitfield.HitField; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; @@ -473,7 +474,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeNumberField(RELEVANCE, hit.getRelevance().getScore()); - if (hit.types().size() > 0) { // TODO: Remove types rendering on Vespa 7 + if (hit.types().size() > 0) { generator.writeArrayFieldStart(TYPES); for (String t : hit.types()) { generator.writeString(t); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java index 0f01261f6a4..783045babf4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java @@ -4,12 +4,9 @@ package com.yahoo.search.rendering; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.prelude.templates.PageTemplateSet; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.prelude.templates.UserTemplate; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.result.PageTemplatesXmlRenderer; import java.util.Collection; import java.util.Collections; @@ -23,14 +20,11 @@ import java.util.concurrent.Executor; */ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processing.rendering.Renderer<Result>> { - public static final ComponentId xmlRendererId = ComponentId.fromString("DefaultRenderer"); - private static final ComponentId newXmlRendererId = ComponentId.fromString("XmlRenderer"); + public static final ComponentId xmlRendererId = ComponentId.fromString("XmlRenderer"); + public static final ComponentId pageRendererId = ComponentId.fromString("PageTemplatesXmlRenderer"); public static final ComponentId jsonRendererId = ComponentId.fromString("JsonRenderer"); public static final ComponentId defaultRendererId = jsonRendererId; - private final ComponentId tiledRendererId; - private final ComponentId pageRendererId; - /** Creates a registry containing the built-in renderers only */ public RendererRegistry() { this(Collections.emptyList()); @@ -61,23 +55,19 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi register(jsonRenderer.getId(), jsonRenderer); // Add xml renderer - Renderer xmlRenderer = new DefaultRenderer(executor); + Renderer xmlRenderer = new XmlRenderer(executor); xmlRenderer.initId(xmlRendererId); register(xmlRenderer.getId(), xmlRenderer); - // Add new Vespa 7 xml renderer - Renderer newXmlRenderer = new XmlRenderer(executor); - newXmlRenderer.initId(newXmlRendererId); - register(newXmlRenderer.getId(), newXmlRenderer); + // Add page templates renderer + Renderer pageRenderer = new PageTemplatesXmlRenderer(executor); + pageRenderer.initId(pageRendererId); + register(pageRenderer.getId(), pageRenderer); // add application renderers for (Renderer renderer : renderers) register(renderer.getId(), renderer); - // add legacy "templates" converted to renderers // TODO: Remove on Vespa 7 - tiledRendererId = addTemplateSet(new TiledTemplateSet()); - pageRendererId = addTemplateSet(new PageTemplateSet()); - freeze(); } @@ -86,20 +76,9 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi // deconstruct the renderers which was created by this getRenderer(jsonRendererId.toSpecification()).deconstruct(); getRenderer(xmlRendererId.toSpecification()).deconstruct(); - getRenderer(newXmlRendererId.toSpecification()).deconstruct(); - getRenderer(tiledRendererId.toSpecification()).deconstruct(); getRenderer(pageRendererId.toSpecification()).deconstruct(); } - @SuppressWarnings({"deprecation", "unchecked"}) - private ComponentId addTemplateSet(UserTemplate<?> templateSet) { - Renderer renderer = new SearchRendererAdaptor(templateSet); - ComponentId rendererId = new ComponentId(templateSet.getName()); - renderer.initId(rendererId); - register(rendererId, renderer); - return rendererId; - } - /** * Returns the default JSON renderer * @@ -120,6 +99,7 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi if (format == null || format.stringValue().equals("default")) return getDefaultRenderer(); if (format.stringValue().equals("json")) return getComponent(jsonRendererId); if (format.stringValue().equals("xml")) return getComponent(xmlRendererId); + if (format.stringValue().equals("page")) return getComponent(pageRendererId); com.yahoo.processing.rendering.Renderer<Result> renderer = getComponent(format); if (renderer == null) diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index 5f99c531c95..5586fd2f996 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -349,15 +349,12 @@ public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { } private Result getResult() { - Result r; try { - r = (Result) getResponse(); + return (Result) getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException( - "XmlRenderer attempted used outside a search context, got a " - + getResponse().getClass().getName()); + throw new IllegalArgumentException("XmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } - return r; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 7bc49fdb0b1..25829b70b5e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -23,20 +23,6 @@ public class Coverage extends com.yahoo.container.handler.Coverage { super(docs, active, nodes, resultSets); } - // TODO: Remove on Vespa 7 - /** @deprecated don't send a "full" boolean */ - @Deprecated // OK - public Coverage(long docs, int nodes, boolean full) { - this(docs, nodes, full, 1); - } - - // TODO: Remove on Vespa 7 - /** @deprecated don't send a "full" boolean */ - @Deprecated // OK - public Coverage(long docs, int nodes, boolean full, int resultSets) { - super(docs, nodes, full, resultSets); - } - /** * Will set number of documents present in ideal state * diff --git a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java index 54b83544b1a..d8ec768325a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java @@ -58,18 +58,6 @@ public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable { } /** - * Returns the main error of this result, never null. - * - * @deprecated use {@link #errors()} - */ - @Override - @Deprecated // OK - // TODO: Remove on Vespa 7 - public ErrorMessage getMainError() { - return errors.get(0); - } - - /** * This is basically a way of making a list simulate a set. */ private void removeAndAdd(ErrorMessage error) { diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java index 091c8d67793..9be80372ba9 100644 --- a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java @@ -14,15 +14,6 @@ public interface ErrorHit extends Cloneable { void setSource(String source); - /** - * Returns the main error of this result, never null - * - * @deprecated use errors().iterator().next() - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - ErrorMessage getMainError(); - /** * Adds an error to this. This may change the main error * and/or the list of detailed errors diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index c5132549d05..db5b3e69803 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -106,9 +106,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** If this is true, then this hit will not be counted as a concrete hit */ private boolean auxiliary = false; - /** - * The hit field used to store rank features. TODO: Remove on Vespa 7 - */ + /** The hit field used to store rank features */ public static final String RANKFEATURES_FIELD = "rankfeatures"; public static final String SDDOCNAME_FIELD = "sddocname"; @@ -420,7 +418,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi */ public void forEachFieldAsRaw(RawUtf8Consumer consumer) { if (fields == null) return; - fields.forEach(consumer); // No utf-8 fields available in Hit + fields.forEach(consumer); } /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ @@ -478,20 +476,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return fields != null && ! fields.isEmpty(); } - /** - * Changes the key under which a value is found. This is useful because it allows keys to be changed - * without accessing the value (which may be lazily created). - * - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void changeFieldKey(String oldKey, String newKey) { - Map<String,Object> fieldMap = getFieldMap(); - Object value = fieldMap.remove(oldKey); - fieldMap.put(newKey, value); - } - private Map<String, Object> getFieldMap() { return getFieldMap(2); } @@ -515,42 +499,16 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return unmodifiableFieldMap; } - /** Generate a HitField from a field if the field exists */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public HitField buildHitField(String key) { - return buildHitField(key, false); - } - - /** Generate a HitField from a field if the field exists */ - @SuppressWarnings("deprecation") - // TODO: Remove on Vespa 7 - @Deprecated // OK public HitField buildHitField(String key, boolean forceNoPreTokenize) { - return buildHitField(key, forceNoPreTokenize, false); - } - - // TODO: Remove third parameter on Vespa 7 (set always true) - @Deprecated // OK - public HitField buildHitField(String key, boolean forceNoPreTokenize, boolean forceStringHandling) { Object o = getField(key); if (o == null) return null; if (o instanceof HitField) return (HitField)o; HitField h; - if (forceNoPreTokenize) { - if (o instanceof XMLString && !forceStringHandling) { - h = new HitField(key, (XMLString) o, false); - } else { - h = new HitField(key, o.toString(), false); - } - } else { - if (o instanceof XMLString && !forceStringHandling) { - h = new HitField(key, (XMLString) o); - } else { - h = new HitField(key, o.toString()); - } - } + if (forceNoPreTokenize) + h = new HitField(key, o.toString(), false); + else + h = new HitField(key, o.toString()); h.setOriginal(o); getFieldMap().put(key, h); return h; @@ -559,34 +517,19 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */ public Set<String> types() { return types; } - /** @deprecated do not use */ - // TODO: FRemove on Vespa 7 - @Deprecated // OK - public String getTypeString() { - return types().stream().collect(Collectors.joining(" ")); - } - /** * Returns the add number, assigned when adding the hit to a Result. * * Used to order equal relevant hit by add order. -1 if this hit * has never been added to a result. - * - * @deprecated do not use */ - // TODO: Make package private on Vespa 7 - @Deprecated // OK - public int getAddNumber() { return addNumber; } + int getAddNumber() { return addNumber; } /** * Sets the add number, assigned when adding the hit to a Result, * used to order equal relevant hit by add order. - * - * @deprecated do not use */ - // TODO: Make package private on Vespa 7 - @Deprecated // OK - public void setAddNumber(int addNumber) { this.addNumber = addNumber; } + void setAddNumber(int addNumber) { this.addNumber = addNumber; } /** * Returns whether this is a concrete hit, containing content of the requested @@ -610,16 +553,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi public void setAuxiliary(boolean auxiliary) { this.auxiliary = auxiliary; } - /** @deprecated do not use */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public int getSourceNumber() { return sourceNumber; } - - /** @deprecated do not use */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setSourceNumber(int number) { this.sourceNumber = number; } - /** Returns the query which produced this hit, or null if not known */ public Query getQuery() { return query; } @@ -633,52 +566,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } } - /** - * Returns a field of this hit XML escaped and without token - * delimiters. - * - * @deprecated do not use - * @return a field of this hit, or null if the property is not set - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getFieldXML(String key) { - Object p = getField(key); - - if (p == null) { - return null; - } else if (p instanceof HitField) { - return ((HitField)p).quotedContent(false); - } else if (p instanceof StructuredData || p instanceof XMLString || p instanceof JSONString) { - return p.toString(); - } else { - return XML.xmlEscape(p.toString(), false, '\u001f'); - } - } - - /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getUnboldedField(String key, boolean escape) { - Object p = getField(key); - - if (p == null) { - return null; - } else if (p instanceof HitField) { - return ((HitField) p).bareContent(escape, false); - } else if (p instanceof StructuredData) { - return p.toString(); - } else if (p instanceof XMLString || p instanceof JSONString) { - return p.toString(); - } else if (escape) { - return XML.xmlEscape(p.toString(), false, '\u001f'); - } else { - return stripCharacter('\u001F', p.toString()); - } - } - /** Attach some data to this hit for this searcher */ public void setSearcherSpecificMetaData(Searcher searcher, Object data) { if (searcherSpecificMetaData == null) { @@ -703,60 +590,21 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return searcherSpecificMetaData != null ? searcherSpecificMetaData.get(searcher) : null; } - /** - * Internal - do not use - * - * @param filled the backing set - */ - // TODO: Make package private on Vespa 7 - protected final void setFilledInternal(Set<String> filled) { + final void setFilledInternal(Set<String> filled) { this.filled = filled; unmodifiableFilled = (filled != null) ? Collections.unmodifiableSet(filled) : null; } /** - * For internal use only. * Gives access to the modifiable backing set of filled summaries. * This set might be unmodifiable if the size is less than or equal to 1 * * @return the set of filled summaries. */ - // TODO: Make package private on Vespa 7 - protected final Set<String> getFilledInternal() { + final Set<String> getFilledInternal() { return filled; } - /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static String stripCharacter(char strip, String toStripFrom) { - StringBuilder builder = null; - - int lastBadChar = 0; - for (int i = 0; i < toStripFrom.length(); i++) { - if (toStripFrom.charAt(i) == strip) { - if (builder == null) { - builder = new StringBuilder(toStripFrom.length()); - } - - builder.append(toStripFrom, lastBadChar, i); - lastBadChar = i + 1; - } - } - - if (builder == null) { - return toStripFrom; - } else { - if (lastBadChar < toStripFrom.length()) { - builder.append(toStripFrom, lastBadChar, toStripFrom.length()); - } - - return builder.toString(); - } - } - /** Releases the resources held by this, making it irreversibly unusable */ protected void close() { query = null; @@ -832,8 +680,8 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * * @param fieldName the name of the field * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data - * @param offset the start index of the data to accept into the utf8Data array - * @param length the length of the data to accept into the utf8Data array + * @param offset the start index in the utf8Data array of the data to accept + * @param length the length starting from offset in the utf8Data array of the data to accept */ void accept(String fieldName, byte[] utf8Data, int offset, int length); diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 3d5cbab2865..af89220e504 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -383,17 +383,6 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< return hit; } - /** - * Sets the main error of this result - * - * @deprecated prefer addError to add some error information. - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setError(ErrorMessage error) { - addError(error); - } - /** Adds an error to this result */ public void addError(ErrorMessage error) { getError(); // update the list of errors diff --git a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java index a07eaa2438d..483849a5435 100644 --- a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java +++ b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java @@ -17,8 +17,8 @@ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { public PositionsData(Inspector value) { this.value = value; - if (value.type() != Type.ARRAY) { - throw new IllegalArgumentException("PositionsData expects an array of positions, got: "+value); + if (value.type() != Type.OBJECT && value.type() != Type.ARRAY) { + throw new IllegalArgumentException("PositionsData expects a position or an array of positions, got: "+value); } } @@ -38,27 +38,35 @@ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { @Override public StringBuilder writeXML(StringBuilder target) { - for (int i = 0; i < value.entryCount(); i++) { - Inspector pos = value.entry(i); - target.append("<position "); - for (java.util.Map.Entry<String, Inspector> entry : pos.fields()) { - Inspector v = entry.getValue(); - if (v.type() == Type.STRING) { - target.append(entry.getKey()); - target.append("=\""); - target.append(entry.getValue().asString()); - target.append("\" "); - } - if (v.type() == Type.LONG) { - target.append(entry.getKey()); - target.append("=\""); - target.append(entry.getValue().asLong()); - target.append("\" "); - } + if (value.type() == Type.OBJECT) { + writeXML(value.inspect(), target); + } else { + for (int i = 0; i < value.entryCount(); i++) { + Inspector pos = value.entry(i); + writeXML(pos, target); } - target.append("/>"); } return target; } + private static void writeXML(Inspector pos, StringBuilder target) { + target.append("<position "); + for (java.util.Map.Entry<String, Inspector> entry : pos.fields()) { + Inspector v = entry.getValue(); + if (v.type() == Type.STRING) { + target.append(entry.getKey()); + target.append("=\""); + target.append(entry.getValue().asString()); + target.append("\" "); + } + if (v.type() == Type.LONG) { + target.append(entry.getKey()); + target.append("=\""); + target.append(entry.getValue().asLong()); + target.append("\" "); + } + } + target.append("/>"); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Templating.java b/container-search/src/main/java/com/yahoo/search/result/Templating.java deleted file mode 100644 index a8b1eedf528..00000000000 --- a/container-search/src/main/java/com/yahoo/search/result/Templating.java +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result; - -import java.util.Map; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TemplateSet; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.processing.rendering.Renderer; -import com.yahoo.search.Result; -import com.yahoo.search.query.Presentation; - -/** - * Helper methods and data store for result attributes geared towards result - * rendering and presentation. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public class Templating { - - private final Result result; - private Renderer<Result> renderer; - - public Templating(Result result) { - super(); - this.result = result; - } - - /** - * Returns The first hit presented in the result as an index into the global - * list of all hits generated by the user query. - */ - public int getFirstHitNo() { - return result.getQuery().getOffset() + 1; - } - - /** - * Returns the first hit of the next result page, 0 if there aren't any more - * hits available - */ - public long getNextFirstHitNo() { - if (result.getQuery().getHits() > result.getConcreteHitCount()) { - return 0; - } - - return Math.min(getLastHitNo() + 1, result.getTotalHitCount()); - } - - /** - * Returns the first hit of the next result page, 0 if there aren't any more - * hits available - */ - public long getNextLastHitNo() { - if (result.getQuery().getHits() > result.getConcreteHitCount()) { - return 0; - } - - return Math.min(getLastHitNo() + result.getConcreteHitCount(), result.getTotalHitCount()); - } - - /** - * Returns the number of the last result of the current hit page. - */ - public int getLastHitNo() { - return getFirstHitNo() + result.getConcreteHitCount() - 1; - } - - /** - * The first hit presented on the previous result page as an index into the - * global list of all hits generated by the user query - */ - public int getPrevFirstHitNo() { - return Math.max(getFirstHitNo() - result.getQuery().getHits(), 1); - } - - /** - * The last hit presented on the previous result page as an index into the - * global list of all hits generated by the user query - */ - public int getPrevLastHitNo() { - return Math.max(getFirstHitNo() - 1, 0); - } - - /** - * An URL that may be used to obtain the next result page. - */ - public String getNextResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder nextURL = new StringBuilder(); - - nextURL.append(getPath(request)).append("?"); - parametersExceptOffset(request, nextURL); - - int offset = getLastHitNo(); - - nextURL.append("&").append("offset=").append(Integer.toString(offset)); - return nextURL.toString(); - } - - /** - * An URL that may be used to obtain the previous result page. - */ - public String getPreviousResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder prevURL = new StringBuilder(); - - prevURL.append(getPath(request)).append("?"); - parametersExceptOffset(request, prevURL); - int offset = getPrevFirstHitNo() - 1; - prevURL.append("&").append("offset=").append(Integer.toString(offset)); - return prevURL.toString(); - } - - public String getCurrentResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder thisURL = new StringBuilder(); - - thisURL.append(getPath(request)).append("?"); - parameters(request, thisURL); - return thisURL.toString(); - } - - private String getPath(HttpRequest request) { - String path = request.getUri().getPath(); - if (path == null) { - path = ""; - } - return path; - } - - private void parametersExceptOffset(HttpRequest request, StringBuilder nextURL) { - int startLength = nextURL.length(); - for (Map.Entry<String, String> property : request.propertyMap().entrySet()) { - if (property.getKey().equals("offset")) continue; - - if (nextURL.length() > startLength) - nextURL.append("&"); - nextURL.append(property.getKey()).append("=").append(property.getValue()); - } - } - - private void parameters(HttpRequest request, StringBuilder nextURL) { - int startLength = nextURL.length(); - for (Map.Entry<String, String> property : request.propertyMap().entrySet()) { - if (nextURL.length() > startLength) - nextURL.append("&"); - nextURL.append(property.getKey()).append("=").append(property.getValue()); - } - } - - /** - * Returns the templates which will render the result. This is never null. - * If default rendering is used, it is a TemplateSet containing no - * templates. - * - * @deprecated use a renderer instead - */ - @SuppressWarnings("rawtypes") - // TODO: Remove on Vespa 7 - @Deprecated // OK - public UserTemplate getTemplates() { - if (renderer == null) { - return TemplateSet.getDefault(); - } else if (renderer instanceof SearchRendererAdaptor) { - return ((SearchRendererAdaptor) renderer).getAdaptee(); - } else { - throw new RuntimeException( - "Please use getTemplate() instead of getTemplates() when using the new template api."); - } - } - - /** - * Sets the template set which should render this result set - * - * @param templates - * the templates which should render this result, or null to - * use the default xml rendering - */ - @SuppressWarnings("deprecation") - public void setTemplates(@SuppressWarnings("rawtypes") UserTemplate templates) { - if (templates == null) { - setTemplates(TemplateSet.getDefault()); - } else { - setRenderer(new SearchRendererAdaptor(templates)); - } - } - - /** - * @deprecated since 5.1.21, use {@link Presentation#getRenderer()} - */ - public Renderer<Result> getRenderer() { - return renderer; - } - - /** - * @deprecated since 5.1.21, use {@link Presentation#setRenderer(com.yahoo.component.ComponentSpecification)} - */ - public void setRenderer(Renderer<Result> renderer) { - this.renderer = renderer; - } - - /** - * For internal use only. - */ - public boolean usesDefaultTemplate() { - return renderer == null || - (renderer instanceof SearchRendererAdaptor && - ((SearchRendererAdaptor) renderer).getAdaptee().isDefaultTemplateSet()); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java deleted file mode 100644 index a679b17b6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchchain.model.federation; - -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import net.jcip.annotations.Immutable; - -import com.yahoo.search.federation.http.HTTPProviderSearcher; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Specifies how a http provider is to be set up. - * - * @author Tony Vaagenes - * @deprecated - */ -@Immutable -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class HttpProviderSpec { - - public enum Type { - vespa(com.yahoo.search.federation.vespa.VespaSearcher.class); - - Type(Class<? extends HTTPProviderSearcher> searcherClass) { - className = searcherClass.getName(); - } - - final String className; - } - - // The default connection parameter values come from the config definition - public static class ConnectionParameters { - public final Double readTimeout; - public final Double connectionTimeout; - public final Double connectionPoolTimeout; - public final Integer retries; - - public ConnectionParameters(Double readTimeout, Double connectionTimeout, - Double connectionPoolTimeout, Integer retries) { - this.readTimeout = readTimeout; - this.connectionTimeout = connectionTimeout; - this.connectionPoolTimeout = connectionPoolTimeout; - this.retries = retries; - } - } - - public static class Node { - public final String host; - public final int port; - - public Node(String host, int port) { - this.host = host; - this.port = port; - } - - @Override - public String toString() { - return "Node{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; - } - } - - public final ConnectionParameters connectionParameters; - - public final Integer cacheSizeMB; - - public final String path; - public final List<Node> nodes; - public final String ycaApplicationId; - public final Integer ycaCertificateTtl; - public final Integer ycaRetryWait; - public final Node ycaProxy; - - public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) { - return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null); - } - - public static boolean includesType(String typeString) { - for (Type type : Type.values()) { - if (type.name().equals(typeString)) { - return true; - } - } - return false; - } - - public HttpProviderSpec(Double cacheWeight, - String path, - List<Node> nodes, - String ycaApplicationId, - Integer ycaCertificateTtl, - Integer ycaRetryWait, - Node ycaProxy, - Integer cacheSizeMB, - ConnectionParameters connectionParameters) { - - this.path = path; - this.nodes = unmodifiable(nodes); - this.ycaApplicationId = ycaApplicationId; - this.ycaProxy = ycaProxy; - this.ycaCertificateTtl = ycaCertificateTtl; - this.ycaRetryWait = ycaRetryWait; - this.cacheSizeMB = cacheSizeMB; - - this.connectionParameters = connectionParameters; - } - - private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) { - return nodes == null ? - Collections.emptyList() : - Collections.unmodifiableList(new ArrayList<>(nodes)); - } -} diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index cfbea04a110..5288b28cad1 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -57,7 +57,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private static final CompoundName streamingUserid=new CompoundName("streaming.userid"); private static final CompoundName streamingGroupname=new CompoundName("streaming.groupname"); private static final CompoundName streamingSelection=new CompoundName("streaming.selection"); - private static final CompoundName streamingHeadersonly=new CompoundName("streaming.headersonly"); private static final CompoundName streamingFromtimestamp=new CompoundName("streaming.fromtimestamp"); private static final CompoundName streamingTotimestamp=new CompoundName("streaming.totimestamp"); private static final CompoundName streamingLoadtype=new CompoundName("streaming.loadtype"); @@ -153,7 +152,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.setSessionTimeoutMs(query.getTimeout()); params.setVisitorLibrary("searchvisitor"); params.setLocalDataHandler(this); - params.setVisitHeadersOnly(query.properties().getBoolean(streamingHeadersonly)); if (query.properties().getDouble(streamingFromtimestamp) != null) { params.setFromTimestamp(query.properties().getDouble(streamingFromtimestamp).longValue()); } diff --git a/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj b/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj index 6a55a32eb8a..e1df35bb24a 100644 --- a/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj +++ b/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj @@ -169,7 +169,6 @@ TOKEN : <XOR: "xor"> | <XORBIT: "xorbit"> | <Y: "y"> | - <YMUM: "ymum"> | <ZCURVE: "zcurve"> | <IDENTIFIER: ["A"-"Z","a"-"z"](["A"-"Z","a"-"z","0"-"9","_","@"])*> } @@ -374,7 +373,6 @@ GroupingExpression value(GroupingOperation grp) : exp = ucaFunction(grp) | exp = xorExpression(grp) | exp = xorBitFunction(grp) | - exp = ymumValue() | exp = zcurveFunction(grp) ) | ( exp = attributeValue() ) ) { return exp; } @@ -912,12 +910,6 @@ XorBitFunction xorBitFunction(GroupingOperation grp) : { return new XorBitFunction(exp, num.intValue()); } } -YmumValue ymumValue() : { } -{ - ( <YMUM> lbrace() rbrace() ) - { return new YmumValue(); } -} - void bucket(GroupingOperation grp, BucketResolver resolver) : { ConstantValue from, to = null; @@ -1076,7 +1068,6 @@ String identifier() : <XOR> | <XORBIT> | <Y> | - <YMUM> | <ZCURVE> ) { ret = token.image; } space() ) { return ret; } } diff --git a/container-search/src/main/resources/configdefinitions/legacy-emulation.def b/container-search/src/main/resources/configdefinitions/legacy-emulation.def deleted file mode 100644 index 95469338bb7..00000000000 --- a/container-search/src/main/resources/configdefinitions/legacy-emulation.def +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.search - -## Keep emulating old (4.2, 5.0, early 5.1) string type fields for structured data -stringBackedStructuredData bool default=false - -## Keep emulating old (4.2, 5.0, early 5.1) string type fields for summaryfeatures / rankfeatures fields -stringBackedFeatureData bool default=false - -## This will populate even if they are empty/undefined. -forceFillEmptyFields bool default=false diff --git a/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def b/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def deleted file mode 100644 index ba2b6ed7802..00000000000 --- a/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: Not in use - remove on Vespa 7 - -namespace=search.cache - -# size of the region (in MB). Cache size (in qr-binary-cache.cfg) -# and region size should be same -region_size int default=1024 - -# Number of threads that would concurrently access cache -concurrency_level int default=50 - -# Initial bucket count that would be created in a region -# (rehashing would grow it if necessary but is expensive) -bucket_count int default=12000 - -# Load factor of the region. -load_factor double default=0.75 - -#TTL in milli-seconds -time_to_live int default=-1 - -entry_size_range int default=500 - - -#Whether lazy invalidation is enabled or no -lazy_invalidation_enabled bool default=false - -# Time interval (in seconds) over which lazy invalidation parameters would be observed -tick_interval int default=1 -tick_count_to_average int default=10 - -# Maximum QPS that can be supported by the back-end. -# Refresh due to lazy invalidation and TTL would be disabled in the -# tick_interval in which total requests to backend have exceeded MAX_QPS -max_backend_qps int default=300 - -# Log file in which lazy invalidation stats would be -# recorded (keep it empty if no stats are required) -lazy_invalidation_stats_file string default="lstats.log" - -# Maximum stale time in seconds after a call to lazy invalidation is made. -# Once this time interval is expired, any request for an expired -# entry (due to TTL or lazy invalidation) would result in a cache miss. -lazy_invalidation_max_stale_time_in_sec int default=1000 diff --git a/container-search/src/main/resources/configdefinitions/qr-binary-cache.def b/container-search/src/main/resources/configdefinitions/qr-binary-cache.def deleted file mode 100644 index 917832e86fe..00000000000 --- a/container-search/src/main/resources/configdefinitions/qr-binary-cache.def +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: Not in use - remove on Vespa 7 - -namespace=search.cache - -# Size of a block in cache. A block is the smallest unit that would -# be allocated to a cache entry (key+value). Multiple blocks can be -# allocated to an entry if required and blocks can not be shared -# between multiple entries. IN BYTES -block_size int default=256 - -# Size of a buffer array that would hold the serialized content, -# maximum size here can be 2 GB. IN MB -buffer_size int default=64 - -# For debugging (for production it should be true) -assertions_disabled bool default=false - -# Whether to create the byte buffers in the NIO memory or in -# JVM heap itself (having these in NIO buffers gives lower GC pause times) -use_direct_buffers bool default=true - -# Size of the byte array pool that the cache uses to hold up -# the serialized content temporarily -pooled_byte_arrays int default=1000 - -# The size of the cahe in MB -# If the size is 0, the cache is disabled -cache_size int default=0 - -# If true, cache would write average time taken in various -# operations per 1000 requests in log -profiling_enabled bool default=false - -# For debugging purposes -lock_verification_enabled bool default=false diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def index d29ebc8a826..8547ad08458 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/qr-start.def @@ -9,7 +9,7 @@ jvm.server bool default=true restart jvm.verbosegc bool default=true restart ## Garbage Collection tuning parameters -jvm.gcopts string default="-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" restart +jvm.gcopts string default="-XX:+UseG1GC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" restart ## Heap size (in megabytes) for the Java VM jvm.heapsize int default=1536 restart @@ -27,6 +27,10 @@ jvm.directMemorySizeCache int default=0 restart ## value above. Setting outside [1, 99] disables this setting. jvm.heapSizeAsPercentageOfPhysicalMemory int default=0 restart +## Number of processors available, can be used to set -XX:ActiveProcessorCount if set to non-zero. +## In that case will be the number returned by the JVM when calling Runtime.getRuntime().availableProcessors() +jvm.availableProcessors int default=0 restart + ## Extra environment variables qrs.env string default="YELL_MA_EURO=INXIGHT" restart diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java index 9b1517454a9..54512e727c2 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java @@ -50,7 +50,7 @@ public class GetDocSumsPacketTestCase { @Test public void requireThatSessionIdIsEncodedAsPropertyWhenUsingSearchSession() throws BufferTooSmallException { Result result = new Result(new Query("?query=foo")); - SessionId sessionId = result.getQuery().getSessionId(true); // create session id. + SessionId sessionId = result.getQuery().getSessionId("node-0"); result.getQuery().getRanking().setQueryCache(true); FastHit hit = new FastHit(); result.hits().add(hit); diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java index 78b2f65305a..e2ce26cb908 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java @@ -12,6 +12,7 @@ import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.WeightedSetItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; +import com.yahoo.search.query.ranking.SoftTimeout; import org.junit.Test; import java.nio.ByteBuffer; @@ -52,6 +53,8 @@ public class QueryTestCase { @Test public void testEncodeQueryPacketWithSomeAdditionalFeatures() { Query query = new Query("/?query=chain&dataset=10&type=phrase&timeout=0"); + query.properties().set(SoftTimeout.enableProperty, false); + // Because the rank mapping now needs config and a searcher, // we do the sledgehammer dance: query.getRanking().setProfile("two"); @@ -79,7 +82,7 @@ public class QueryTestCase { "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + - "&model.searchPath=7/3"); + "&model.searchPath=7/3&ranking.softtimeout.enable=false"); query.getRanking().setFreshness(new Freshness("123456")); query.getRanking().setSorting("+field1 -field2"); query.getRanking().setProfile("two"); @@ -105,7 +108,7 @@ public class QueryTestCase { "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + - "&model.searchPath=7/3"); + "&model.searchPath=7/3&ranking.softtimeout.enable=false"); query.getRanking().setFreshness("123456"); query.getRanking().setSorting("+field1 -field2"); query.getRanking().setProfile("two"); @@ -127,6 +130,7 @@ public class QueryTestCase { @Test public void testEncodeQueryPacketWithLabelsConnectivityAndSignificance() { Query query = new Query(); + query.properties().set(SoftTimeout.enableProperty, false); AndItem and = new AndItem(); WeightedSetItem taggable1 = new WeightedSetItem("field1"); taggable1.setLabel("foo"); diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 646c466eba9..25d6a6e9777 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -6,7 +6,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.QrConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.search.Fs4Config; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.fs4.QueryPacket; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; @@ -540,7 +539,6 @@ public class ClusterSearcherTestCase { qrSearchersConfig.build(), clusterConfig.build(), documentDbConfig.build(), - new LegacyEmulationConfig.Builder().build(), new QrMonitorConfig.Builder().build(), new DispatchConfig.Builder().build(), createClusterInfoConfig(), @@ -555,7 +553,7 @@ public class ClusterSearcherTestCase { clusterInfoConfigBuilder.nodeCount(1); return new ClusterInfoConfig(clusterInfoConfigBuilder); } - + private static class QueryTimeoutFixture { ClusterSearcher searcher; Execution exec; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 81f16f7f261..ad4d0cf82e5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableSet; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; @@ -66,33 +65,6 @@ public class SlimeSummaryTestCase { } @Test - public void testDecodingEmptyWithLegacyEmulation() { - LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig); - FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", emptySummary(), hit)); - assertEquals(NanNumber.NaN, hit.getField("integer_field")); - assertEquals(NanNumber.NaN, hit.getField("short_field")); - assertEquals(NanNumber.NaN, hit.getField("byte_field")); - assertEquals(NanNumber.NaN, hit.getField("float_field")); - assertEquals(NanNumber.NaN, hit.getField("double_field")); - assertEquals(NanNumber.NaN, hit.getField("int64_field")); - assertEquals("", hit.getField("string_field")); - assertEquals(RawData.class, hit.getField("data_field").getClass()); - assertEquals("", hit.getField("data_field").toString()); - assertEquals("", hit.getField("longstring_field")); - assertEquals(RawData.class, hit.getField("longdata_field").getClass()); - assertEquals("", hit.getField("longdata_field").toString()); - assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); - assertEquals("", hit.getField("xmlstring_field").toString()); - // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); - assertEquals("", hit.getField("jsonstring_field").toString()); - // Empty tensors are represented by null because we don't have type information here to create the right empty tensor - assertNull(hit.getField("tensor_field1")); - assertNull(hit.getField("tensor_field2")); - } - - @Test public void testTimeout() { DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); @@ -424,11 +396,6 @@ public class SlimeSummaryTestCase { return new DocsumDefinitionSet(config.documentdb(0)); } - private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); - } - private static class Utf8FieldTraverser implements Hit.RawUtf8Consumer { private Map<String, Object> traversed; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index c4d32111124..fd01fb9a740 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -153,7 +153,7 @@ public class FastSearcherTestCase { } { // direct.summaries due to query cache - String query = "?query=sddocname:a&ranking.queryCache"; + String query = "?query=sddocname:a&ranking.queryCache&timeout=5000ms"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); @@ -162,7 +162,7 @@ public class FastSearcherTestCase { } { // direct.summaries due to no summary features - String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler"; + String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler&timeout=5000ms"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); @@ -185,6 +185,7 @@ public class FastSearcherTestCase { documentdbConfigWithOneDb); Query query = new Query("?query=foo&model.restrict=testDb"); + query.getRanking().getSoftTimeout().setEnable(false); query.prepare(); doSearch(fastSearcher, query, 0, 10); @@ -330,7 +331,7 @@ public class FastSearcherTestCase { byte[] actual = new byte[buf.remaining()]; buf.get(actual); - SessionId sessionId = query.getSessionId(false); + SessionId sessionId = query.getSessionId(); byte IGNORE = 69; ByteBuffer answer = ByteBuffer.allocate(1024); answer.put(new byte[] { 0, 0, 0, (byte)(141+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 16, 17, 0, 0, 0, 0, diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java index db5397e5292..b697841a969 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java @@ -60,12 +60,6 @@ public class TestLinguistics implements Linguistics { return linguistics.getCharacterClasses(); } - @Override - @Deprecated - public Tuple2<String, Version> getVersion(Linguistics.Component component) { - return linguistics.getVersion(component); - } - } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg deleted file mode 100644 index a8790cc4051..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg +++ /dev/null @@ -1,29 +0,0 @@ -indexinfo[1] -indexinfo[0].name combinedattributeandindexsearch -indexinfo[0].command[12] -indexinfo[0].command[0].indexname sddocname -indexinfo[0].command[0].command index -indexinfo[0].command[1].indexname index1 -indexinfo[0].command[1].command index -indexinfo[0].command[2].indexname default -indexinfo[0].command[2].command index -indexinfo[0].command[3].indexname index1 -indexinfo[0].command[3].command stem -indexinfo[0].command[4].indexname default -indexinfo[0].command[4].command stem -indexinfo[0].command[5].indexname index1 -indexinfo[0].command[5].command index1 -indexinfo[0].command[5].command normalize -indexinfo[0].command[6].indexname default -indexinfo[0].command[6].command normalize -indexinfo[0].command[7].indexname attribute1 -indexinfo[0].command[7].command index -indexinfo[0].command[8].indexname attribute1 -indexinfo[0].command[8].command attribute -indexinfo[0].command[9].indexname attribute2 -indexinfo[0].command[9].command attribute -indexinfo[0].command[10].indexname default -indexinfo[0].command[10].command "match-group default attribute1 attribute2" -indexinfo[0].command[11].indexname ranklog -indexinfo[0].command[11].command xmlstring - diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index fae869c5235..47009eb703b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -51,20 +51,20 @@ public class BlendingSearcherTestCase { private final Map<String, Searcher> searchers = new HashMap<>(); private SearchChainRegistry chainRegistry; - private final String blendingDocumentId; + private final String blendingField; public BlendingSearcherWrapper() { - blendingDocumentId = null; + blendingField = null; } - public BlendingSearcherWrapper(String blendingDocumentId) { - this.blendingDocumentId = blendingDocumentId; + public BlendingSearcherWrapper(String blendingField) { + this.blendingField = blendingField; } @SuppressWarnings("serial") public BlendingSearcherWrapper(QrSearchersConfig cfg) { QrSearchersConfig.Com.Yahoo.Prelude.Searcher.BlendingSearcher s = cfg.com().yahoo().prelude().searcher().BlendingSearcher(); - blendingDocumentId = s.docid().length() > 0 ? s.docid() : null; + blendingField = s.docid().length() > 0 ? s.docid() : null; } public boolean addChained(Searcher searcher, String sourceName) { @@ -109,7 +109,7 @@ public class BlendingSearcherTestCase { FederationSearcher fedSearcher = new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>()); - BlendingSearcher blendingSearcher = new BlendingSearcher(blendingDocumentId); + BlendingSearcher blendingSearcher = new BlendingSearcher(blendingField); blendingChain = new SearchChain(ComponentId.createAnonymousComponentId("blendingChain"), blendingSearcher, fedSearcher); return true; } @@ -217,7 +217,7 @@ public class BlendingSearcherTestCase { r2.setTotalHitCount(1); chain2.addResult(q, r2); - BlendingSearcherWrapper blender = new BlendingSearcherWrapper("uri"); + BlendingSearcherWrapper blender = new BlendingSearcherWrapper("[id]"); blender.addChained(new FillSearcher(chain1), "a"); blender.addChained(new FillSearcher(chain2), "b"); blender.initialize(); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index bf86ca4544b..4875121a501 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -336,11 +336,6 @@ public class FieldCollapsingSearcherTestCase { assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); } - // This test depends on DocumentSourceSearcher filling the hits - // with whatever data it got, ignoring actual summary arguments - // in the fill call, then saying the hits are filled for the - // ignored argument. Rewrite to contain different summaries if - // DocumentSourceSearcher gets extended. @Test public void testFieldCollapsingTwoPhaseSelectSummary() { // Set up diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java index 9f4a12d24e6..24af91cb5c0 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -47,7 +47,7 @@ public class JuniperSearcherTestCase { DocumentSourceSearcher docsource = new DocumentSourceSearcher(); addResult(new Query("?query=12"), sdName, content, docsource); addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); - return new Chain<Searcher>(searcher, docsource); + return new Chain<>(searcher, docsource); } private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java deleted file mode 100644 index 7b92e2c1a7c..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.prelude.searcher.QuerySnapshotSearcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class QuerySnapshotSearcherTestCase { - - @Test - public void test() { - Searcher searcher=new QuerySnapshotSearcher(); - Result result = doSearch(searcher, new Query(), 0,10); - Hit hit=result.hits().get(0); - assertEquals(String.valueOf(Double.POSITIVE_INFINITY), - hit.getRelevance().toString()); - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher).search(query); - } - - private Execution createExecution(Searcher searcher) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher), context); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java deleted file mode 100644 index 3967ac63e36..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; -import com.yahoo.prelude.searcher.QueryValidatingSearcher; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Tests correct denial of query. - * - * @author Steinar Knutsen - */ -public class QueryValidatingSearcherTestCase { - - @Test - public void testBasic() { - // Setup - Map<Searcher, Searcher> chained = new HashMap<>(); - Query query = new Query("?query=test"); - - Result result = new Result(query); - result.hits().add(new Hit("ymail://1111111111/AQAAAP7JgwEAj6XjQQAAAO/+ggA=",950)); - - Searcher validator = new QueryValidatingSearcher(); - DocumentSourceSearcher source = new DocumentSourceSearcher(); - chained.put(validator, source); - source.addResult(query, result); - - // Exercise - Result returnedResult = doSearch(validator, query, 0, 10, chained); - - // Validate - assertEquals(1, returnedResult.getHitCount()); - assertNull(returnedResult.hits().getError()); - - returnedResult = doSearch(validator, query, 0, 1001, chained); - assertEquals(0, returnedResult.getConcreteHitCount()); - assertEquals(4, returnedResult.hits().getError().getCode()); - - returnedResult = doSearch(validator, query, 1001, 10, chained); - assertEquals(0, returnedResult.getConcreteHitCount()); - assertEquals(4, returnedResult.hits().getError().getCode()); - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits, Map<Searcher, Searcher> chained) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher, chained).search(query); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) { - List<Searcher> searchers = new ArrayList<>(); - for (Searcher current = topOfChain; current != null; current = chained.get(current)) { - searchers.add(current); - } - return new Chain<>(searchers); - } - - private Execution createExecution(Searcher searcher, Map<Searcher, Searcher> chained) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher, chained), context); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java deleted file mode 100644 index 0bbc5151b40..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import java.io.IOException; -import java.io.Writer; - -import com.yahoo.prelude.templates.Context; -import com.yahoo.prelude.templates.UserTemplate; - -/** - * Test template which throws a runtime exception in its footer. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("rawtypes") -public class BoomTemplate extends UserTemplate { - public BoomTemplate(String name, String mimeType, String encoding) { - super(name, mimeType, encoding); - } - - @Override - public void error(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void footer(Context context, Writer writer) throws IOException { - throw new RuntimeException("Boom!"); - } - - @Override - public void header(Context context, Writer writer) throws IOException { - writer.write("header"); - } - - @Override - public void hit(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void hitFooter(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void noHits(Context context, Writer writer) throws IOException { - // NOP - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java deleted file mode 100644 index 07d5dcc9e2d..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * Tests composition of grouped results using the HitGroup class - * - * @author bratseth - */ -public class GroupedResultTestCase { - - @Test - public void testGroupedResult() { - Result result=new Result(new Query("?query=foo")); - HitGroup hitGroup1=new HitGroup("group1",300); - hitGroup1.add(new Hit("group1.1",200)); - HitGroup hitGroup2=new HitGroup("group2",600); - Hit topLevelHit1=new Hit("toplevel.1",500); - Hit topLevelHit2=new Hit("toplevel.2",700); - result.hits().add(hitGroup1); - result.hits().add(topLevelHit1); - result.hits().add(hitGroup2); - result.hits().add(topLevelHit2); - hitGroup1.add(new Hit("group1.2",800)); - hitGroup2.add(new Hit("group2.1",800)); - hitGroup2.add(new Hit("group2.2",300)); - hitGroup2.add(new Hit("group2.3",500)); - - // Should have 7 concrete hits, ordered as - // toplevel.2 - // group2 - // group2.1 - // group2.3 - // group2.2 - // toplevel.1 - // group1 - // group1.2 - // group1.1 - // Assert this: - - assertEquals(7,result.getConcreteHitCount()); - assertEquals(4,result.getHitCount()); - - Hit topLevel2=result.hits().get(0); - assertEquals("toplevel.2",topLevel2.getId().stringValue()); - - HitGroup returnedGroup2=(HitGroup)result.hits().get(1); - assertEquals(3,returnedGroup2.getConcreteSize()); - assertEquals(3,returnedGroup2.size()); - assertEquals("group2.1",returnedGroup2.get(0).getId().stringValue()); - assertEquals("group2.3",returnedGroup2.get(1).getId().stringValue()); - assertEquals("group2.2",returnedGroup2.get(2).getId().stringValue()); - - Hit topLevel1=result.hits().get(2); - assertEquals("toplevel.1",topLevel1.getId().stringValue()); - - HitGroup returnedGroup1=(HitGroup)result.hits().get(3); - assertEquals(2,returnedGroup1.getConcreteSize()); - assertEquals(2,returnedGroup1.size()); - assertEquals("group1.2",returnedGroup1.get(0).getId().stringValue()); - assertEquals("group1.1",returnedGroup1.get(1).getId().stringValue()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java deleted file mode 100644 index 6ff8c2f9d6c..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import static org.junit.Assert.assertEquals; - -import java.lang.reflect.Method; -import java.util.List; - -import org.junit.Test; - -import com.yahoo.prelude.templates.HitContext; -import com.yahoo.protect.ClassValidator; - -/** - * Check the entire Context class is correctly masked. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class HitContextTestCase { - - @Test - public void checkMethods() { - List<Method> unmasked = ClassValidator.unmaskedMethodsFromSuperclass(HitContext.class); - assertEquals("Unmasked methods from superclass: " + unmasked, 0, unmasked.size()); - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java deleted file mode 100644 index 6f49ebbdee0..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import java.io.ByteArrayOutputStream; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; - -import com.yahoo.io.ByteWriter; -import com.yahoo.prelude.templates.UserTemplate; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author Steinar Knutsen - */ -public class TemplateTestCase { - - private CharsetEncoder encoder; - private ByteArrayOutputStream stream; - - public TemplateTestCase () { - Charset cs = Charset.forName("UTF-8"); - encoder = cs.newEncoder(); - stream = new ByteArrayOutputStream(); - } - - @Test - public void testASCIIQuoting() throws java.io.IOException { - stream.reset(); - byte[] c = new byte[] { 97, 98, 99, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }; - ByteWriter bw = new ByteWriter(stream, encoder); - UserTemplate.dumpAndXMLQuoteUTF8(bw, c); - bw.close(); - String res = stream.toString("UTF-8"); - String correct = "abc\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\t\n\\u000B\\u000C\r\\u000E\\u000F\\u0010\\u0011"; - assertEquals(correct, res); - - } - - @Test - public void testXMLQuoting() throws java.io.IOException { - stream.reset(); - // c = <s>> - byte[] c = new byte[] { 60, 115, 62, 38, 103, 116, 59 }; - ByteWriter bw = new ByteWriter(stream, encoder); - UserTemplate.dumpAndXMLQuoteUTF8(bw, c); - bw.close(); - String res = stream.toString("UTF-8"); - String correct = "<s>&gt;"; - assertEquals(correct, res); - - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java deleted file mode 100644 index 0f5e126dae8..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import java.io.IOException; -import java.io.Writer; - -import com.yahoo.prelude.templates.Context; -import com.yahoo.prelude.templates.UserTemplate; - -/** - * Test basic UserTemplate functionality of detecting - * overridden group rendering methods. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("rawtypes") -public class TestTemplate extends UserTemplate { - - public TestTemplate(String name, String mimeType, String encoding) { - super(name, mimeType, encoding); - } - - @Override - public void error(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void footer(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void header(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void hit(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void hitFooter(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void noHits(Context context, Writer writer) throws IOException { - // NOP - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java deleted file mode 100644 index f86d76a12a4..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.io.IOUtils; -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.federation.http.HTTPProviderSearcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests representing a federated and grouped result as a Result object and - * rendering a tiled output of the result - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class TilingTestCase { - - /** - * This result contains two blocks (center and right). - * The center block contains multiple subblocks while the right one contains a single block of ads. - */ - @Test - public void testTiling() throws IOException { - Chain<Searcher> chain=new Chain<>("tiling", new TiledResultProducer()); - - // Query it - Query query = new Query("/tiled?query=foo"); - Result result = callSearchAndSetRenderer(chain, query); - assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml")),result); - } - - /** - * This result contains center section and meta blocks. - */ - @Test - public void testTiling2() throws IOException { - Chain<Searcher> chain= new Chain<>("tiling", new TiledResultProducer2()); - - // Query it - Query query=new Query("/tiled?query=foo"); - Result result= callSearchAndSetRenderer(chain, query); - assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml")),result); - } - - private Result callSearchAndSetRenderer(Chain<Searcher> chain, Query query) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - Result result = new Execution(chain, context).search(query); - result.getTemplating().setRenderer(new SearchRendererAdaptor(new TiledTemplateSet())); - return result; - } - - public static void assertRenderedStartsWith(String expected,Result result) throws IOException { - assertRendered(expected,result,false); - } - - public static void assertRendered(String expected,Result result) throws IOException { - assertRendered(expected,result,true); - } - - public static void assertRendered(String expected, Result result,boolean checkFullEquality) throws IOException { - if (checkFullEquality) - assertEquals(filterComments(expected), getRendered(result)); - else - assertTrue(getRendered(result).startsWith(expected)); - } - - private static String filterComments(String s) { - StringBuilder b = new StringBuilder(); - for (String line : s.split("\\\n")) - if ( ! line.startsWith("<!--")) - b.append(line).append("\n"); - return b.toString(); - } - - public static String getRendered(Result result) throws IOException { - if (result.getTemplating().getRenderer() == null) - result.getTemplating().setTemplates(null); - - // Done in a roundabout way to simulate production code path - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - Charset cs = Charset.forName(result.getTemplating().getRenderer().getEncoding()); - CharsetDecoder decoder = cs.newDecoder(); - SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); - } - - private static class TiledResultProducer extends Searcher { - - @Override - public Result search(Query query, Execution execution) { - Result result = new Result(query); - result.setTotalHitCount(2800000000l); - - // Blocks - HitGroup centerBlock=(HitGroup)result.hits().add(new HitGroup("section:center")); - centerBlock.types().add("section"); - centerBlock.setField("region","center"); - - HitGroup rightBlock=(HitGroup)result.hits().add(new HitGroup("section:right")); - rightBlock.types().add("section"); - rightBlock.setField("region","right"); - - // Center groups - HitGroup navigation=(HitGroup)centerBlock.add(new HitGroup("group:navigation",1.0)); - navigation.types().add("group"); - navigation.setField("type","navigation"); - - HitGroup adsNorth=(HitGroup)centerBlock.add(new HitGroup("group:ads:north",0.9)); - adsNorth.types().add("group"); - adsNorth.setField("type","ads"); - - HitGroup hits=(HitGroup)centerBlock.add(new HitGroup("group:navigation",0.8)); - hits.types().add("group"); - hits.setField("type","hits"); - - HitGroup adsSouth=(HitGroup)centerBlock.add(new HitGroup("group:ads:south",0.7)); - adsSouth.types().add("group"); - adsSouth.setField("type","ads"); - - // Right group - HitGroup adsRight=(HitGroup)rightBlock.add(new HitGroup("group:ads:right",0.7)); - adsRight.types().add("group"); - adsRight.setField("type","ads"); - - // Navigation content - /* - Hit alsoTry=navigation.add(new Hit("alsotry")); - alsoTry.types().add("alsotry"); - alsoTry.setMeta(true); - LinkList links=new LinkList(); - links.add(new Link("Hilton","?search=Hilton hotel")); - links.add(new Link("Habbo hotel","?search=Habbo hotel")); - links.add(new Link("Marriott","?search=Marriott hotel")); - alsoTry.setField("links",links); - */ - - // North ads content - Hit ad1=adsNorth.add(new Hit("http://www.hotels.com",0.7)); - ad1.types().add("ad"); - ad1.setAuxiliary(true); - ad1.setField("title",new XMLString("Cheap <hi>hotels</hi>")); - ad1.setField("body",new XMLString("Low Rates Guaranteed. Call a <hi>Hotel</hi> Expert.")); - - Hit ad2=adsNorth.add(new Hit("http://www.expedia.com",0.6)); - ad2.types().add("ad"); - ad2.setAuxiliary(true); - ad2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia")); - ad2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms."); - -// // Hits content -// // - news hit -// HitGroup news1=(HitGroup)hits.add(new HitGroup("newsarticles",0.9)); -// news1.setMeta(false); -// news1.types().add("news"); -// news1.setField("title","Hotel - News results"); -// Hit article1=news1.add(new Hit("www.miamiherald.com/?article=jhsgd7323",0.5)); -// article1.setAuxiliary(true); -// article1.setField("title","Celebrity blackout: The Hilton of Paris changes name to regain search traffic"); -// article1.types().add("newsarticle"); -// article1.setField("age",23); -// article1.setField("source","Miami Herald"); -// Hit article2=news1.add(new Hit("www.sfgate.com/?article=8763khj7",0.4)); -// article2.setAuxiliary(true); -// article2.setField("title","Hotels - more expensive than staying at home"); -// article2.types().add("newsarticle"); -// article2.setField("age",3500); -// article2.setField("source","SF Gate"); - - // - collapsed hit - Hit hit1=hits.add(new Hit("www.hotels.com",0.8)); - hit1.types().add("hit.collapsed"); - hit1.setField("title","Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging"); - hit1.setField("body",new XMLString("Hotels.com helps you find great rates on hotels and discount <hi>hotel</hi> packages.")); - /* - LinkList collapsed1=new LinkList(); - collapsed1.add(new Link("Last Minute Deals","www.hotels.com/lastminutedeals")); - collapsed1.add(new Link("Hotel Savings","www.hotels.com/deals")); - collapsed1.add(new Link("800-994-6835","www.hotels.com/?PSRC=OT2")); - hit1.setField("links",collapsed1); - */ - - // regular hit with links - Hit hit2=hits.add(new Hit("www.indigohotels.com",0.7)); - hit2.types().add("hit"); - hit2.setField("title","Hotel Indigo Hotels United States - Official Web Site"); - hit2.setField("body","Make Hotel Indigo online hotel reservations and book your hotel rooms today."); - /* - LinkList collapsed2=new LinkList(); - collapsed2.add(new Link("800-333-6835","www.indigohotels.com/order")); - collapsed2.add(new Link("Reservations","www.indigohotels.com/reservations")); - hit2.setField("links",collapsed2); - */ - - // boring old hit - Hit hit3=hits.add(new Hit("www.all-hotels.com",0.6)); - hit3.types().add("hit"); - hit3.setField("title","All hotels"); - hit3.setField("body","Online hotel directory and reservations."); - - // South ads - Hit southAd1=adsSouth.add(new Hit("www.daysinn.com",1.0)); - southAd1.types().add("ad"); - southAd1.setAuxiliary(true); - southAd1.setField("title","Days Inn Special Deal"); - southAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn."); - Hit southAd2=adsSouth.add(new Hit("http://www.expedia.com",0.9)); - southAd2.types().add("ad"); - southAd2.setAuxiliary(true); - southAd2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia")); - southAd2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms."); - - // Right ads - Hit rightAd1=adsRight.add(new Hit("www.daysinn.com",1.0)); - rightAd1.types().add("ad"); - rightAd1.setAuxiliary(true); - rightAd1.setField("title","Days Inn Special Deal"); - rightAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn."); - Hit rightAd2=adsRight.add(new Hit("www.holidayinn.com",0.9)); - rightAd2.types().add("ad"); - rightAd2.setAuxiliary(true); - rightAd2.setField("title","Holiday Inn: Official Site"); - rightAd2.setField("body","Book with Holiday Inn. Free Internet. Kids eat free."); - - // Done creating result - must analyze because we add ads then later set them as auxiliary - result.analyzeHits(); - - return result; - } - - } - - private static class TiledResultProducer2 extends Searcher { - - @Override - public Result search(Query query,Execution execution) { - Result result=new Result(query); - result.setTotalHitCount(1); - - HitGroup section = new HitGroup("section:center"); - result.hits().add(section); - section.setMeta(false); - section.types().add("section"); - section.setField("region", "center"); - - HitGroup yst = new HitGroup("yst"); - section.add(yst); - yst.setMeta(false); - yst.setSource("sr"); - yst.types().add("sr"); - yst.setField("provider", "yst"); - - Hit theHit = new Hit("159"); - yst.add(theHit); - theHit.setSource("sr"); - theHit.types().add("sr"); - theHit.setField("provider", "yst"); - theHit.setField("title", "Yahoo"); - - HitGroup meta = new HitGroup("meta"); - result.hits().add(meta); - meta.types().add("meta"); - - Hit log = new Hit("com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search"); - meta.add(log); - log.setMeta(true); - log.setSource("sr"); - log.setField("provider", "yst"); - log.types().add("logging"); - log.setField(HTTPProviderSearcher.LOG_URI, "http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2"); - log.setField(HTTPProviderSearcher.LOG_SCHEME, "http"); - log.setField(HTTPProviderSearcher.LOG_HOST, "proxy-tw1cache.idp.inktomisearch.com"); - log.setField(HTTPProviderSearcher.LOG_PORT, "55556"); - log.setField(HTTPProviderSearcher.LOG_PATH, "/search"); - log.setField(HTTPProviderSearcher.LOG_STATUS, "200"); - log.setField(HTTPProviderSearcher.LOG_LATENCY_CONNECT, "757"); - log.setField(HTTPProviderSearcher.LOG_RESPONSE_HEADER_PREFIX + "content-length", "16217"); - - result.analyzeHits(); - - return result; - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ deleted file mode 100644 index ca186b86259..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ +++ /dev/null @@ -1 +0,0 @@ -<ERROR CODE="$result.error.code">$result.error.message</ERROR> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ deleted file mode 100644 index 07a5dd6a881..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ +++ /dev/null @@ -1 +0,0 @@ -</RESULTSET>
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ deleted file mode 100644 index 8e3a001545d..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<RESULTSET TOTALHITS="$result.totalHitCount"> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ deleted file mode 100644 index 428a2f15ef5..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ +++ /dev/null @@ -1,5 +0,0 @@ -<XTEMPLATEHIT RELEVANCY="$relevancy" SOURCE="$hit.source" TYPE="$hit.typeString" OFFSET="$hitno"> -<FIELD NAME="uri">$uri</FIELD> -<FIELD NAME="category">$category</FIELD> -<FIELD NAME="bsumtitle">$bsumtitle</FIELD> -</XTEMPLATEHIT> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ deleted file mode 100644 index d466f0bb7d2..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ +++ /dev/null @@ -1 +0,0 @@ -<XTEMPLATENOHITS/>
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml b/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml deleted file mode 100644 index ea5d56d2b31..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml +++ /dev/null @@ -1,65 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result version="1.0"> - - <section id="section:center" region="center"> - <group type="navigation" relevance="1.0"></group> - <group type="ads" relevance="0.9"> - <hit type="ad" relevance="0.7"> - <id>http://www.hotels.com/</id> - <title>Cheap <hi>hotels</hi></title> - <body>Low Rates Guaranteed. Call a <hi>Hotel</hi> Expert.</body> - </hit> - <hit type="ad" relevance="0.6"> - <id>http://www.expedia.com/</id> - <title>Cheap <hi>hotels</hi> at Expedia</title> - <body>Expedia Special Rates Means We Guarantee Our Low Rates on Rooms.</body> - </hit> - </group> - <group type="hits" relevance="0.8"> - <hit type="hit.collapsed" relevance="0.8"> - <id>www.hotels.com</id> - <title>Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging</title> - <body>Hotels.com helps you find great rates on hotels and discount <hi>hotel</hi> packages.</body> - </hit> - <hit type="hit" relevance="0.7"> - <id>www.indigohotels.com</id> - <title>Hotel Indigo Hotels United States - Official Web Site</title> - <body>Make Hotel Indigo online hotel reservations and book your hotel rooms today.</body> - </hit> - <hit type="hit" relevance="0.6"> - <id>www.all-hotels.com</id> - <title>All hotels</title> - <body>Online hotel directory and reservations.</body> - </hit> - </group> - <group type="ads" relevance="0.7"> - <hit type="ad" relevance="1.0"> - <id>www.daysinn.com</id> - <title>Days Inn Special Deal</title> - <body>Buy now and Save 15% Off Our Best Available Rate with Days Inn.</body> - </hit> - <hit type="ad" relevance="0.9"> - <id>http://www.expedia.com/</id> - <title>Cheap <hi>hotels</hi> at Expedia</title> - <body>Expedia Special Rates Means We Guarantee Our Low Rates on Rooms.</body> - </hit> - </group> - </section> - - <section id="section:right" region="right"> - <group type="ads" relevance="0.7"> - <hit type="ad" relevance="1.0"> - <id>www.daysinn.com</id> - <title>Days Inn Special Deal</title> - <body>Buy now and Save 15% Off Our Best Available Rate with Days Inn.</body> - </hit> - <hit type="ad" relevance="0.9"> - <id>www.holidayinn.com</id> - <title>Holiday Inn: Official Site</title> - <body>Book with Holiday Inn. Free Internet. Kids eat free.</body> - </hit> - </group> - </section> - -</result> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml b/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml deleted file mode 100644 index 2e936b0015a..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result version="1.0"> - - <section id="section:center" region="center"> - <group type="sr" relevance="1.0" source="sr" provider="yst"> - <hit type="sr" relevance="1.0" source="sr" provider="yst"> - <id>159</id> - <title>Yahoo</title> - </hit> - </group> - </section> - - <meta> - <provider name="yst" scheme="http" host="proxy-tw1cache.idp.inktomisearch.com" port="55556" path="/search" result="200"> - <id>com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search</id> - <uri>http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2</uri> - <latency type="connect">757</latency> - <response-header name="content-length">16217</response-header> - </provider> - </meta> - -</result> diff --git a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java deleted file mode 100644 index 3e57c48e7f3..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.templates.test.TilingTestCase; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -/** - * Tests that rank features are rendered when requested in the query - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class RankFeatureDumpTestCase { - - private static final String rankFeatureString= - "{\"match.weight.as1\":10,\"attribute(ai1)\":1.000000,\"proximity(as1, 1, 2)\":2.000000}"; - - @Test - public void test() throws IOException { - Query query=new Query("?query=five&rankfeatures"); - assertTrue(query.getRanking().getListFeatures()); // New api - Result result = doSearch(new MockBackend(), query, 0,10); - assertTrue(TilingTestCase.getRendered(result).contains( - "<field name=\"" + com.yahoo.search.result.Hit.RANKFEATURES_FIELD + "\">" + rankFeatureString + "</field>")); - } - - private static class MockBackend extends Searcher { - - @Override - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result=new Result(query); - Hit hit=new FastHit("test",1000); - hit.setField(com.yahoo.search.result.Hit.RANKFEATURES_FIELD,rankFeatureString); - result.hits().add(hit); - return result; - } - - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher).search(query); - } - - private Execution createExecution(Searcher searcher) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher), context); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java deleted file mode 100644 index 6babba5a36a..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import static org.junit.Assert.*; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Random; -import java.util.zip.GZIPOutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.message.BasicHeader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.text.Utf8; - -/** - * Test GZip support for the HTTP integration introduced in 4.2. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class GzipDecompressingEntityTestCase { - private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000"; - private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT); - GzipDecompressingEntity testEntity; - - private static final class MockEntity implements HttpEntity { - - private final InputStream inStream; - - MockEntity(InputStream is) { - inStream = is; - } - - @Override - public boolean isRepeatable() { - return false; - } - - @Override - public boolean isChunked() { - return false; - } - - @Override - public long getContentLength() { - return -1; - } - - @Override - public Header getContentType() { - return new BasicHeader("Content-Type", "text/plain"); - } - - @Override - public Header getContentEncoding() { - return new BasicHeader("Content-Encoding", "gzip"); - } - - @Override - public InputStream getContent() throws IOException, - IllegalStateException { - return inStream; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - } - - @Override - public boolean isStreaming() { - return false; - } - - @Override - public void consumeContent() throws IOException { - } - } - - @Before - public void setUp() throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(CONTENT_AS_BYTES); - gzip.finish(); - gzip.close(); - byte[] compressed = out.toByteArray(); - InputStream inStream = new ByteArrayInputStream(compressed); - testEntity = new GzipDecompressingEntity(new MockEntity(inStream)); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testGetContentLength() throws UnknownHostException { - assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength()); - } - - @Test - public final void testGetContent() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length]; - int read = in.read(buffer); - assertEquals(CONTENT_AS_BYTES.length, read); - assertArrayEquals(CONTENT_AS_BYTES, buffer); - } - - @Test - public final void testGetContentToBigArray() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - in.read(buffer); - byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); - assertArrayEquals(expected, buffer); - } - - @Test - public final void testGetContentAvailable() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - assertEquals(CONTENT_AS_BYTES.length, in.available()); - } - - @Test - public final void testLargeZip() throws IOException { - byte [] input = new byte [10000000]; - Random random = new Random(89); - for (int i = 0; i < input.length; i++) { - input[i] = (byte) random.nextInt(); - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(input); - gzip.finish(); - gzip.close(); - byte[] compressed = out.toByteArray(); - assertEquals(10003073, compressed.length); - InputStream inStream = new ByteArrayInputStream(compressed); - GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream)); - assertEquals(input.length, gunzipper.getContentLength()); - byte[] buffer = new byte[input.length]; - InputStream content = gunzipper.getContent(); - assertEquals(input.length, content.available()); - int read = content.read(buffer); - assertEquals(input.length, read); - assertArrayEquals(input, buffer); - } - - @Test - public final void testGetContentReadByte() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - int i = 0; - while (i < buffer.length) { - int r = in.read(); - if (r == -1) { - break; - } else { - buffer[i++] = (byte) r; - } - } - byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); - assertEquals(CONTENT_AS_BYTES.length, i); - assertArrayEquals(expected, buffer); - } - - @Test - public final void testGetContentReadWithOffset() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length); - assertEquals(CONTENT_AS_BYTES.length, read); - byte[] expected = new byte[CONTENT_AS_BYTES.length * 2]; - for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) { - expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i]; - } - assertArrayEquals(expected, buffer); - read = in.read(buffer, 0, CONTENT_AS_BYTES.length); - assertEquals(-1, read); - } - - @Test - public final void testGetContentSkip() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - final long n = 5L; - long skipped = in.skip(n); - assertEquals(n, skipped); - int read = in.read(); - assertEquals(CONTENT_AS_BYTES[(int) n], read); - skipped = in.skip(5000); - assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped); - assertEquals(-1L, in.skip(1L)); - } - - - @Test - public final void testWriteToOutputStream() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - testEntity.writeTo(out); - assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java deleted file mode 100644 index 11f5d7d14fd..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.search.federation.ProviderConfig; -import org.junit.Test; - -import static com.yahoo.search.federation.ProviderConfig.Yca; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - * @author Steinar Knutsen - */ -public class HttpParametersTest { - - @Test - public void create_from_config() throws Exception { - ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder() - .connectionTimeout(1.0) - .maxConnectionPerRoute(2) - .maxConnections(3) - .path("myPath") - .readTimeout(4) - .socketBufferBytes(5) - .yca(new Yca.Builder() - .applicationId("myId") - .host("myYcaHost") - .port(7) - .retry(8) - .ttl(9) - .useProxy(true))); - - HTTPParameters httpParameters = new HTTPParameters(config); - - // Written to configuredConnectionTimeout, but it is not accessible!? - //assertThat(httpParameters.getConnectionTimeout(), is(1000)); - - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2)); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getMaxTotalConnections(), is(3)); - - assertThat(httpParameters.getPath(), is("/myPath")); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getReadTimeout(), is(4)); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5)); - - - assertThat(httpParameters.getYcaUseProxy(), is(true)); - assertThat(httpParameters.getYcaApplicationId(), is("myId")); - assertThat(httpParameters.getYcaProxy(), is("myYcaHost")); - assertThat(httpParameters.getYcaPort(), is(7)); - assertThat(httpParameters.getYcaRetry(), is(8000L)); - assertThat(httpParameters.getYcaTtl(), is(9000L)); - } - - @Test - public void requireFreezeWorksForAccessors() { - HTTPParameters p = new HTTPParameters(); - boolean caught = false; - final int expected = 37; - p.setConnectionTimeout(expected); - assertEquals(expected, p.getConnectionTimeout()); - p.freeze(); - try { - p.setConnectionTimeout(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setReadTimeout(expected); - assertEquals(expected, p.getReadTimeout()); - p.freeze(); - try { - p.setReadTimeout(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setPersistentConnections(true); - assertTrue(p.getPersistentConnections()); - p.freeze(); - try { - p.setPersistentConnections(false); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - assertEquals("http", p.getProxyType()); - - p = new HTTPParameters(); - caught = false; - p.setEnableProxy(true); - assertTrue(p.getEnableProxy()); - p.freeze(); - try { - p.setEnableProxy(false); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setProxyHost("nalle"); - assertEquals("nalle", p.getProxyHost()); - p.freeze(); - try { - p.setProxyHost("jappe"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setProxyPort(expected); - assertEquals(expected, p.getProxyPort()); - p.freeze(); - try { - p.setProxyPort(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMethod("POST"); - assertEquals("POST", p.getMethod()); - p.freeze(); - try { - p.setMethod("GET"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setSchema("gopher"); - assertEquals("gopher", p.getSchema()); - p.freeze(); - try { - p.setSchema("http"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setInputEncoding("iso-8859-15"); - assertEquals("iso-8859-15", p.getInputEncoding()); - p.freeze(); - try { - p.setInputEncoding("shift-jis"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setOutputEncoding("iso-8859-15"); - assertEquals("iso-8859-15", p.getOutputEncoding()); - p.freeze(); - try { - p.setOutputEncoding("shift-jis"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMaxTotalConnections(expected); - assertEquals(expected, p.getMaxTotalConnections()); - p.freeze(); - try { - p.setMaxTotalConnections(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMaxConnectionsPerRoute(expected); - assertEquals(expected, p.getMaxConnectionsPerRoute()); - p.freeze(); - try { - p.setMaxConnectionsPerRoute(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setSocketBufferSizeBytes(expected); - assertEquals(expected, p.getSocketBufferSizeBytes()); - p.freeze(); - try { - p.setSocketBufferSizeBytes(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setRetries(expected); - assertEquals(expected, p.getRetries()); - p.freeze(); - try { - p.setRetries(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java deleted file mode 100644 index e0b2afe4e1f..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.federation.http.HTTPProviderSearcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.entity.StringEntity; -import org.junit.Test; - -import java.io.InputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.core.StringContains.containsString; -import static org.junit.Assert.assertThat; - -/** - * See bug #3234696. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public class HttpPostTestCase { - - @Test - public void testPostingSearcher() throws Exception { - StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer(); - server.start(); - - TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"), - Arrays.asList(new Connection("localhost", server.getServerPort())), - "/"); - Query q = new Query(""); - q.setTimeout(10000000L); - Execution e = new Execution(searcher, Execution.Context.createContextStub()); - - searcher.search(q, e); - - assertThat(server.getRequest(), containsString("My POST body")); - server.stop(); - } - - private static class TestPostSearcher extends HTTPProviderSearcher { - public TestPostSearcher(ComponentId id, List<Connection> connections, String path) { - super(id, connections, httpParameters(path), Statistics.nullImplementation); - } - - private static HTTPParameters httpParameters(String path) { - HTTPParameters httpParameters = new HTTPParameters(path); - httpParameters.setPingOption(PingOption.Enum.DISABLE); - return httpParameters; - } - - @Override - protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) { - HttpPost request = new HttpPost(uri); - request.setEntity(entity); - return request; - } - - @Override - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - try { - return new StringEntity("My POST body"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return new HashMap<>(0); - } - - @Override - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - // do nothing with the result - } - - @Override - protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { - //Empty - } - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java deleted file mode 100644 index b4f3c13b8e2..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import javax.xml.bind.JAXBException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Rudimentary http searcher test. - * - * @author bratseth - */ -public class HttpTestCase { - - private StupidSingleThreadedHttpServer httpServer; - private TestHTTPClientSearcher searcher; - - @Test - public void testSearcher() throws JAXBException { - Result result = searchUsingLocalhost(); - - assertEquals("ok", result.getQuery().properties().get("gotResponse")); - assertEquals(0, result.getQuery().errors().size()); - } - - private Result searchUsingLocalhost() { - searcher = new TestHTTPClientSearcher("test","localhost",getPort()); - Query query = new Query("/?query=test"); - - query.setWindow(0,10); - return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub())); - } - - @Test - public void test_that_ip_address_set_on_meta_hit() { - Result result = searchUsingLocalhost(); - Hit metaHit = getFirstMetaHit(result.hits()); - String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS); - - assertEquals(ip, "127.0.0.1"); - } - - private Hit getFirstMetaHit(HitGroup hits) { - for (Iterator<Hit> i = hits.unorderedDeepIterator(); i.hasNext();) { - Hit hit = i.next(); - if (hit.isMeta()) - return hit; - } - return null; - } - - @Before - public void setUp() throws Exception { - httpServer = new StupidSingleThreadedHttpServer(0, 0) { - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 5\r\n" + - "\r\n" + - "hello"); - } - }; - httpServer.start(); - } - - private int getPort() { - return httpServer.getServerPort(); - } - - @After - public void tearDown() throws Exception { - httpServer.stop(); - if (searcher != null) { - searcher.shutdownConnectionManagers(); - } - } - - private static class TestHTTPClientSearcher extends HTTPClientSearcher { - - public TestHTTPClientSearcher(String id, String hostName, int port) { - super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); - } - - private static List<Connection> toConnections(String hostName,int port) { - List<Connection> connections=new ArrayList<>(); - connections.add(new Connection(hostName,port)); - return connections; - } - - @Override - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - query.properties().set("gotResponse","ok"); - return query; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return null; - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java deleted file mode 100644 index 8c1ff69666b..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; -import com.yahoo.yolean.Exceptions; -import org.apache.http.HttpEntity; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Check for different keep-alive scenarios. What we really want to test - * is the server does not hang. - * - * @author Steinar Knutsen - */ -public class PingTestCase { - - private static final int TIMEOUT_MS = 60000; - - @Test - public void testNiceCase() throws Exception { - NiceStupidServer server = new NiceStupidServer(); - server.start(); - checkSearchAndPing(true, true, true, server.getServerPort()); - server.stop(); - } - - private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) { - String resultThing; - String comment; - TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test", - "localhost", port); - try { - - Query query = new Query("/?query=test"); - - query.setWindow(0, 10); - // high timeout to allow for overloaded test machine - query.setTimeout(TIMEOUT_MS); - Ping ping = new Ping(TIMEOUT_MS); - - long start = System.currentTimeMillis(); - Execution exe = new Execution(searcher, Execution.Context.createContextStub()); - exe.search(query); - - resultThing = firstSearch ? "ok" : null; - comment = firstSearch ? "First search should have succeeded." : "First search should fail."; - assertEquals(comment, resultThing, query.properties().get("gotResponse")); - Pong pong = searcher.ping(ping, searcher.getConnection()); - if (pongCheck) { - assertEquals("Ping should not have failed.", 0, pong.getErrorSize()); - } else { - assertEquals("Ping should have failed.", 1, pong.getErrorSize()); - } - exe = new Execution(searcher, Execution.Context.createContextStub()); - exe.search(query); - - resultThing = secondSearch ? "ok" : null; - comment = secondSearch ? "Second search should have succeeded." : "Second search should fail."; - - assertEquals(resultThing, query.properties().get("gotResponse")); - long duration = System.currentTimeMillis() - start; - // target for duration based on the timeout values + some slack - assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000); - searcher.shutdownConnectionManagers(); - } finally { - searcher.deconstruct(); - } - } - - @Test - public void testUselessCase() throws Exception { - UselessStupidServer server = new UselessStupidServer(); - server.start(); - checkSearchAndPing(false, true, false, server.getServerPort()); - server.stop(); - } - - @Test - public void testGrumpyCase() throws Exception { - GrumpyStupidServer server = new GrumpyStupidServer(); - server.start(); - checkSearchAndPing(false, false, false, server.getServerPort()); - server.stop(); - } - - @Test - public void testPassiveAggressiveCase() throws Exception { - PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer(); - server.start(); - checkSearchAndPing(true, false, true, server.getServerPort()); - server.stop(); - } - - // OK on ping and search - private static class NiceStupidServer extends StupidSingleThreadedHttpServer { - private NiceStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } - } - - // rejects ping and accepts search - private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer { - - private PassiveAggressiveStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - if (request.contains("/ping")) { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } else { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } - } - } - - // accepts ping and rejects search - private static class UselessStupidServer extends StupidSingleThreadedHttpServer { - private UselessStupidServer() throws IOException { - super(0, 0); - } - - - @Override - protected byte[] getResponse(String request) { - if (request.contains("/ping")) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } else { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } - } - } - - // rejects ping and search - private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer { - private GrumpyStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } - } - - private static class TestHTTPClientSearcher extends HTTPClientSearcher { - - public TestHTTPClientSearcher(String id, String hostName, int port) { - super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); - } - - private static List<Connection> toConnections(String hostName,int port) { - List<Connection> connections=new ArrayList<>(); - connections.add(new Connection(hostName,port)); - return connections; - } - - @Override - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - query.properties().set("gotResponse","ok"); - return query; - } - - @Override - public Result search(Query query, Execution execution, - Connection connection) { - URI uri; - try { - uri = new URL("http", connection.getHost(), connection - .getPort(), "/search").toURI(); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query, e)); - return execution.search(query); - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query, e)); - return execution.search(query); - } - - HttpEntity entity; - try { - entity = getEntity(uri, query); - } catch (IOException e) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in " - + this + " using " + connection - + " for " + query + ": " - + Exceptions.toMessageString(e))); - return execution.search(query); - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return execution.search(query); - } - if (entity == null) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in " - + this + " using " + connection + " for " + query)); - return execution.search(query); - } - - try { - query = handleResponse(entity, query); - } catch (IOException e) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("Error when trying to consume input in " - + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return execution.search(query); - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return null; - } - - @Override - protected URI getPingURI(Connection connection) - throws MalformedURLException, URISyntaxException { - return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI(); - } - - Connection getConnection() { - return getHasher().getNodes().select(0, 0); - } - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java deleted file mode 100644 index 19750cf84cc..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.vespa.defaults.Defaults; -import org.junit.Test; - -import java.util.Collections; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Tests that source and backend specific parameters from the query are added correctly to the backend requests - * - * @author bratseth - */ -public class QueryParametersTestCase { - - @Test - public void testQueryParameters() { - Query query=new Query(); - query.properties().set("a","a-value"); - query.properties().set("b.c","b.c-value"); - query.properties().set("source.otherSource.d","d-value"); - query.properties().set("source.testSource.e","e-value"); - query.properties().set("source.testSource.f.g","f.g-value"); - query.properties().set("provider.testProvider.h","h-value"); - query.properties().set("provider.testProvider.i.j","i.j-value"); - - query.properties().set("sourceName","testSource"); // Done by federation searcher - query.properties().set("providerName","testProvider"); // Done by federation searcher - - TestHttpProvider searcher=new TestHttpProvider(); - Map<String,String> parameters=searcher.getQueryMap(query); - searcher.deconstruct(); - - assertEquals(4,parameters.size()); // the appropriate 4 of the above - assertEquals(parameters.get("e"),"e-value"); - assertEquals(parameters.get("f.g"),"f.g-value"); - assertEquals(parameters.get("h"),"h-value"); - assertEquals(parameters.get("i.j"),"i.j-value"); - } - - public static class TestHttpProvider extends HTTPProviderSearcher { - - public TestHttpProvider() { - super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation); - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return Collections.singletonMap("nocaching", String.valueOf(Math.random())); - } - - @Override - protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { - } - - } - -} - diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java deleted file mode 100644 index 022177bc42b..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.QueryMarshaller; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.test.QueryTestCase; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class QueryMarshallerTestCase { - - private static final Linguistics linguistics = new SimpleLinguistics(); - - @Test - public void testCommonCommonCase() { - AndItem root = new AndItem(); - addThreeWords(root); - assertEquals("a AND b AND c", new QueryMarshaller().marshal(root)); - } - - @Test - public void testPhrase() { - PhraseItem root = new PhraseItem(); - root.setIndexName("habla"); - addThreeWords(root); - assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root)); - } - - @Test - public void testPhraseDefaultIndex() { - PhraseItem root = new PhraseItem(); - addThreeWords(root); - assertEquals("\"a b c\"", new QueryMarshaller().marshal(root)); - } - - @Test - public void testLittleMoreComplex() { - AndItem root = new AndItem(); - addThreeWords(root); - OrItem ambig = new OrItem(); - root.addItem(ambig); - addThreeWords(ambig); - AndItem but = new AndItem(); - addThreeWords(but); - ambig.addItem(but); - assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )", - new QueryMarshaller().marshal(root)); - } - - @Test - public void testRank() { - RankItem root = new RankItem(); - addThreeWords(root); - assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root)); - } - - @Test - public void testNear() { - NearItem near = new NearItem(3); - addThreeWords(near); - assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near)); - } - - @Test - public void testONear() { - ONearItem oNear = new ONearItem(3); - addThreeWords(oNear); - assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear)); - } - - private void addThreeWords(CompositeItem root) { - root.addItem(new WordItem("a")); - root.addItem(new WordItem("b")); - root.addItem(new WordItem("c")); - } - - @Test - public void testNegativeGroupedTerms() { - testQueryString(new QueryMarshaller(), "a -(b c) -(d e)", - "a ANDNOT ( b AND c ) ANDNOT ( d AND e )"); - } - - @Test - public void testPositiveGroupedTerms() { - testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )"); - } - - @Test - public void testInt() { - testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123"); - } - - @Test - public void testCJKOneWord() { - testQueryString(new QueryMarshaller(), "天龍人"); - } - - @Test - public void testTwoWords() { - testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics()); - } - - @Test - public void testTwoWordsInPhrase() { - testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics()); - } - - @Test - public void testCJKTwoSentences() { - testQueryString(new QueryMarshaller(), "是不是這樣的夜晚 你才會這樣地想起我", "是不是這樣的夜晚 AND 你才會這樣地想起我"); - } - - @Test - public void testCJKTwoSentencesWithLanguage() { - testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2萬", "助妳好孕 AND 生1胎北市發2萬", "zh-Hant"); - } - - @Test - public void testCJKTwoSentencesInPhrase() { - QueryMarshaller marshaller = new QueryMarshaller(); - testQueryString(marshaller, "\"助妳好孕 生1胎北市發2萬\"", "\"助妳好孕 生1胎北市發2萬\"", "zh-Hant"); - testQueryString(marshaller, "\"是不是這樣的夜晚 你才會這樣地想起我\"", "\"是不是這樣的夜晚 你才會這樣地想起我\""); - } - - @Test - public void testCJKMultipleSentences() { - testQueryString(new QueryMarshaller(), "염부장님과 함께했던 좋은 추억들은", "염부장님과 AND 함께했던 AND 좋은 AND 추억들은"); - } - - @Test - public void testIndexRestriction() { - /** ticket 3707606, comment #29 */ - testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\""); - } - - private void testQueryString(QueryMarshaller marshaller, String uq) { - testQueryString(marshaller, uq, uq, null); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq) { - testQueryString(marshaller, uq, mq, null); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) { - testQueryString(marshaller, uq, mq, lang, linguistics); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) { - Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : "")); - query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics))); - assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot())); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java deleted file mode 100644 index 8d1ecf4085e..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.VespaSearcher; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Tests that source and backend specific parameters from the query are added correctly to the backend requests - * - * @author bratseth - */ -public class QueryParametersTestCase { - - public void testQueryParameters() { - Query query=new Query(); - query.properties().set("a","a-value"); - query.properties().set("b.c","b.c-value"); - query.properties().set("source.otherSource.d","d-value"); - query.properties().set("source.testSource.e","e-value"); - query.properties().set("source.testSource.f.g","f.g-value"); - query.properties().set("provider.testProvider.h","h-value"); - query.properties().set("provider.testProvider.i.j","i.j-value"); - - query.properties().set("sourceName","testSource"); // Done by federation searcher - query.properties().set("providerName","testProvider"); // Done by federation searcher - - VespaSearcher searcher=new VespaSearcher("testProvider","",0,""); - Map<String,String> parameters=searcher.getQueryMap(query); - searcher.deconstruct(); - - assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above - assertEquals(parameters.get("e"),"e-value"); - assertEquals(parameters.get("f.g"),"f.g-value"); - assertEquals(parameters.get("h"),"h-value"); - assertEquals(parameters.get("i.j"),"i.j-value"); - } - -} - diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java deleted file mode 100644 index 50186e03ae8..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import java.util.Iterator; - -import com.yahoo.net.URI; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.vespa.ResultBuilder; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.HitGroup; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * Test XML parsing of results. - * - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -public class ResultBuilderTestCase { - - private boolean quickCompare(double a, double b) { - double z = Math.min(Math.abs(a), Math.abs(b)); - if (Math.abs((a - b)) < (z / 1e14)) { - return true; - } else { - return false; - } - } - - @Test - public void testSimpleResult() { - boolean gotErrorDetails = false; - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a")); - assertEquals(3, res.getConcreteHitCount()); - assertEquals(4, res.getHitCount()); - ErrorHit e = (ErrorHit) res.hits().get(0); - // known problem, if the same error is the main error is - // in details, it'll be added twice. Not sure how to fix that, - // because old Vespa systems give no error details, and there - // is no way of nuking an existing error if the details exist. - for (Iterator<?> i = e.errorIterator(); i.hasNext();) { - ErrorMessage err = (ErrorMessage) i.next(); - assertEquals(5, err.getCode()); - String details = err.getDetailedMessage(); - if (details != null) { - gotErrorDetails = true; - assertEquals("An error as ordered", details.trim()); - } - } - assertTrue("Error details are missing", gotErrorDetails); - assertEquals(new URI("http://def"), res.hits().get(1).getId()); - assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category")); - assertEquals("<field>habla</field>" - + "<hi>blbl</hi><br /><>&fdlkkgj</field>;lk<a b=\"1\" c=\"2\" />" - + "<x><y><z /></y></x>", res.hits().get(3).getField("annoying").toString()); - } - - @Test - public void testNestedResult() { - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a")); - assertNull(res.hits().getError()); - assertEquals(3, res.hits().size()); - assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString()); - HitGroup g1 = (HitGroup) res.hits().get(1); - HitGroup g2 = (HitGroup) res.hits().get(2); - assertEquals(15, g1.size()); - assertEquals("reward_for_thumb", g1.get(1).getField("id").toString()); - assertEquals(10, g2.size()); - HitGroup g3 = (HitGroup) g2.get(3); - assertEquals("badge", g3.types().iterator().next()); - assertEquals(2, g3.size()); - assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString()); - } - - @Test - public void testWeirdDocumentID() { - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a")); - assertNull(res.hits().getError()); - assertEquals(3, res.hits().size()); - assertEquals(new URI("nalle"), res.hits().get(0).getId()); - assertEquals(new URI("tralle"), res.hits().get(1).getId()); - assertEquals(new URI("kalle"), res.hits().get(2).getId()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java deleted file mode 100644 index 7fbe883bfb7..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.federation.vespa.VespaSearcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class VespaIntegrationTestCase { - - // TODO: Setup the answering vespa searcher from this test.... - @Test - public void testIt() { - if (System.currentTimeMillis() > 0) return; - Chain<Searcher> chain = new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,"")); - Result result = new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test")); - assertEquals(23, result.hits().size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java deleted file mode 100644 index cc00c6739fa..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.VespaSearcher; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.Parsable; -import com.yahoo.search.query.parser.Parser; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import org.apache.http.HttpEntity; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; - -import static org.junit.Assert.assertEquals; - -/** - * Check query marshaling in VespaSearcher works. - * - * @author Steinar Knutsen - */ -public class VespaSearcherTestCase { - - private VespaSearcher searcher; - - @Before - public void setUp() { - searcher = new VespaSearcher("cache1","",0,""); - } - - @After - public void tearDown() { - searcher.deconstruct(); - } - - @Test - public void testMarshalQuery() { - RankItem root = new RankItem(); - QueryTree r = new QueryTree(root); - AndItem recall = new AndItem(); - PhraseItem usual = new PhraseItem(); - PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); - WordItem filterWord = new WordItem("silly"); - - filterPhrase.setFilter(true); - filterWord.setFilter(true); - - root.addItem(recall); - usual.addItem(new WordItem("new")); - usual.addItem(new WordItem("york")); - recall.addItem(usual); - recall.addItem(new WordItem("shoes")); - root.addItem(new WordItem("nike")); - root.addItem(new WordItem("adidas")); - root.addItem(filterPhrase); - recall.addItem(filterWord); - - assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r)); - } - - @Test - public void testMarshalQuerySmallTree() { - RankItem root = new RankItem(); - QueryTree r = new QueryTree(root); - AndItem recall = new AndItem(); - PhraseItem usual = new PhraseItem(); - PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); - WordItem filterWord = new WordItem("silly"); - - filterPhrase.setFilter(true); - filterWord.setFilter(true); - - root.addItem(recall); - usual.addItem(new WordItem("new")); - usual.addItem(new WordItem("york")); - recall.addItem(usual); - recall.addItem(new WordItem("shoes")); - root.addItem(filterPhrase); - recall.addItem(filterWord); - - assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r)); - // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly - // assertMarshals(root) - } - - @Test - public void testWandMarshalling() { - WeakAndItem root = new WeakAndItem(); - root.setN(32); - root.addItem(new WordItem("a")); - root.addItem(new WordItem("b")); - root.addItem(new WordItem("c")); - assertMarshals(root); - } - - @Test - public void testWandMarshalling2() { - // AND (WAND(10) a!1 the!10) source:yahoonews - AndItem root = new AndItem(); - WeakAndItem wand = new WeakAndItem(10); - wand.addItem(newWeightedWordItem("a",1)); - wand.addItem(newWeightedWordItem("the",10)); - root.addItem(wand); - root.addItem(new WordItem("yahoonews","source")); - assertMarshals(root); - } - - private WordItem newWeightedWordItem(String word,int weight) { - WordItem wordItem=new WordItem(word); - wordItem.setWeight(weight); - return wordItem; - } - - private void assertMarshals(Item root) { - QueryTree r = new QueryTree(root); - String marshalledQuery=searcher.marshalQuery(r); - assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original", - r,parseQuery(marshalledQuery,"")); - } - - private static Item parseQuery(String query, String filter) { - Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment()); - return parser.parse(new Parsable().setQuery(query).setFilter(filter)); - } - - @Test - public void testSourceProviderProperties() throws Exception { - /* TODO: update test - Server httpServer = new Server(); - try { - SocketConnector listener = new SocketConnector(); - listener.setHost("0.0.0.0"); - httpServer.addConnector(listener); - httpServer.setHandler(new DummyHandler()); - httpServer.start(); - - int port=httpServer.getConnectors()[0].getLocalPort(); - - List<SourcesConfig.Source> sourcesConfig = new ArrayList<SourcesConfig.Source>(); - SourcesConfig.Source sourceConfig = new SourcesConfig.Source(); - sourceConfig.chain.setValue("news"); - sourceConfig.provider.setValue("news"); - sourceConfig.id.setValue("news"); - sourceConfig.timelimit.value = 10000; - sourcesConfig.add(sourceConfig); - FederationSearcher federator = - new FederationSearcher(ComponentId.createAnonymousComponentId(), - new ArrayList<SourcesConfig.Source>(sourcesConfig)); - SearchChain mainChain=new OrderedSearchChain(federator); - - SearchChainRegistry registry=new SearchChainRegistry(); - SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,"")); - registry.register(sourceChain); - Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1"); - Result result=new Execution(mainChain,registry).search(query); - assertNull(result.hits().getError()); - Hit testHit=result.hits().get("testHit"); - assertNotNull(testHit); - assertEquals("testValue",testHit.fields().get("testField")); - assertEquals("a1",testHit.fields().get("a")); - assertEquals("b1",testHit.fields().get("b")); - } - finally { - httpServer.stop(); - } - */ - } - - @Test - public void testVespaSearcher() { - VespaSearcher v=new VespaSearcherValidatingSubclass(); - new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter"))); - } - - private class VespaSearcherValidatingSubclass extends VespaSearcher { - - public VespaSearcherValidatingSubclass() { - super("configId","host",80,"path"); - } - - @Override - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString()); - return super.getEntity(uri,requestMeta,query); - } - - } - - // used by the old testSourceProviderProperties() -// private class DummyHandler extends AbstractHandler { -// public void handle(String s, Request request, HttpServletRequest httpServletRequest, -// HttpServletResponse httpServletResponse) throws IOException, ServletException { -// -// try { -// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse(); -// -// httpResponse.setStatus(HttpStatus.OK_200); -// httpResponse.setContentType("text/xml"); -// httpResponse.setCharacterEncoding("UTF-8"); -// Result r=new Result(new Query()); -// Hit testHit=new Hit("testHit"); -// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate... -// testHit.setField("testField","testValue"); -// // Write back all incoming properties: -// for (Object e : httpServletRequest.getParameterMap().entrySet()) { -// Map.Entry entry=(Map.Entry)e; -// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue())); -// } -// -// r.hits().add(testHit); -// -// //StringWriter sw=new StringWriter(); -// //r.render(sw); -// //System.out.println(sw.toString()); -// -// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r); -// httpResponse.complete(); -// } -// catch (Exception e) { -// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e)); -// e.printStackTrace(); -// } -// } -// -// private String getFirstValue(Object entry) { -// if (entry instanceof String[]) -// return ((String[])entry)[0].toString(); -// else -// return entry.toString(); -// } -// } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml deleted file mode 100644 index c17bbf8474d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result total-hit-count="3"> - <hit relevancy="75" source="test" type="summary"> - <field name="uri">nalle</field> - <field name="relevancy">75</field> - <field name="collapseId">0</field> - </hit> - <hit relevancy="73" source="test" type="summary test other"> - <field name="documentId">tralle</field> - <field name="relevancy">73</field> - <field name="collapseId">0</field> - <field name="category">test/stuff\tsome/other</field> - <field name="bsumtitle">dklf øæå sdf > & < -Ipsum, etc.</field> - </hit> - <hit relevancy="70" source="test" type="summary"> - <field name="DOCUMENTID">kalle</field> - <field name="relevancy">75</field> - <field name="collapseId">0</field> - <field name="annoying"><field>habla</field><hi>blbl</hi><br /><![CDATA[<>&fdlkkgj</field>]]>;lk<a b="1" c="2" /><x><y><z /></y></x></field> - </hit> -</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml deleted file mode 100644 index 8b5ab710378..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml +++ /dev/null @@ -1,318 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result total-hit-count="36"> -<hit type="user_reputation"> -<field name="guid">ABCDEFGHIJKLMNOPQRSTUVWXYZ</field> -<field name="level">zero</field> -<field name="points">0</field> -<field name="created">1287600988</field> -<field name="updated">1287600988</field> -</hit> -<group type="actions"> -<hit type="action"> -<field name="id">thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> -<hit type="action"> -<field name="id">reward_for_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> -<hit type="action"> -<field name="id">undo_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">buzz</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">undo_reward_for_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">vote</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">report_abuse</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">reward_for_vote</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">signup</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">registered</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">get_points</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">contrib_SignedUp</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">contrib_AgreedToTos</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">500</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">Create Feature</field> -<field name="created"/> -<field name="updated"/> -<field name="points">0</field> -<field name="level"/> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">add_theme</field> -<field name="created"/> -<field name="updated"/> -<field name="points">0</field> -<field name="level"/> -<field name="isEnabled">1</field> -</hit> -</group> - -<group type="awards"> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/First Feature</field> -<field name="description">You’ve created your First Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/1stfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/25th Feature</field> -<field name="description">You’ve created your 25th Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/25thfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/50th Feature</field> -<field name="description">You’ve created your 50th Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/10thfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 5</field> -<field name="description">You’ve added a Feature to your 5th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/5thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 15</field> -<field name="description">You’ve added a Feature to your 15th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/15thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 30</field> -<field name="description">You’ve added a Feature to your 30th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/30thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Pollster</field> -<field name="description">You’ve created your 5th Poll Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/pollster.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Reporter</field> -<field name="description">You’ve created your 5th Article Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/newsreporter.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Paparazzi</field> -<field name="description">You’ve created your 5th Image Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/paparazzi.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Video Reporter</field> -<field name="description">You’ve created your 5th Video Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/director.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -</group> -</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java index 60ada5124a0..326e37ede38 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -203,7 +203,6 @@ public class GroupingParserBenchmarkTest { "all(group(artist) max(2) each(each(output(summary()))))", "all(group(artist) max(2) each(each(output(summary(simple)))))", "all(group(artist) max(5) each(output(count()) each(output(summary()))))", - "all(group(ymum()))", "all(group(strlen(attr)))", "all(group(normalizesubject(attr)))", "all(group(strcat(attr, attr2)))", diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index fcf1c3bcdd0..5d311361edf 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -150,7 +150,6 @@ public class GroupingParserTestCase { "xor", "xorbit", "y", - "ymum", "zcurve"); for (String image : images) { assertParse("all(group(" + image + "))", "all(group(" + image + "))"); @@ -512,7 +511,6 @@ public class GroupingParserTestCase { assertParse("all(group(artist) max(2) each(each(output(summary()))))"); assertParse("all(group(artist) max(2) each(each(output(summary(simple)))))"); assertParse("all(group(artist) max(5) each(output(count()) each(output(summary()))))"); - assertParse("all(group(ymum()))"); assertParse("all(group(strlen(attr)))"); assertParse("all(group(normalizesubject(attr)))"); assertParse("all(group(strcat(attr, attr2)))"); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java index c64c4d624f2..f61299bd317 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java @@ -160,7 +160,6 @@ public class RequestBuilderTestCase { assertLayout("all(group(time.year(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); assertLayout("all(group(xor(a,b)) each(output(count())))", "[[{ Xor, result = [Count] }]]"); assertLayout("all(group(xorbit(a,1)) each(output(count())))", "[[{ XorBit, result = [Count] }]]"); - assertLayout("all(group(ymum()) each(output(count())))", "[[{ GetYMUMChecksum, result = [Count] }]]"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index fa398efd293..9617e0ceb25 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -232,7 +232,7 @@ public class JSONSearchHandlerTestCase { public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { JSONObject json = new JSONObject(); json.put("query", "abc"); - json.put("format", "DefaultRenderer"); + json.put("format", "XmlRenderer"); assertXmlResult(json, driver); } @@ -244,22 +244,6 @@ public class JSONSearchHandlerTestCase { assertJsonResult(json, driver); } - @Test - public void testResultLegacyTiledFormat() throws Exception { - JSONObject json = new JSONObject(); - json.put("query", "abc"); - json.put("format", "tiled"); - assertTiledResult(json, driver); - } - - @Test - public void testResultLegacyPageFormat() throws Exception { - JSONObject json = new JSONObject(); - json.put("query", "abc"); - json.put("format", "page"); - assertPageResult(json, driver); - } - private static final String xmlResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<result total-hit-count=\"0\">\n" + @@ -284,21 +268,6 @@ public class JSONSearchHandlerTestCase { } - private static final String tiledResult = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<result version=\"1.0\">\n" + - "\n" + - " <hit relevance=\"1.0\">\n" + - " <id>testHit</id>\n" + - " <uri>testHit</uri>\n" + - " </hit>\n" + - "\n" + - "</result>\n"; - - private void assertTiledResult(JSONObject json, RequestHandlerTestDriver driver) { - assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), tiledResult); - } - private static final String pageResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<page version=\"1.0\">\n" + @@ -312,10 +281,6 @@ public class JSONSearchHandlerTestCase { "\n" + "</page>\n"; - private void assertPageResult(JSONObject json, RequestHandlerTestDriver driver) { - assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), pageResult); - } - private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { assertEquals(expected, response.readAll()); assertEquals(200, response.getStatus()); @@ -508,7 +473,6 @@ public class JSONSearchHandlerTestCase { assertEquals("Should have same mapping for properties", map, propertyMap); } - @Test public void testContentTypeParsing() throws Exception { JSONObject json = new JSONObject(); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java index 5ef13eba2ed..20b18ba6723 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -15,7 +15,7 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.handler.HttpSearchResponse; import com.yahoo.search.handler.SearchHandler; -import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; @@ -105,12 +105,12 @@ public class SearchHandlerTestCase { } @Test - public void testFailing() throws Exception { + public void testFailing() { assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=classLoadingError").readAll().contains("NoClassDefFoundError")); } @Test - public synchronized void testPluginError() throws Exception { + public synchronized void testPluginError() { assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=exceptionInPlugin").readAll().contains("NullPointerException")); } @@ -158,11 +158,11 @@ public class SearchHandlerTestCase { Result r = new Result(q); r.hits().addError(ErrorMessage.createUnspecifiedError("bamse")); r.hits().add(new Hit("http://localhost/dummy", 0.5)); - HttpSearchResponse s = new HttpSearchResponse(200, r, q, new DefaultRenderer()); + HttpSearchResponse s = new HttpSearchResponse(200, r, q, new XmlRenderer()); assertEquals("text/xml", s.getContentType()); assertNull(s.getCoverage()); assertEquals("query 'dummy'", s.getParsedQuery()); - assertEquals(5000, s.getTiming().getTimeout()); + assertEquals(500, s.getTiming().getTimeout()); } @Test @@ -191,7 +191,7 @@ public class SearchHandlerTestCase { } } @Test - public void testInvalidQueryParamWithoutQueryProfile() throws Exception { + public void testInvalidQueryParamWithoutQueryProfile() { testInvalidQueryParam(driver); } private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) { @@ -213,45 +213,35 @@ public class SearchHandlerTestCase { } @Test - public void testNormalResultImplicitDefaultRendering() throws Exception { + public void testNormalResultImplicitDefaultRendering() { assertJsonResult("http://localhost?query=abc", driver); } @Test - public void testNormalResultExplicitDefaultRendering() throws Exception { + public void testNormalResultExplicitDefaultRendering() { assertJsonResult("http://localhost?query=abc&format=default", driver); } @Test - public void testNormalResultXmlAliasRendering() throws Exception { + public void testNormalResultXmlAliasRendering() { assertXmlResult("http://localhost?query=abc&format=xml", driver); } @Test - public void testNormalResultJsonAliasRendering() throws Exception { + public void testNormalResultJsonAliasRendering() { assertJsonResult("http://localhost?query=abc&format=json", driver); } @Test - public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { - assertXmlResult("http://localhost?query=abc&format=DefaultRenderer", driver); + public void testNormalResultExplicitDefaultRenderingFullRendererName1() { + assertXmlResult("http://localhost?query=abc&format=XmlRenderer", driver); } @Test - public void testNormalResultExplicitDefaultRenderingFullRendererName2() throws Exception { + public void testNormalResultExplicitDefaultRenderingFullRendererName2() { assertJsonResult("http://localhost?query=abc&format=JsonRenderer", driver); } - @Test - public void testResultLegacyTiledFormat() throws Exception { - assertTiledResult("http://localhost?query=abc&format=tiled", driver); - } - - @Test - public void testResultLegacyPageFormat() throws Exception { - assertPageResult("http://localhost?query=abc&format=page", driver); - } - private static final String xmlResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<result total-hit-count=\"0\">\n" + @@ -260,10 +250,12 @@ public class SearchHandlerTestCase { " <field name=\"uri\">testHit</field>\n" + " </hit>\n" + "</result>\n"; - private void assertXmlResult(String request, RequestHandlerTestDriver driver) throws Exception { + + private void assertXmlResult(String request, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(request), xmlResult); } - private void assertXmlResult(RequestHandlerTestDriver driver) throws Exception { + + private void assertXmlResult(RequestHandlerTestDriver driver) { assertXmlResult("http://localhost?query=abc", driver); } @@ -272,38 +264,9 @@ public class SearchHandlerTestCase { + "\"children\":[" + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + "]}}"; - private void assertJsonResult(String request, RequestHandlerTestDriver driver) throws Exception { - assertOkResult(driver.sendRequest(request), jsonResult); - } - - private static final String tiledResult = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<result version=\"1.0\">\n" + - "\n" + - " <hit relevance=\"1.0\">\n" + - " <id>testHit</id>\n" + - " <uri>testHit</uri>\n" + - " </hit>\n" + - "\n" + - "</result>\n"; - private void assertTiledResult(String request, RequestHandlerTestDriver driver) throws Exception { - assertOkResult(driver.sendRequest(request), tiledResult); - } - private static final String pageResult = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<page version=\"1.0\">\n" + - "\n" + - " <content>\n" + - " <hit relevance=\"1.0\">\n" + - " <id>testHit</id>\n" + - " <uri>testHit</uri>\n" + - " </hit>\n" + - " </content>\n" + - "\n" + - "</page>\n"; - private void assertPageResult(String request, RequestHandlerTestDriver driver) throws Exception { - assertOkResult(driver.sendRequest(request), pageResult); + private void assertJsonResult(String request, RequestHandlerTestDriver driver) { + assertOkResult(driver.sendRequest(request), jsonResult); } private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java index 999a6d32ac6..569f219f635 100644 --- a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java @@ -3,9 +3,6 @@ package com.yahoo.search.pagetemplates.engine.test; import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.io.IOUtils; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.prelude.templates.test.TilingTestCase; import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.PageTemplate; import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; diff --git a/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java index 83ef955a6d9..5026250ab91 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java @@ -13,14 +13,14 @@ public class SoftTimeoutTestCase { @Test public void testDefaultsInQuery() { Query query=new Query("?query=test"); - assertNull(query.getRanking().getSoftTimeout().getEnable()); + assertTrue(query.getRanking().getSoftTimeout().getEnable()); assertNull(query.getRanking().getSoftTimeout().getFactor()); assertNull(query.getRanking().getSoftTimeout().getTailcost()); } @Test public void testQueryOverride() { - Query query=new Query("?query=test&ranking.softtimeout.enable&ranking.softtimeout.factor=0.7&ranking.softtimeout.tailcost=0.3"); + Query query=new Query("?query=test&ranking.softtimeout.factor=0.7&ranking.softtimeout.tailcost=0.3"); assertTrue(query.getRanking().getSoftTimeout().getEnable()); assertEquals(Double.valueOf(0.7), query.getRanking().getSoftTimeout().getFactor()); assertEquals(Double.valueOf(0.3), query.getRanking().getSoftTimeout().getTailcost()); @@ -30,6 +30,14 @@ public class SoftTimeoutTestCase { assertEquals("0.3", query.getRanking().getProperties().get("vespa.softtimeout.tailcost").get(0)); } + @Test + public void testDisable() { + Query query=new Query("?query=test&ranking.softtimeout.enable=false"); + assertFalse(query.getRanking().getSoftTimeout().getEnable()); + query.prepare(); + assertTrue(query.getRanking().getProperties().isEmpty()); + } + private void verifyException(String key, String value) { try { new Query("?query=test&ranking.softtimeout."+key+"="+value); diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java index 1794f8989b4..276ff107769 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java @@ -41,7 +41,7 @@ public class ModelTestCase { @Test public void testCopyParameters() { - Query q1 = new Query("?query=test1&filter=test2&defidx=content&default-index=lala&encoding=iso8859-1"); + Query q1 = new Query("?query=test1&filter=test2&default-index=content&default-index=lala&encoding=iso8859-1"); Query q2 = q1.clone(); Model r1 = q1.getModel(); Model r2 = q2.getModel(); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java deleted file mode 100644 index 4796937eb65..00000000000 --- a/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import com.yahoo.container.protect.Error; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.WordItem; -import com.yahoo.search.Query; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Unit testing of the searcher com.yahoo.search.querytransform.LegacyCombinator. - * - * @author Steinar Knutsen - */ -public class LegacyCombinatorTestCase { - - Searcher searcher; - - @Before - public void setUp() throws Exception { - searcher = new LegacyCombinator(); - } - - @Test - public void testStraightForwardSearch() { - Query q = new Query("?query=a&query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu="); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - q = new Query("?query=a+c&query.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a c b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testNoBaseQuery() { - Query q = new Query("?query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testIncompatibleNewAndOldQuery() { - Query q = new Query("?query.juhu=b&defidx.juhu=a&query.juhu.defidx=c"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - assertTrue("No expected error found.", q.errors().size() > 0); - assertEquals("Did not get invalid query parameter error as expected.", - Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); - } - - @Test - public void testNotCombinatorWithoutRoot() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - assertTrue("No expected error found.", q.errors().size() > 0); - System.out.println(q.errors()); - assertEquals("Did not get invalid query parameter error as expected.", - Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); - } - - @Test - public void testRankCombinator() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("nalle:b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testRankAndNot() { - Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+nalle:b -1:2", q.getModel().getQueryTree().toString()); - } - - @Test - public void testReqAndRankAndNot() { - Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+(RANK y:z nalle:b) -1:2", q.getModel().getQueryTree().toString()); - } - - @Test - public void testReqAndRank() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("RANK y:z nalle:b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testReqAndNot() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+y:z -nalle:b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testNewAndOld() { - Query q = new Query("?query.juhu=b&defidx.juhu=nalle&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - Set<StringPair> nastierItems = new HashSet<>(); - nastierItems.add(new StringPair("nalle", "b")); - nastierItems.add(new StringPair("y", "z")); - e.search(q); - AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); - Iterator<?> iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); - if (nastierItems.contains(asPair)) { - nastierItems.remove(asPair); - } else { - assertFalse("Got unexpected item in query tree: (" - + word.getIndexName() + ", " + word.stringValue() + ")", - true); - } - } - assertEquals("Not all expected items found in query.", 0, nastierItems.size()); - } - - @Test - public void testReqAndNotWithQuerySyntaxAll() { - Query q = new Query("?query.juhu=b+c&query.juhu.defidx=nalle&query.juhu.operator=not&query.juhu.type=any&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+y:z -(OR nalle:b nalle:c)", q.getModel().getQueryTree().toString()); - } - - @Test - public void testDefaultIndexWithoutQuery() { - Query q = new Query("?defidx.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&defidx.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - } - - private static class StringPair { - - public final String index; - public final String value; - - StringPair(String index, String value) { - super(); - this.index = index; - this.value = value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((index == null) ? 0 : index.hashCode()); - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - final StringPair other = (StringPair) obj; - if (index == null) { - if (other.index != null) - return false; - } else if (!index.equals(other.index)) - return false; - if (value == null) { - if (other.value != null) - return false; - } else if (!value.equals(other.value)) - return false; - return true; - } - - } - - @Test - public void testMultiPart() { - Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - Set<String> items = new HashSet<>(); - items.add("a"); - items.add("b"); - items.add("c"); - e.search(q); - // OK, the problem here is we have no way of knowing whether nalle or - // juhu was added first, since we have passed through HashMap instances - // inside the implementation - - AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); - Iterator<?> iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - if (items.contains(word.stringValue())) { - items.remove(word.stringValue()); - } else { - assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); - } - } - assertEquals("Not all expected items found in query.", 0, items.size()); - - Set<StringPair> nastierItems = new HashSet<>(); - nastierItems.add(new StringPair("", "a")); - nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); - nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - root = (AndItem) q.getModel().getQueryTree().getRoot(); - iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); - if (nastierItems.contains(asPair)) { - nastierItems.remove(asPair); - } else { - assertFalse("Got unexpected item in query tree: (" - + word.getIndexName() + ", " + word.stringValue() + ")", - true); - } - } - assertEquals("Not all expected items found in query.", 0, nastierItems.size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java deleted file mode 100644 index 995c1cc7a72..00000000000 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform.test; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import com.yahoo.component.ComponentId; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.WordItem; -import com.yahoo.search.Query; -import com.yahoo.search.Searcher; -import com.yahoo.search.querytransform.QueryCombinator; -import com.yahoo.search.searchchain.Execution; - -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Unit testing of the searcher com.yahoo.search.querytransform.QueryCombinator. - * - * @author Steinar Knutsen - */ -public class QueryCombinatorTestCase { - - Searcher searcher; - - @Before - @SuppressWarnings("deprecation") - public void setUp() throws Exception { - searcher = new QueryCombinator(new ComponentId("combinationTest")); - } - - @Test - public void testStraightForwardSearch() { - Query q = new Query("?query=a&query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu="); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - q = new Query("?query=a+c&query.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a c b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testNoBaseQuery() { - Query q = new Query("?query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testDefaultIndexWithoutQuery() { - Query q = new Query("?defidx.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&defidx.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - } - - private static class StringPair { - - public final String index; - public final String value; - - StringPair(String index, String value) { - super(); - this.index = index; - this.value = value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((index == null) ? 0 : index.hashCode()); - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - final StringPair other = (StringPair) obj; - if (index == null) { - if (other.index != null) - return false; - } else if (!index.equals(other.index)) - return false; - if (value == null) { - if (other.value != null) - return false; - } else if (!value.equals(other.value)) - return false; - return true; - } - - } - - @Test - public void testMultiPart() { - Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - Set<String> items = new HashSet<>(); - items.add("a"); - items.add("b"); - items.add("c"); - e.search(q); - // OK, the problem here is we have no way of knowing whether nalle or - // juhu was added first, since we have passed through HashMap instances - // inside the implementation - - AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); - Iterator<?> iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - if (items.contains(word.stringValue())) { - items.remove(word.stringValue()); - } else { - assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); - } - } - assertEquals("Not all expected items found in query.", 0, items.size()); - - Set<StringPair> nastierItems = new HashSet<>(); - nastierItems.add(new StringPair("", "a")); - nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); - nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - root = (AndItem) q.getModel().getQueryTree().getRoot(); - iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); - if (nastierItems.contains(asPair)) { - nastierItems.remove(asPair); - } else { - assertFalse("Got unexpected item in query tree: (" - + word.getIndexName() + ", " + word.stringValue() + ")", - true); - } - } - assertEquals("Not all expected items found in query.", 0, nastierItems.size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 5b3b5ca6d73..10a3b695f64 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1,14 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.rendering; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; +import com.yahoo.container.QrSearchersConfig; import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.document.DataType; @@ -19,8 +18,13 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.predicate.Predicate; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.searcher.JuniperSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -38,6 +42,7 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.Relevance; import com.yahoo.search.result.StructuredData; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import com.yahoo.search.statistics.ElapsedTimeTestCase; import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; @@ -51,32 +56,31 @@ import com.yahoo.yolean.trace.TraceNode; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.times; /** * Functional testing of {@link JsonRenderer}. * * @author Steinar Knutsen + * @author bratseth */ public class JsonRendererTestCase { - JsonRenderer originalRenderer; - JsonRenderer renderer; + private JsonRenderer originalRenderer; + private JsonRenderer renderer; public JsonRendererTestCase() { originalRenderer = new JsonRenderer(); @@ -84,23 +88,11 @@ public class JsonRendererTestCase { @Before public void setUp() throws Exception { - // Do the same dance as in production + // Use the shared renderer as a prototype object, as specified in the API contract renderer = (JsonRenderer) originalRenderer.clone(); renderer.init(); } - @After - public void tearDown() throws Exception { - renderer = null; - } - - private static final class Thingie { - @Override - public String toString() { - return "thingie"; - } - } - @Test public void testDocumentId() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" @@ -130,14 +122,6 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - private Result newEmptyResult(String[] args) { - return new Result(new Query("/?" + String.join("&", args))); - } - - private Result newEmptyResult() { - return newEmptyResult(new String[] {"query=a"}); - } - @Test public void testDataTypes() throws IOException, InterruptedException, ExecutionException, JSONException { String expected = "{\n" @@ -188,7 +172,7 @@ public class JsonRendererTestCase { @Test - public final void testTracing() throws IOException, InterruptedException, ExecutionException { + public void testTracing() throws IOException, InterruptedException, ExecutionException { // which clearly shows a trace child is created once too often... String expected = "{\n" + " \"root\": {\n" @@ -243,7 +227,7 @@ public class JsonRendererTestCase { } @Test - public final void testEmptyTracing() throws IOException, InterruptedException, ExecutionException { + public void testEmptyTracing() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -272,7 +256,7 @@ public class JsonRendererTestCase { @SuppressWarnings("unchecked") @Test - public final void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { + public void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -346,15 +330,8 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - private void subExecution(Execution execution, String color, int traceLevel) { - Execution e2 = new Execution(new Chain<Searcher>(), execution.context()); - Query subQuery = new Query("/?query=b&tracelevel=" + traceLevel); - e2.search(subQuery); - subQuery.trace(color, 1); - } - @Test - public final void testTracingOfNodesWithBothChildrenAndData() throws IOException, InterruptedException, ExecutionException { + public void testTracingOfNodesWithBothChildrenAndData() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -400,7 +377,7 @@ public class JsonRendererTestCase { @Test - public final void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws IOException, InterruptedException, ExecutionException { + public void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -441,7 +418,7 @@ public class JsonRendererTestCase { } @Test - public final void testTracingOfNestedNodesWithDataAndSubnodes() throws IOException, InterruptedException, ExecutionException { + public void testTracingOfNestedNodesWithDataAndSubnodes() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -490,7 +467,7 @@ public class JsonRendererTestCase { @Test - public final void test() throws IOException, InterruptedException, ExecutionException { + public void test() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -498,8 +475,7 @@ public class JsonRendererTestCase { + " \"children\": [\n" + " {\n" + " \"fields\": {\n" - + " \"c\": \"d\",\n" - + " \"uri\": \"http://localhost/1\"\n" + + " \"c\": \"d\"\n" + " },\n" + " \"id\": \"http://localhost/1\",\n" + " \"relevance\": 0.9,\n" @@ -523,8 +499,7 @@ public class JsonRendererTestCase { + " },\n" + " {\n" + " \"fields\": {\n" - + " \"b\": \"foo\",\n" - + " \"uri\": \"http://localhost/\"\n" + + " \"b\": \"foo\"\n" + " },\n" + " \"id\": \"http://localhost/\",\n" + " \"relevance\": 0.95,\n" @@ -555,7 +530,7 @@ public class JsonRendererTestCase { + " \"relevance\": 1.0\n" + " }\n" + "}"; - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Query q = new Query("/?query=a&tracelevel=5"); Execution execution = new Execution(Execution.Context.createContextStub()); Result r = new Result(q); r.setCoverage(new Coverage(500, 500,1,1)); @@ -603,7 +578,7 @@ public class JsonRendererTestCase { + " \"relevance\": 1.0\n" + " }\n" + "}"; - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Query q = new Query("/?query=a&tracelevel=5"); Execution execution = new Execution(Execution.Context.createContextStub()); Result r = new Result(q); r.setCoverage(new Coverage(500, 600).setDegradedReason(5)); @@ -725,7 +700,7 @@ public class JsonRendererTestCase { + " \"relevance\": 1.0\n" + " }\n" + "}\n"; - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Query q = new Query("/?query=a&tracelevel=5"); Result r = new Result(q); Throwable t = new Throwable(); StackTraceElement[] stack = new StackTraceElement[1]; @@ -885,7 +860,7 @@ public class JsonRendererTestCase { }); GroupList gl = new GroupList("customer"); Group g = new Group(new DoubleBucketId(1.0, 2.0), new Relevance(1.0)); - g.setField("something()", Integer.valueOf(7)); + g.setField("something()", 7); gl.add(g); rg.add(gl); r.hits().add(rg); @@ -958,7 +933,7 @@ public class JsonRendererTestCase { } @Test - public final void testFieldValueInHit() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testFieldValueInHit() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -993,7 +968,7 @@ public class JsonRendererTestCase { } @Test - public final void testHiddenFields() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testHiddenFields() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1017,17 +992,8 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - private Hit createHitWithOnlyHiddenFields() { - Hit h = new Hit("hiddenFields"); - h.setField("NaN", NanNumber.NaN); - h.setField("emptyString", ""); - h.setField("emptyStringFieldValue", new StringFieldValue("")); - h.setField("$vespaImplementationDetail", "Hello, World!"); - return h; - } - @Test - public final void testDebugRendering() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testDebugRendering() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1058,7 +1024,7 @@ public class JsonRendererTestCase { } @Test - public final void testTimingRendering() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + public void testTimingRendering() throws InterruptedException, ExecutionException, IOException { String expected = "{" + " \"root\": {" + " \"fields\": {" @@ -1093,7 +1059,7 @@ public class JsonRendererTestCase { } @Test - public final void testJsonCallback() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testJsonCallback() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1131,7 +1097,7 @@ public class JsonRendererTestCase { } @Test - public final void testMapInField() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testMapInField() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1187,6 +1153,54 @@ public class JsonRendererTestCase { + "}"; assertEquals("Duplicate key \"duplicate\"", validateJSON(json)); } + + @Test + public void testDynamicSummary() throws Exception { + String content = "\uFFF9Feeding\uFFFAfeed\uFFFB \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F into Vespa \uFFF9is\uFFFAbe\u001Eincrement of a set of \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F fed into Vespa \uFFF9is\u001Efloat in XML when \u001Fdocument\u001F attribute \uFFF9is\uFFFAbe\uFFFB int\u001E"; + Result result = createResult("one", content, true); + + String summary = render(result); + + String expected = + "{ \n" + + " \"root\":{ " + + " \"id\":\"toplevel\"," + + " \"relevance\":1.0," + + " \"fields\":{ " + + " \"totalCount\":0" + + " }," + + " \"children\":[ " + + " { " + + " \"id\":\"http://abc.html/\"," + + " \"relevance\":1.0," + + " \"fields\":{ " + + " \"sddocname\":\"one\",\n" + + " \"dynteaser\":\"Feeding <hi>documents</hi> into Vespa is<sep />increment of a set of <hi>documents</hi> fed into Vespa <sep />float in XML when <hi>document</hi> attribute is int<sep />\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + assertEqualJson(expected, summary); + } + + private Result newEmptyResult(String[] args) { + return new Result(new Query("/?" + String.join("&", args))); + } + + private Result newEmptyResult() { + return newEmptyResult(new String[] {"query=a"}); + } + + private Hit createHitWithOnlyHiddenFields() { + Hit h = new Hit("hiddenFields"); + h.setField("NaN", NanNumber.NaN); + h.setField("emptyString", ""); + h.setField("emptyStringFieldValue", new StringFieldValue("")); + h.setField("$vespaImplementationDetail", "Hello, World!"); + return h; + } + private String render(Result r) throws InterruptedException, ExecutionException { Execution execution = new Execution(Execution.Context.createContextStub()); return render(execution, r); @@ -1209,6 +1223,7 @@ public class JsonRendererTestCase { assertEquals("", validateJSON(expected)); assertEquals("", validateJSON(generated)); } + private String validateJSON(String presumablyValidJson) { try { new JSONObject(presumablyValidJson); @@ -1218,4 +1233,76 @@ public class JsonRendererTestCase { } } + private static final class Thingie { + @Override + public String toString() { + return "thingie"; + } + } + + private Result createResult(String sdName, String content, boolean bolding) { + Chain<Searcher> chain = createSearchChain(sdName, content); + Query query = new Query("?query=12"); + if ( ! bolding) + query = new Query("?query=12&bolding=false"); + Execution execution = createExecution(chain); + Result result = execution.search(query); + execution.fill(result); + return result; + } + + /** + * Creates a search chain which always returns a result with one hit containing information given in this + * + * @param sdName the search definition type of the returned hit + * @param content the content of the "dynteaser" field of the returned hit + */ + private Chain<Searcher> createSearchChain(String sdName, String content) { + JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), + new QrSearchersConfig(new QrSearchersConfig.Builder())); + + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + addResult(new Query("?query=12"), sdName, content, docsource); + addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); + return new Chain<>(searcher, docsource); + } + + private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + Result r = new Result(query); + FastHit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField(Hit.SDDOCNAME_FIELD, sdName); + hit.setField("dynteaser", content); + r.hits().add(hit); + docsource.addResult(query, r); + } + + private Execution createExecution(Chain<Searcher> chain) { + Map<String, List<String>> clusters = new LinkedHashMap<>(); + Map<String, SearchDefinition> searchDefs = new LinkedHashMap<>(); + searchDefs.put("one", createSearchDefinitionOne()); + SearchDefinition union = new SearchDefinition("union"); + IndexModel indexModel = new IndexModel(clusters, searchDefs, union); + return new Execution(chain, Execution.Context.createContextStub(new IndexFacts(indexModel))); + } + + private SearchDefinition createSearchDefinitionOne() { + SearchDefinition one = new SearchDefinition("one"); + + Index dynteaser = new Index("dynteaser"); + dynteaser.setDynamicSummary(true); + one.addIndex(dynteaser); + + Index bigteaser = new Index("bigteaser"); + dynteaser.setHighlightSummary(true); + one.addIndex(bigteaser); + + Index otherteaser = new Index("otherteaser"); + otherteaser.setDynamicSummary(true); + one.addIndex(otherteaser); + + return one; + } + } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java index 1d2187376a4..acb69670b14 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java @@ -58,8 +58,8 @@ public class SyncDefaultRendererTestCase { @SuppressWarnings("deprecation") @Test - public final void testRenderWriterResult() throws InterruptedException, ExecutionException { - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + public void testRenderWriterResult() throws InterruptedException, ExecutionException { + Query q = new Query("/?query=a&tracelevel=5"); q.getPresentation().setTiming(true); Result r = new Result(q); r.setCoverage(new Coverage(500, 1)); @@ -96,7 +96,7 @@ public class SyncDefaultRendererTestCase { assertTrue(f.get()); String summary = Utf8.toString(bs.toByteArray()); // TODO figure out a reasonably strict and reasonably flexible way to test - assertTrue(summary.length() > 1000); + assertTrue(summary.length() > 900); } } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java index c95019d37d6..184a8c1aa43 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java @@ -4,13 +4,21 @@ package com.yahoo.search.rendering; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; - -import com.yahoo.search.handler.SearchHandler; -import org.junit.After; -import org.junit.Before; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.searcher.JuniperSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; import com.google.common.util.concurrent.ListenableFuture; @@ -25,50 +33,112 @@ import com.yahoo.search.result.HitGroup; import com.yahoo.search.statistics.ElapsedTimeTestCase; import com.yahoo.search.statistics.TimeTracker; import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; -import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; import com.yahoo.text.Utf8; /** * Test the XML renderer * * @author Steinar Knutsen + * @author bratseth */ public class XMLRendererTestCase { - DefaultRenderer d; - - @Before - public void setUp() throws Exception { - d = new DefaultRenderer(); - d.init(); + @Test + public void testGetEncoding() { + XmlRenderer renderer = new XmlRenderer(); + renderer.init(); + assertEquals("utf-8", renderer.getEncoding()); } - @After - public void tearDown() throws Exception { + @Test + public void testGetMimeType() { + XmlRenderer renderer = new XmlRenderer(); + renderer.init(); + assertEquals("text/xml", renderer.getMimeType()); } @Test - public final void testGetEncoding() { - assertEquals("utf-8", d.getEncoding()); + public void testXmlRendering() throws Exception { + Query q = new Query("/?query=a"); + + Result result = new Result(q); + result.setCoverage(new Coverage(500, 1)); + + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + result.hits().add(h); + + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + result.hits().add(g); + + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + result.hits().add(gg); + result.hits().addError(ErrorMessage.createInternalServerError("message")); + + String summary = render(result); + + String expected = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\" coverage-docs=\"500\" coverage-nodes=\"1\" coverage-full=\"false\" coverage=\"0\" results-full=\"0\" results=\"1\">\n" + + " <error code=\"18\">Internal server error.</error>\n" + + " <errordetails>\n" + + " <error error=\"Internal server error.\" code=\"18\">message</error>\n" + + " </errordetails>\n" + + " <group relevancy=\"1.0\">\n" + + " <hit type=\"summary\" relevancy=\"0.9\">\n" + + " <field name=\"relevancy\">0.9</field>\n" + + " <field name=\"c\">d</field>\n" + + " </hit>\n" + + " </group>\n" + + " <hit type=\"grouphit\" relevancy=\"1.0\">\n" + + " <id>type grouphit</id>\n" + + " </hit>\n" + + " <hit type=\"summary\" relevancy=\"0.95\">\n" + + " <field name=\"relevancy\">0.95</field>\n" + + " <field name=\"b\">foo</field>\n" + + " </hit>\n" + + "</result>\n"; + + assertEquals(expected, summary); } @Test - public final void testGetMimeType() { - assertEquals("text/xml", d.getMimeType()); + public void testXmlRenderingOfDynamicSummary() throws Exception { + String content = "\uFFF9Feeding\uFFFAfeed\uFFFB \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F into Vespa \uFFF9is\uFFFAbe\u001Eincrement of a set of \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F fed into Vespa \uFFF9is\u001Efloat in XML when \u001Fdocument\u001F attribute \uFFF9is\uFFFAbe\uFFFB int\u001E"; + Result result = createResult("one", content, true); + + String summary = render(result); + + String expected = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\">\n" + + " <hit relevancy=\"1.0\">\n" + + " <field name=\"relevancy\">1.0</field>\n" + + " <field name=\"sddocname\">one</field>\n" + + " <field name=\"dynteaser\">Feeding <hi>documents</hi> into Vespa is<sep />increment of a set of <hi>documents</hi> fed into Vespa <sep />float in XML when <hi>document</hi> attribute is int<sep /></field>\n" + + " </hit>\n" + + "</result>\n"; + assertEquals(expected, summary); } @Test - public final void testImplicitDefaultRender() throws Exception { - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + public void testXmlRenderingWithTimeTracking() throws Exception { + Query q = new Query("/?query=a&tracelevel=5"); q.getPresentation().setTiming(true); - Result r = new Result(q); - r.setCoverage(new Coverage(500, 1)); - - TimeTracker t = new TimeTracker(new Chain<Searcher>( - new UselessSearcher("first"), new UselessSearcher("second"), - new UselessSearcher("third"))); - ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( - new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + + Result result = new Result(q); + result.setCoverage(new Coverage(500, 1)); + + TimeTracker t = new TimeTracker(new Chain<Searcher>(new NoopSearcher("first"), + new NoopSearcher("second"), + new NoopSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource(new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); t.sampleSearch(0, true); t.sampleSearch(1, true); t.sampleSearch(2, true); @@ -76,47 +146,103 @@ public class XMLRendererTestCase { t.sampleSearchReturn(2, true, null); t.sampleSearchReturn(1, true, null); t.sampleSearchReturn(0, true, null); - r.getElapsedTime().add(t); - FastHit h = new FastHit("http://localhost/", .95); - h.setField("$a", "Hello, world."); - h.setField("b", "foo"); - r.hits().add(h); - HitGroup g = new HitGroup("usual"); - h = new FastHit("http://localhost/1", .90); - h.setField("c", "d"); - g.add(h); - r.hits().add(g); - HitGroup gg = new HitGroup("type grouphit"); - gg.types().add("grouphit"); - gg.setField("e", "f"); - r.hits().add(gg); - r.hits().addError(ErrorMessage.createInternalServerError("boom")); + result.getElapsedTime().add(t); + String summary = render(result); + + assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<result total-hit-count=\"0\"", + summary.substring(0, 67)); + assertTrue(summary.contains("querytime=")); + assertTrue(summary.contains("summaryfetchtime=")); + assertTrue(summary.contains("searchtime=")); + assertTrue(summary.contains("<meta type=\"context\">")); + } + + private String render(Result result) throws Exception { + XmlRenderer renderer = new XmlRenderer(); + renderer.init(); ByteArrayOutputStream bs = new ByteArrayOutputStream(); - ListenableFuture<Boolean> f = d.render(bs, r, null, null); + ListenableFuture<Boolean> f = renderer.render(bs, result, null, null); assertTrue(f.get()); - String summary = Utf8.toString(bs.toByteArray()); + return Utf8.toString(bs.toByteArray()); + } - assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<result total-hit-count=\"0\"", - summary.substring(0, 67) - ); - assertTrue(summary.contains("<meta type=\"context\">")); - assertTrue(summary.contains("<error code=\"18\">Internal server error.</error>")); - assertTrue(summary.contains("<hit type=\"grouphit\" relevancy=\"1.0\">")); - assertTrue(summary.contains("<hit type=\"summary\" relevancy=\"0.95\">")); - assertEquals(2, occurrences("<error ", summary)); - assertTrue(summary.length() > 1000); + private Result createResult(String sdName, String content, boolean bolding) { + Chain<Searcher> chain = createSearchChain(sdName, content); + Query query = new Query("?query=12"); + if ( ! bolding) + query = new Query("?query=12&bolding=false"); + Execution execution = createExecution(chain); + Result result = execution.search(query); + execution.fill(result); + return result; + } + + /** + * Creates a search chain which always returns a result with one hit containing information given in this + * + * @param sdName the search definition type of the returned hit + * @param content the content of the "dynteaser" field of the returned hit + */ + private Chain<Searcher> createSearchChain(String sdName, String content) { + JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), + new QrSearchersConfig(new QrSearchersConfig.Builder())); + + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + addResult(new Query("?query=12"), sdName, content, docsource); + addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); + return new Chain<>(searcher, docsource); + } + + private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + Result r = new Result(query); + FastHit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField(Hit.SDDOCNAME_FIELD, sdName); + hit.setField("dynteaser", content); + r.hits().add(hit); + docsource.addResult(query, r); + } + + private Execution createExecution(Chain<Searcher> chain) { + Map<String, List<String>> clusters = new LinkedHashMap<>(); + Map<String, SearchDefinition> searchDefs = new LinkedHashMap<>(); + searchDefs.put("one", createSearchDefinitionOne()); + SearchDefinition union = new SearchDefinition("union"); + IndexModel indexModel = new IndexModel(clusters, searchDefs, union); + return new Execution(chain, Execution.Context.createContextStub(new IndexFacts(indexModel))); } - private int occurrences(String fragment, String string) { - int occurrences = 0; - int cursor = 0; - while ( -1 != (cursor = string.indexOf(fragment, cursor))) { - occurrences++; - cursor += fragment.length(); + private SearchDefinition createSearchDefinitionOne() { + SearchDefinition one = new SearchDefinition("one"); + + Index dynteaser = new Index("dynteaser"); + dynteaser.setDynamicSummary(true); + one.addIndex(dynteaser); + + Index bigteaser = new Index("bigteaser"); + dynteaser.setHighlightSummary(true); + one.addIndex(bigteaser); + + Index otherteaser = new Index("otherteaser"); + otherteaser.setDynamicSummary(true); + one.addIndex(otherteaser); + + return one; + } + + public static class NoopSearcher extends Searcher { + + public NoopSearcher(String name) { + super(new ComponentId(name)); } - return occurrences; + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + } } diff --git a/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java b/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java index c77ff2eca3c..91c8acbab12 100644 --- a/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java @@ -5,8 +5,6 @@ import com.yahoo.data.access.simple.Value; import static org.junit.Assert.*; -import org.junit.After; -import org.junit.Before; import org.junit.Test; /** @@ -15,41 +13,43 @@ import org.junit.Test; public class PositionsDataTestCase { @Test - public void testRendering() { - Value.ArrayValue arr = new Value.ArrayValue(); - Value.ObjectValue p1 = new Value.ObjectValue(); - p1.put("x", new Value.LongValue(-122057174)); - p1.put("y", new Value.LongValue(37374821)); - p1.put("latlong", new Value.StringValue("N37.374821;W122.057174")); - arr.add(p1); + public void testRenderingOfSinglePosition() { + Value.ObjectValue pos = createPosition(-122057174, 37374821, "N37.374821;W122.057174"); - PositionsData pd = new PositionsData(arr.inspect()); + PositionsData pd = new PositionsData(pos.inspect()); - String rendered = pd.toXML(); - String correct = "<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />"; - assertEquals(correct, rendered); + assertXml("<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />", pd); + assertJson("{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}", pd); + } + + @Test + public void testRenderingOfMultiplePositions() { + Value.ArrayValue arr = new Value.ArrayValue(); + arr.add(createPosition(-122057174, 37374821, "N37.374821;W122.057174")); + arr.add(createPosition(3, -7, "S0.000007;E0.000003")); - rendered = pd.toJson(); - correct = "[{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}]"; - assertEquals(correct, rendered); + PositionsData pd = new PositionsData(arr.inspect()); - Value.ObjectValue p2 = new Value.ObjectValue(); - p2.put("x", new Value.LongValue(3)); - p2.put("y", new Value.LongValue(-7)); - p2.put("latlong", new Value.StringValue("S0.000007;E0.000003")); - arr.add(p2); + assertXml("<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />" + + "<position x=\"3\" y=\"-7\" latlong=\"S0.000007;E0.000003\" />", pd); + assertJson("[{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}," + + "{\"x\":3,\"y\":-7,\"latlong\":\"S0.000007;E0.000003\"}]", pd); + } - pd = new PositionsData(arr.inspect()); + private Value.ObjectValue createPosition(long x, long y, String latlong) { + Value.ObjectValue result = new Value.ObjectValue(); + result.put("x", new Value.LongValue(x)); + result.put("y", new Value.LongValue(y)); + result.put("latlong", new Value.StringValue(latlong)); + return result; + } - rendered = pd.toXML(); - correct = "<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />" + - "<position x=\"3\" y=\"-7\" latlong=\"S0.000007;E0.000003\" />"; - assertEquals(correct, rendered); + private void assertXml(String expected, PositionsData pd) { + assertEquals(expected, pd.toXML()); + } - rendered = pd.toJson(); - correct = "[{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}," + - "{\"x\":3,\"y\":-7,\"latlong\":\"S0.000007;E0.000003\"}]"; - assertEquals(correct, rendered); + private void assertJson(String expected, PositionsData pd) { + assertEquals(expected, pd.toJson()); } } diff --git a/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java deleted file mode 100644 index 13d0c64a0c2..00000000000 --- a/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.io.Writer; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import com.yahoo.search.rendering.Renderer; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.base.Splitter; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.prelude.templates.test.BoomTemplate; -import com.yahoo.search.Query; -import com.yahoo.search.Result; - -/** - * Control helper method for result rendering/result templates. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class TemplatingTestCase { - Result result; - - @Before - public void setUp() throws Exception { - Query q = new Query("/?query=a&presentation.format=nalle&offset=1&hits=5"); - result = new Result(q); - result.setTotalHitCount(1000L); - result.hits().add(new FastHit("http://localhost/1", .95)); - result.hits().add(new FastHit("http://localhost/2", .90)); - result.hits().add(new FastHit("http://localhost/3", .85)); - result.hits().add(new FastHit("http://localhost/4", .80)); - result.hits().add(new FastHit("http://localhost/5", .75)); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testGetFirstHitNo() { - assertEquals(2, result.getTemplating().getFirstHitNo()); - } - - @Test - public final void testGetNextFirstHitNo() { - assertEquals(7, result.getTemplating().getNextFirstHitNo()); - result.getQuery().setHits(6); - assertEquals(0, result.getTemplating().getNextFirstHitNo()); - } - - @Test - public final void testGetNextLastHitNo() { - assertEquals(11, result.getTemplating().getNextLastHitNo()); - result.getQuery().setHits(6); - assertEquals(0, result.getTemplating().getNextLastHitNo()); - } - - @Test - public final void testGetLastHitNo() { - assertEquals(6, result.getTemplating().getLastHitNo()); - } - - @Test - public final void testGetPrevFirstHitNo() { - assertEquals(1, result.getTemplating().getPrevFirstHitNo()); - } - - @Test - public final void testGetPrevLastHitNo() { - assertEquals(1, result.getTemplating().getPrevLastHitNo()); - } - - @Test - public final void testGetNextResultURL() { - String next = result.getTemplating().getNextResultURL(); - Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { - "hits=5", - "query=a", - "presentation.format=nalle", - "offset=6" - })); - Set<String> actualParameters = new HashSet<>(); - Splitter s = Splitter.on("&"); - for (String parameter : s.split(next.substring(next.indexOf('?') + 1))) { - actualParameters.add(parameter); - } - assertEquals(expectedParameters, actualParameters); - } - - @Test - public final void testGetPreviousResultURL() { - String previous = result.getTemplating().getPreviousResultURL(); - Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { - "hits=5", - "query=a", - "presentation.format=nalle", - "offset=0" - })); - Set<String> actualParameters = new HashSet<>(); - Splitter s = Splitter.on("&"); - for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { - actualParameters.add(parameter); - } - assertEquals(expectedParameters, actualParameters); - } - - @Test - public final void testGetCurrentResultURL() { - String previous = result.getTemplating().getCurrentResultURL(); - Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { - "hits=5", - "query=a", - "presentation.format=nalle", - "offset=1" - })); - Set<String> actualParameters = new HashSet<>(); - Splitter s = Splitter.on("&"); - for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { - actualParameters.add(parameter); - } - assertEquals(expectedParameters, actualParameters); - } - - @Test - public final void testGetTemplates() { - @SuppressWarnings({ "unchecked", "deprecation" }) - UserTemplate<Writer> t = result.getTemplating().getTemplates(); - assertEquals("default", t.getName()); - } - - @SuppressWarnings("deprecation") - @Test - public final void testSetTemplates() { - result.getTemplating().setTemplates(new BoomTemplate("gnuff", "text/plain", "ISO-8859-15")); - @SuppressWarnings("unchecked") - UserTemplate<Writer> t = result.getTemplating().getTemplates(); - assertEquals("gnuff", t.getName()); - } - - private static class TestRenderer extends Renderer { - - @Override - public void render(Writer writer, Result result) throws IOException { - } - - @Override - public String getEncoding() { - return null; - } - - @Override - public String getMimeType() { - return null; - } - } - - @SuppressWarnings("deprecation") - @Test - public final void testUsesDefaultTemplate() { - assertTrue(result.getTemplating().usesDefaultTemplate()); - result.getTemplating().setRenderer(new TestRenderer()); - assertFalse(result.getTemplating().usesDefaultTemplate()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java deleted file mode 100644 index 11d163373ac..00000000000 --- a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result.test; - -import java.io.IOException; - -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.test.TilingTestCase; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class ArrayOutputTestCase { - - @Test - public void testArrayOutput() throws IOException { - Result r=new Result(new Query("?query=ignored")); - Hit hit=new Hit("test"); - hit.setField("phone",new XMLString("\n <item>408-555-1234</item>" + "\n <item>408-555-5678</item>\n ")); - r.hits().add(hit); - - String rendered = TilingTestCase.getRendered(r); - String[] lines= rendered.split("\n"); - assertEquals(" <field name=\"phone\">",lines[4]); - assertEquals(" <item>408-555-1234</item>",lines[5]); - assertEquals(" <item>408-555-5678</item>",lines[6]); - assertEquals(" </field>",lines[7]); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java index 15937e77899..7491e970b23 100644 --- a/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java @@ -50,6 +50,7 @@ public class ElapsedTimeTestCase { } public static class UselessSearcher extends Searcher { + public UselessSearcher(String name) { super(new ComponentId(name)); } @@ -58,9 +59,11 @@ public class ElapsedTimeTestCase { public Result search(Query query, Execution execution) { return execution.search(query); } + } private static class AlmostUselessSearcher extends Searcher { + AlmostUselessSearcher(String name) { super(new ComponentId(name)); } @@ -73,9 +76,11 @@ public class ElapsedTimeTestCase { r.hits().add(h); return r; } + } private static class NoForwardSearcher extends Searcher { + @Override public Result search(Query query, Execution execution) { Result r = new Result(query); @@ -84,6 +89,7 @@ public class ElapsedTimeTestCase { r.hits().add(h); return r; } + } private class TestingSearcher extends Searcher { diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index ed80c0bf256..a5c46fc7887 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -684,34 +684,34 @@ public class QueryTestCase { @Test public void testThatSessionIdIsUniquePerQuery() { Query q = new Query(); - assertNull(q.getSessionId(false)); - assertNull(q.getSessionId(false)); - SessionId s1 = q.getSessionId(true); + assertNull(q.getSessionId()); + assertNull(q.getSessionId()); + SessionId s1 = q.getSessionId("node-0"); assertNotNull(s1); - SessionId s2 = q.getSessionId(true); + SessionId s2 = q.getSessionId("node-0"); assertNotSame(s1, s2); assertEquals(s1, s2); assertEquals(s1.toString(), s2.toString()); Query q2 = new Query(); - assertNotEquals(q.getSessionId(false), q2.getSessionId(true)); - assertNotEquals(q.getSessionId(false).toString(), q2.getSessionId(true).toString()); + assertNotEquals(q.getSessionId(), q2.getSessionId("node-0")); + assertNotEquals(q.getSessionId().toString(), q2.getSessionId("node-0").toString()); } @Test public void testThatCloneGetANewSessionId() { Query q = new Query(); - q.getSessionId(true); + q.getSessionId("node-0"); Query clonedQ = q.clone(); - assertNull(clonedQ.getSessionId(false)); - assertNotEquals(q.getSessionId(false), clonedQ.getSessionId(true)); + assertNull(clonedQ.getSessionId()); + assertNotEquals(q.getSessionId(), clonedQ.getSessionId("node-0")); } @Test public void testThatSessionIdIsUniquePerRankProfilePerQuery() { Query q = new Query(); - SessionId s1 = q.getSessionId(true); + SessionId s1 = q.getSessionId("node-0"); q.getRanking().setProfile("my-profile"); - SessionId s2 = q.getSessionId(false); + SessionId s2 = q.getSessionId(); assertNotEquals(s1, s2); } @@ -719,14 +719,14 @@ public class QueryTestCase { public void testThatSessionIdIsNotSharedIfCreatedAfterClone() { Query q = new Query(); Query q2 = q.clone(); - assertNull(q.getSessionId(false)); - assertNull(q2.getSessionId(false)); + assertNull(q.getSessionId()); + assertNull(q2.getSessionId()); - assertNotNull(q.getSessionId(true)); - assertNull(q2.getSessionId(false)); + assertNotNull(q.getSessionId("node-0")); + assertNull(q2.getSessionId()); - assertNotNull(q2.getSessionId(true)); - assertNotEquals(q.getSessionId(false), q2.getSessionId(false)); + assertNotNull(q2.getSessionId("node-0")); + assertNotEquals(q.getSessionId(), q2.getSessionId()); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index f75b78fd36f..d7fc53a6c00 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -100,7 +100,7 @@ public class VdsVisitorTestCase { private class QueryArguments { // General query parameters String query = "test"; - long timeout = 5; + double timeout = 0.5; int offset = 0; int hits = 10; int traceLevel = 0; @@ -114,7 +114,6 @@ public class VdsVisitorTestCase { String userId = null; String groupName = null; String selection = null; - boolean headersOnly = false; long from = 0; long to = 0; String loadTypeName = null; @@ -140,7 +139,6 @@ public class VdsVisitorTestCase { userId = "1234"; groupName = null; selection = null; - headersOnly = true; from = 123; to = 456; loadTypeName = "low"; @@ -155,7 +153,7 @@ public class VdsVisitorTestCase { private Query buildQuery(QueryArguments qa) throws Exception { StringBuilder queryString = new StringBuilder(); queryString.append("/?query=").append(qa.query); - if (qa.timeout != 5) { + if (qa.timeout != 0.5) { queryString.append("&timeout=").append(qa.timeout); } if (qa.offset != 0) { @@ -192,9 +190,6 @@ public class VdsVisitorTestCase { if (qa.selection != null) { queryString.append("&streaming.selection=").append(URLEncoder.encode(qa.selection, "UTF-8")); } - if (qa.headersOnly) { - queryString.append("&streaming.headersonly=").append(qa.headersOnly); - } if (qa.from != 0) { queryString.append("&streaming.fromtimestamp=").append(qa.from); } @@ -236,7 +231,6 @@ public class VdsVisitorTestCase { } else { assertEquals(docType + " and ( " + qa.selection + " )", params.getDocumentSelection()); } - assertEquals(qa.headersOnly, params.getVisitHeadersOnly()); assertEquals(qa.from, params.getFromTimestamp()); assertEquals(qa.to, params.getToTimestamp()); if (qa.loadTypeName != null && loadTypeSet.getNameMap().get(qa.loadTypeName) != null) { @@ -276,8 +270,8 @@ public class VdsVisitorTestCase { } // Verify parameters based only on query - assertEquals(qa.timeout*1000, params.getTimeoutMs()); - assertEquals(qa.timeout*1000, params.getSessionTimeoutMs()); + assertEquals(qa.timeout*1000, params.getTimeoutMs(),0.0000001); + assertEquals(qa.timeout*1000, params.getSessionTimeoutMs(), 0.0000001); assertEquals("searchvisitor", params.getVisitorLibrary()); assertEquals(Integer.MAX_VALUE, params.getMaxPending()); assertEquals(qa.traceLevel, params.getTraceLevel()); |