diff options
author | Jon Bratseth <bratseth@oath.com> | 2019-11-12 16:00:13 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-11-12 16:00:13 +0100 |
commit | 1f9f2bac2bdac8bd0ea93cca69c2e7585e73b4c2 (patch) | |
tree | 59052462ea48ed6c56ed677a2af34776b89e382b | |
parent | a437b05abe9610d0cc4a69fd52a5908336f845e8 (diff) | |
parent | 5aab29c7d5800cf9d0d6e4aa9093d8eb080dcb07 (diff) |
Merge pull request #11280 from vespa-engine/bratseth/summary-feature-access-improvements
Bratseth/summary feature access improvements
6 files changed, 57 insertions, 22 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index c6d378bd64b..51ce07c40bd 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -7278,6 +7278,7 @@ ], "methods": [ "public void <init>(com.yahoo.data.access.Inspector)", + "public static com.yahoo.search.result.FeatureData empty()", "public com.yahoo.data.access.Inspector inspect()", "public java.lang.String toJson()", "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)", @@ -7383,6 +7384,7 @@ "public com.yahoo.search.Query getQuery()", "public com.yahoo.processing.Request request()", "public final void setQuery(com.yahoo.search.Query)", + "public com.yahoo.search.result.FeatureData features()", "public void setSearcherSpecificMetaData(com.yahoo.search.Searcher, java.lang.Object)", "public java.lang.Object getSearcherSpecificMetaData(com.yahoo.search.Searcher)", "protected void close()", diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 244fad4efde..338add37213 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -5,6 +5,7 @@ import com.yahoo.data.access.ObjectTraverser; import com.yahoo.document.GlobalId; import com.yahoo.net.URI; import com.yahoo.search.query.Sorting; +import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; import com.yahoo.search.result.Relevance; import com.yahoo.data.access.Inspector; @@ -29,6 +30,7 @@ import java.util.function.BiConsumer; * @author Steinar Knutsen */ public class FastHit extends Hit { + private static final byte [] emptyGID = new byte[GlobalId.LENGTH]; /** The index of the content node this hit originated at */ private int distributionKey = 0; @@ -187,6 +189,16 @@ public class FastHit extends Hit { } /** + * Returns values for the features listed in + * <a href="https://docs.vespa.ai/documentation/reference/search-definitions-reference.html#summary-features">summary-features</a> + * in the rank profile specified in the query producing this. + */ + public FeatureData features() { + FeatureData data = (FeatureData)getField("summaryfeatures"); + return data == null ? super.features() : data; + } + + /** * <p>Returns a field value from this Hit. The value is either a stored value from the Document represented by * this Hit, or a generated value added during later processing.</p> * diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 1fd8f6e7e17..4895db04462 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -6,12 +6,14 @@ import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Type; import com.yahoo.data.JsonProducer; import com.yahoo.data.access.simple.JsonRender; +import com.yahoo.data.access.simple.Value; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.JsonFormat; import com.yahoo.tensor.serialization.TypedBinaryFormat; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; @@ -22,6 +24,9 @@ import java.util.Set; */ public class FeatureData implements Inspectable, JsonProducer { + // WARNING: Not thread safe but using a shared empty. Take care if adding mutating methods. + private static final FeatureData empty = new FeatureData(Value.empty()); + private final Inspector value; private Set<String> featureNames = null; @@ -32,6 +37,8 @@ public class FeatureData implements Inspectable, JsonProducer { this.value = value; } + public static FeatureData empty() { return empty; } + /** * Returns the fields of this as an inspector, where tensors are represented as binary data * which can be decoded using @@ -42,6 +49,7 @@ public class FeatureData implements Inspectable, JsonProducer { @Override public String toJson() { + if (this == empty) return "{}"; if (jsonForm != null) return jsonForm; jsonForm = writeJson(new StringBuilder()).toString(); @@ -60,7 +68,7 @@ public class FeatureData implements Inspectable, JsonProducer { * (that is, if it is a tensor with nonzero rank) */ public Double getDouble(String featureName) { - Inspector featureValue = value.field(featureName); + Inspector featureValue = getInspector(featureName); if ( ! featureValue.valid()) return null; switch (featureValue.type()) { @@ -75,7 +83,7 @@ public class FeatureData implements Inspectable, JsonProducer { * This will return any feature value: Scalars are returned as a rank 0 tensor. */ public Tensor getTensor(String featureName) { - Inspector featureValue = value.field(featureName); + Inspector featureValue = getInspector(featureName); if ( ! featureValue.valid()) return null; switch (featureValue.type()) { @@ -85,8 +93,17 @@ public class FeatureData implements Inspectable, JsonProducer { } } + private Inspector getInspector(String featureName) { + Inspector featureValue = value.field(featureName); + if (featureValue.valid()) return featureValue; + + // Try to wrap by rankingExpression(name) + return value.field("rankingExpression(" + featureName + ")"); + } + /** Returns the names of the features available in this */ public Set<String> featureNames() { + if (this == empty) return Collections.emptySet(); if (featureNames != null) return featureNames; featureNames = new HashSet<>(); diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index bbfa6ec62ad..fc416c0d930 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -21,15 +21,11 @@ import java.util.TreeMap; import java.util.function.BiConsumer; /** - * <p>A search hit. The identifier of the hit is the uri - * (the uri is immutable once set). - * If two hits have the same uri they are equal per definition. - * Hits are naturally ordered by decreasing relevance. - * Note that this definition of equals and natural ordering is inconsistent.</p> + * <p>An item in the result of executing a query.</p> * * <p>Hits may be of the <i>meta</i> type, meaning that they contain some information - * about the query or result which does not represent a particular piece of matched - * content. Meta hits are not counted in the hit count of the result, and should + * about the query or result which does not represent a particular matched item. + * Meta hits are not counted in the hit count of the result, and should * usually never be filtered out.</p> * * <p>Some hit sources may produce hits which are not <i>filled</i>. A non-filled @@ -112,7 +108,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } /** - * Creates a minimal valid hit having relevance 1000 + * Creates a minimal valid hit having relevance 1 * * @param id the URI of a hit. This should be unique for this hit (but not for this * <i>object instance</i> of course). For hit types refering to resources, @@ -203,7 +199,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * it is simply any unique string identification * @param relevance a relevance measure, preferably normalized between 0 and 1 * @param source the name of the source of this hit, or null if no source is being specified - * @throws IllegalArgumentException if the given relevance is not between 0 and 1000 */ public Hit(String id, double relevance, String source) { this(id, new Relevance(relevance), source, null); @@ -219,7 +214,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * @param relevance a relevance measure, preferably normalized between 0 and 1 * @param source the name of the source of this hit, or null if no source is being specified * @param query the query having this as a hit - * @throws IllegalArgumentException if the given relevance is not between 0 and 1000 */ public Hit(String id, double relevance, String source, Query query) { this(id, new Relevance(relevance), source); @@ -234,7 +228,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * it is simply any unique string identification * @param relevance the relevance of this hit * @param source the name of the source of this hit - * @throws IllegalArgumentException if the given relevance is not between 0 and 1000 */ public Hit(String id, Relevance relevance, String source) { this(id, relevance, source, null); @@ -250,7 +243,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * @param relevance the relevance of this hit * @param source the name of the source of this hit * @param query the query having this as a hit - * @throws IllegalArgumentException if the given relevance is not between 0 and 1000 */ public Hit(String id, Relevance relevance, String source, Query query) { this.id = new URI(id); @@ -569,6 +561,14 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } } + /** + * Returns the features computed for this hit. This is never null but may be empty. + * This default implementation always returns empty. + */ + public FeatureData features() { + return FeatureData.empty(); + } + /** Attach some data to this hit for this searcher */ public void setSearcherSpecificMetaData(Searcher searcher, Object data) { if (searcherSpecificMetaData == null) { diff --git a/container-search/src/main/java/com/yahoo/search/result/Relevance.java b/container-search/src/main/java/com/yahoo/search/result/Relevance.java index da4a1942bc7..90dbc493b9d 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Relevance.java +++ b/container-search/src/main/java/com/yahoo/search/result/Relevance.java @@ -2,8 +2,8 @@ package com.yahoo.search.result; /** - * A relevance double value. These values should always be normalized between 0 and 1 (where 1 means perfect), - * however, this is not enforced. + * A relevance double value. These values should ideally be normalized between 0 and 1 (where 1 means "perfect"), + * however this is not enforced. * <p> * Sources may create subclasses of this to include additional information or functionality. * @@ -19,10 +19,10 @@ public class Relevance implements Comparable<Relevance> { * This initial value should ideally be a normalized value * between 0 and 1, but that is not enforced. * - * @param score the inital value (rank score) + * @param score the initial value (rank score) */ public Relevance(double score) { - this.score=score; + this.score = score; } /** diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index d8924e16ff9..d1399cabc75 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -67,6 +67,7 @@ public class SlimeSummaryTestCase { assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); assertNull(hit.getField("summaryfeatures")); + assertTrue(hit.features().featureNames().isEmpty()); } @Test @@ -116,11 +117,12 @@ public class SlimeSummaryTestCase { } assertEquals(tensor1, hit.getField("tensor_field1")); assertEquals(tensor2, hit.getField("tensor_field2")); - FeatureData featureData = (FeatureData)hit.getField("summaryfeatures"); - assertEquals("double_feature,tensor1_feature,tensor2_feature", + FeatureData featureData = hit.features(); + assertEquals("double_feature,rankingExpression(tensor1_feature),tensor2_feature", featureData.featureNames().stream().sorted().collect(Collectors.joining(","))); assertEquals(0.5, featureData.getDouble("double_feature"), 0.00000001); assertEquals(tensor1, featureData.getTensor("tensor1_feature")); + assertEquals(tensor1, featureData.getTensor("rankingExpression(tensor1_feature)")); assertEquals(tensor2, featureData.getTensor("tensor2_feature")); } @@ -266,7 +268,7 @@ public class SlimeSummaryTestCase { Slime slime = new Slime(); Cursor summaryFeatures = slime.setObject(); summaryFeatures.setDouble("double_feature", 0.5); - summaryFeatures.setData("tensor1_feature", TypedBinaryFormat.encode(tensor1)); + summaryFeatures.setData("rankingExpression(tensor1_feature)", TypedBinaryFormat.encode(tensor1)); summaryFeatures.setData("tensor2_feature", TypedBinaryFormat.encode(tensor2)); expected.put("summaryfeatures", new FeatureData(new SlimeAdapter(slime.get()))); @@ -419,7 +421,9 @@ public class SlimeSummaryTestCase { if (tensor1 !=null && tensor2 != null) { Cursor summaryFeatures = docsum.setObject("summaryfeatures"); summaryFeatures.setDouble("double_feature", 0.5); - summaryFeatures.setData("tensor1_feature", TypedBinaryFormat.encode(tensor1)); + + // Values produced by functions are wrapped in rankingExpression(function-name) + summaryFeatures.setData("rankingExpression(tensor1_feature)", TypedBinaryFormat.encode(tensor1)); summaryFeatures.setData("tensor2_feature", TypedBinaryFormat.encode(tensor2)); } } |