Publish

5 files changed, 531 insertions, 0 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Diversity.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Diversity.java
new file mode 100644
index 00000000000..b1865ad9d75
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Diversity.java
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Objects;
+
+/**
+ * <p>The diversity settings during match phase of a query.
+ * These are the same settings for diversity during match phase that can be set in a rank profile
+ * and is used for achieving guaranteed diversity at the cost of slightly higher cost as more hits must be
+ * considered compared to plain match-phase.</p>
+ *
+ * <p>You specify an additional attribute to be the diversifier and also min diversity needed.</p>
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class Diversity implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+
+    public static final String ATTRIBUTE = "attribute";
+    public static final String MINGROUPS = "minGroups";
+    public static final String CUTOFF = "cutoff";
+    public static final String FACTOR = "factor";
+    public static final String STRATEGY = "strategy";
+
+
+    static {
+        argumentType =new QueryProfileType(Ranking.DIVERSITY);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(ATTRIBUTE, "string"));
+        argumentType.addField(new FieldDescription(MINGROUPS, "long"));
+        argumentType.freeze();
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    public enum CutoffStrategy {loose, strict};
+    private String attribute = null;
+    private Long minGroups = null;
+    private Double cutoffFactor = null;
+    private CutoffStrategy cutoffStrategy= null;
+
+    /**
+     * Sets the attribute field which will be used to guarantee diversity.
+     * Set to null (default) to disable diversification.
+     * <p>
+     * If this is set, make sure to also set the maxGroups value.
+     * <p>
+     * This attribute must be singlevalue.
+     */
+    public void setAttribute(String attribute) { this.attribute = attribute; }
+
+    /** Returns the attribute to use for diversity, or null if none */
+    public String getAttribute() { return attribute; }
+
+    /**
+     * Sets the max hits to aim for producing in the match phase.
+     * This must be set if an attribute value is set.
+     * It should be set to a reasonable fraction of the total documents on each partition.
+     */
+    public void setMinGroups(long minGroups) { this.minGroups = minGroups; }
+
+    /** Returns the max hits to aim for producing in the match phase on each content node, or null if not set */
+    public Long getMinGroups() { return minGroups; }
+
+    public void setCutoffFactor(double cutoffFactor) { this.cutoffFactor = cutoffFactor; }
+    public Double getCutoffFactor() { return cutoffFactor; }
+    public void setCutoffStrategy(String cutoffStrategy) { this.cutoffStrategy = CutoffStrategy.valueOf(cutoffStrategy); }
+    public CutoffStrategy getCutoffStrategy() { return cutoffStrategy; }
+
+    /** Internal operation - DO NOT USE */
+    public void prepare(RankProperties rankProperties) {
+        if (attribute == null && minGroups == null) return;
+
+        if (attribute != null && !attribute.isEmpty()) {
+            rankProperties.put("vespa.matchphase.diversity.attribute", attribute);
+        }
+        if (minGroups != null) {
+            rankProperties.put("vespa.matchphase.diversity.mingroups", String.valueOf(minGroups));
+        }
+        if (cutoffFactor != null) {
+            rankProperties.put("vespa.matchphase.diversity.cutoff.factor", String.valueOf(cutoffFactor));
+        }
+        if (cutoffStrategy != null) {
+            rankProperties.put("vespa.matchphase.diversity.cutoff.strategy", cutoffStrategy);
+        }
+    }
+
+    @Override
+    public Diversity clone() {
+        try {
+            return (Diversity)super.clone();
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Won't happen", e);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        if (attribute != null) hash += 11 * attribute.hashCode();
+        if (minGroups != null) hash += 13 * minGroups.hashCode();
+        if (cutoffFactor != null) hash += 17 * cutoffFactor.hashCode();
+        if (cutoffStrategy != null) hash += 19 * cutoffStrategy.hashCode();
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof Diversity)) return false;
+
+        Diversity other = (Diversity)o;
+        if ( ! Objects.equals(this.attribute, other.attribute)) return false;
+        if ( ! Objects.equals(this.minGroups, other.minGroups)) return false;
+        if ( ! Objects.equals(this.cutoffFactor, other.cutoffFactor)) return false;
+        if ( ! Objects.equals(this.cutoffStrategy, other.cutoffStrategy)) return false;
+        return true;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java
new file mode 100644
index 00000000000..ba25ddbe7e6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Objects;
+
+/**
+ * The match phase ranking settings of this query.
+ * These are the same settings for match phase that can be set in a rank profile
+ * and is used for achieving reasonable query behavior given a query which causes too many matches:
+ * The engine will fall back to retrieving the best values according to the attribute given here
+ * during matching.
+ * <p>
+ * For this feature to work well, the order given by the attribute should correlate reasonably with the order
+ * of results produced if full evaluation is performed.
+ *
+ * @author bratseth
+ */
+public class MatchPhase implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+
+    public static final String ATTRIBUTE = "attribute";
+    public static final String ASCENDING = "ascending";
+    public static final String MAX_HITS = "maxHits";
+    public static final String MAX_FILTER_COVERAGE = "maxFilterCoverage";
+
+    static {
+        argumentType =new QueryProfileType(Ranking.MATCH_PHASE);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(ATTRIBUTE, "string"));
+        argumentType.addField(new FieldDescription(ASCENDING, "boolean"));
+        argumentType.addField(new FieldDescription(MAX_HITS, "long"));
+        argumentType.addField(new FieldDescription(MAX_FILTER_COVERAGE, "double"));
+        argumentType.addField(new FieldDescription(Ranking.DIVERSITY, "query-profile", "diversity"));
+        argumentType.freeze();
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    private String attribute = null;
+    private boolean ascending = false;
+    private Long maxHits = null;
+    private Double maxFilterCoverage = 1.0;
+    private Diversity diversity = new Diversity();
+
+    /**
+     * Sets the attribute field which will be used to decide the best matches after it has been determined
+     * during matching that this query is going to cause too many matches.
+     * Set to null (default) to disable degradation.
+     * <p>
+     * If this is set, make sure to also set the maxHits value.
+     * Otherwise, the attribute setting is ignored.
+     * <p>
+     * This attribute should have fast-search turned on.
+     */
+    public void setAttribute(String attribute) { this.attribute = attribute; }
+
+    /** Returns the attribute to use for degradation, or null if none */
+    public String getAttribute() { return attribute; }
+
+    /**
+     * Set to true to sort by the attribute in ascending order when this is in use during the match phase,
+     * false (default) to use descending order.
+     */
+    public void setAscending(boolean ascending) { this.ascending = ascending; }
+
+    /**
+     * Returns the order to sort the attribute during the path phase when this takes effect.
+     */
+    public boolean getAscending() { return ascending; }
+
+    /**
+     * Sets the max hits to aim for producing in the match phase.
+     * This must be set if an attribute value is set.
+     * It should be set to a reasonable fraction of the total documents on each partition.
+     */
+    public void setMaxHits(long maxHits) { this.maxHits = maxHits; }
+
+    public void setMaxFilterCoverage(double maxFilterCoverage) {
+        if ((maxFilterCoverage < 0.0) || (maxFilterCoverage > 1.0)) {
+            throw new IllegalArgumentException("maxFilterCoverage must be in the range [0.0, 1.0]. It is " + maxFilterCoverage);
+        }
+        this.maxFilterCoverage = maxFilterCoverage;
+    }
+
+    /** Returns the max hits to aim for producing in the match phase on each content node, or null if not set */
+    public Long getMaxHits() { return maxHits; }
+
+    public Double getMaxFilterCoverage() { return maxFilterCoverage; }
+
+    public Diversity getDiversity() { return diversity; }
+
+    public void setDiversity(Diversity diversity) {
+        this.diversity = diversity;
+    }
+
+    /** Internal operation - DO NOT USE */
+    public void prepare(RankProperties rankProperties) {
+        if (attribute == null || maxHits == null) return;
+
+        rankProperties.put("vespa.matchphase.degradation.attribute", attribute);
+        if (ascending) { // backend default is descending
+            rankProperties.put("vespa.matchphase.degradation.ascendingorder", "true");
+        }
+        rankProperties.put("vespa.matchphase.degradation.maxhits", String.valueOf(maxHits));
+        rankProperties.put("vespa.matchphase.degradation.maxfiltercoverage", String.valueOf(maxFilterCoverage));
+        diversity.prepare(rankProperties);
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += 13 * Boolean.hashCode(ascending);
+        hash += 19 * diversity.hashCode();
+        if (attribute != null) hash += 11 * attribute.hashCode();
+        if (maxHits != null) hash += 17 * maxHits.hashCode();
+        hash += 23 * maxFilterCoverage.hashCode();
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof MatchPhase)) return false;
+
+        MatchPhase other = (MatchPhase)o;
+        if ( this.ascending != other.ascending) return false;
+        if ( ! Objects.equals(this.attribute, other.attribute)) return false;
+        if ( ! Objects.equals(this.maxHits, other.maxHits)) return false;
+        if ( ! Objects.equals(this.diversity, other.diversity)) return false;
+        if ( ! Objects.equals(this.maxFilterCoverage, other.maxFilterCoverage)) return false;
+        return true;
+    }
+
+    @Override
+    public MatchPhase clone() {
+        try {
+            MatchPhase clone = (MatchPhase)super.clone();
+            clone.diversity = diversity.clone();
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Won't happen", e);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java
new file mode 100644
index 00000000000..1bcd548882c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.fs4.MapEncoder;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.text.JSON;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Contains the rank features of a query.
+ *
+ * @author bratseth
+ */
+public class RankFeatures implements Cloneable {
+
+    private final Map<String, Object> features;
+
+    public RankFeatures() {
+        this(new LinkedHashMap<>());
+    }
+
+    private RankFeatures(Map<String, Object> features) {
+        this.features = features;
+    }
+
+    /** Sets a rank feature by full name to a value */
+    public void put(String name, String value) {
+        features.put(name, value);
+    }
+
+    /** Sets a tensor rank feature */
+    public void put(String name, Tensor value) {
+        features.put(name, value);
+    }
+
+    /** Returns a rank feature as a string by full name or null if not set */
+    public String get(String name) {
+        Object value = features.get(name);
+        if (value == null) return null;
+        return value.toString();
+    }
+
+    /** Returns this value as whatever type it was stored as. Returns null if the value is not set. */
+    public Object getObject(String name) {
+        return features.get(name);
+    }
+
+    /**
+     * Returns a tensor rank feature, or empty if there is no value with this name.
+     *
+     * @throws IllegalArgumentException if the value is set but is not a tensor
+     */
+    public Optional<Tensor> getTensor(String name) {
+        Object feature = features.get(name);
+        if (feature == null) return Optional.empty();
+        if (feature instanceof Tensor) return Optional.of((Tensor)feature);
+        throw new IllegalArgumentException("Expected a tensor value of '" + name + "' but has " + feature);
+    }
+
+    /**
+     * Returns the map holding the features of this.
+     * This map may be modified to change the rank features of the query.
+     */
+    public Map<String, Object> asMap() { return features; }
+
+    public boolean isEmpty() {
+        return features.isEmpty();
+    }
+
+    /**
+     * Prepares this for encoding, not for external use. See encode on Query for details.
+     * <p>
+     * If the query feature is found in the rank feature set,
+     * remove all these entries and insert them into the rank property set instead.
+     * We want to hide from the user that the query feature value is sent down as a rank property
+     * and picked up by the query feature executor in the backend.
+     */
+    public void prepare(RankProperties rankProperties) {
+        if (isEmpty()) return;
+
+        List<String> featuresToRemove = new ArrayList<>();
+        List<String> propertiesToInsert = new ArrayList<>();
+        for (String key : features.keySet()) {
+            if (key.startsWith("query(") && key.endsWith(")")) {
+                featuresToRemove.add(key);
+                propertiesToInsert.add(key.substring("query(".length(), key.length() - 1));
+            } else if (key.startsWith("$")) {
+                featuresToRemove.add(key);
+                propertiesToInsert.add(key.substring(1));
+            }
+        }
+        for (int i = 0; i < featuresToRemove.size(); ++i) {
+            rankProperties.put(propertiesToInsert.get(i), features.remove(featuresToRemove.get(i)));
+        }
+    }
+
+    public int encode(ByteBuffer buffer) {
+        return MapEncoder.encodeMap("feature", features, buffer);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if ( ! (other instanceof RankFeatures)) return false;
+
+        return this.features.equals(((RankFeatures)other).features);
+    }
+
+    @Override
+    public int hashCode() {
+        return features.hashCode();
+    }
+
+    @Override
+    public RankFeatures clone() {
+        return new RankFeatures(new LinkedHashMap<>(features));
+    }
+
+    @Override
+    public String toString() {
+        return JSON.encode(features);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java
new file mode 100644
index 00000000000..eccb8bac2d4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.fs4.GetDocSumsPacket;
+import com.yahoo.fs4.MapEncoder;
+import com.yahoo.text.JSON;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Contains the properties properties of a query.
+ * This is a multimap: Multiple properties may be set for the same key.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class RankProperties implements Cloneable {
+
+    private Map<String, List<Object>> properties = new LinkedHashMap<>();
+
+    public RankProperties() {
+        this(new LinkedHashMap<>());
+    }
+
+    private RankProperties(Map<String, List<Object>> properties) {
+        this.properties = properties;
+    }
+
+    public void put(String name, String value) {
+        put(name, (Object)value);
+    }
+
+    /** Adds a property by full name to a value */
+    public void put(String name, Object value) {
+        List<Object> list = properties.get(name);
+        if (list == null) {
+            list = new ArrayList<>();
+            properties.put(name, list);
+        }
+        list.add(value);
+    }
+
+    /**
+     * Returns a read-only list of properties properties by full name.
+     * If this is not set, null is returned. If this is explicitly set to
+     * have no values, and empty list is returned.
+     */
+    public List<String> get(String name) {
+        List<Object> values = properties.get(name);
+        if (values == null) return null;
+        if (values.isEmpty()) return Collections.<String>emptyList();
+
+        // Compatibility ...
+        List<String> stringValues = new ArrayList<>(values.size());
+        for (Object value : values)
+            stringValues.add(value.toString());
+        return Collections.unmodifiableList(stringValues);
+    }
+
+    /** Removes all properties properties for a given name */
+    public void remove(String name) {
+        properties.remove(name);
+    }
+
+    public boolean isEmpty() {
+        return properties.isEmpty();
+    }
+
+    /** Returns a modifiable map of the properties of this */
+    public Map<String, List<Object>> asMap() { return properties; }
+
+    /** Encodes this in a binary internal representation and returns the number of property maps encoded (0 or 1) */
+    public int encode(ByteBuffer buffer, boolean encodeQueryData) {
+        if (encodeQueryData) {
+            return MapEncoder.encodeObjectMultiMap("rank", properties, buffer);
+        }
+        else {
+            List<Object> sessionId = properties.get(GetDocSumsPacket.sessionIdKey);
+            if (sessionId == null) return 0;
+            return MapEncoder.encodeSingleValue("rank", GetDocSumsPacket.sessionIdKey, sessionId.get(0), buffer);
+        }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if ( ! (other instanceof RankProperties)) return false;
+
+        return this.properties.equals(((RankProperties)other).properties);
+    }
+
+    @Override
+    public int hashCode() {
+        return properties.hashCode();
+    }
+
+    @Override
+    public RankProperties clone() {
+        Map<String, List<Object>> clone = new LinkedHashMap<>();
+        for (Map.Entry<String, List<Object>> entry : properties.entrySet())
+            clone.put(entry.getKey(), new ArrayList<>(entry.getValue()));
+        return new RankProperties(clone);
+    }
+
+    @Override
+    public String toString() {
+        return JSON.encode(properties);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/package-info.java b/container-search/src/main/java/com/yahoo/search/query/ranking/package-info.java
new file mode 100644
index 00000000000..f254b327f96
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;