Publish

6 files changed, 683 insertions, 0 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
new file mode 100644
index 00000000000..064e38d91fc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * Searcher that sets cache control HTTP headers in response based on query/GET parameters to
+ * control caching done by proxy/caches such as YSquid and YTS:
+ * <ul>
+ *    <li>max-age=XXX - set with &amp;cachecontrol.maxage parameter
+ *    <li>stale-while-revalidate=YYY - set with &amp;cachecontrol.staleage
+ *    <li>no-cache - if Vespa &amp;noCache or &amp;cachecontrol.nocache parameter is set to true
+ * </ul>
+ *
+ * <p>This is controlled through the three query parameters <code>cachecontrol.maxage</code>,
+ * <code>cachecontrol.staleage</code> and <code>cachecontrol.nocache</code>, with the obvious meanings.</p>
+ *
+ * Example:
+ * <ul>
+ *    <li>Request: "?query=foo&amp;cachecontrol.maxage=60&amp;cachecontrol.staleage=3600"
+ *    <li>Response HTTP header: "Cache-Control: max-age=60, revalidate-while-stale=3600"
+ * </ul>
+ *
+ * Further documentation on use of Cache-Control headers:
+ * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
+ *
+ * @author Frode Lundgren
+ */
+public class CacheControlSearcher extends Searcher {
+
+    private static final CompoundName cachecontrolNocache=new CompoundName("cachecontrol.nocache");
+    private static final CompoundName cachecontrolMaxage=new CompoundName("cachecontrol.maxage");
+    private static final CompoundName cachecontrolStaleage=new CompoundName("cachecontrol.staleage");
+
+    public static final String CACHE_CONTROL_HEADER = "Cache-Control";
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        query.trace("CacheControlSearcher: Running version $Revision$", false, 6);
+        Result result = execution.search(query);
+        query = result.getQuery();
+
+        if (result.getHeaders(true) == null) {
+            query.trace("CacheControlSearcher: No HTTP header map available - skipping searcher.", false, 5);
+            return result;
+        }
+
+        // If you specify no-cache, no further cache control headers make sense
+        if (query.properties().getBoolean(cachecontrolNocache, false) || query.getNoCache()) {
+            result.getHeaders(true).put(CACHE_CONTROL_HEADER, "no-cache");
+            query.trace("CacheControlSearcher: Added no-cache header", false, 4);
+            return result;
+        }
+
+        // Handle max-age header
+        int maxage = query.properties().getInteger(cachecontrolMaxage, -1);
+        if (maxage > 0) {
+            result.getHeaders(true).put(CACHE_CONTROL_HEADER, "max-age=" + maxage);
+            query.trace("CacheControlSearcher: Set max-age header to " + maxage, false, 4);
+        }
+
+        // Handle stale-while-revalidate header
+        int staleage = query.properties().getInteger(cachecontrolStaleage, -1);
+        if (staleage > 0) {
+            result.getHeaders(true).put(CACHE_CONTROL_HEADER, "stale-while-revalidate=" + staleage);
+            query.trace("CacheControlSearcher: Set stale-while-revalidate header to " + maxage, false, 4);
+        }
+
+        return result;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
new file mode 100644
index 00000000000..cdbf864f7fd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+/**
+ * Searcher which can enforce HTTP connection close based on query properties.
+ *
+ * <p>
+ * This searcher informs the client to close a persistent HTTP connection if the
+ * connection is older than the configured max lifetime. This is done by adding
+ * the "Connection" HTTP header with the value "Close" to the result.
+ * </p>
+ *
+ * <p>
+ * The searcher reads the query property "connectioncontrol.maxlifetime", which
+ * is an integer number of seconds, to get the value for maximum connection
+ * lifetime. Setting it to zero will enforce connection close independently of
+ * the age of the connection. Typical usage would be as follows:
+ * </p>
+ *
+ * <ol>
+ * <li>Add the ConnectionControlSearcher to the default search chain of your
+ * application. (It has no special ordering considerations.)</li>
+ *
+ * <li>For the default query profile of your application, set a reasonable value
+ * for "connectioncontrol.maxlifetime". The definition of reasonable will be
+ * highly application dependent, but it should always be less than the grace
+ * period when taking the container out of production traffic.</li>
+ *
+ * <li>Deploy application. The container will now inform clients to close
+ * connections/reconnect within the configured time limit.
+ * </ol>
+ *
+ * @author frodelu
+ * @author Steinar Knutsen
+ */
+public class ConnectionControlSearcher extends Searcher {
+
+    private final String simpleName = this.getClass().getSimpleName();
+
+    private final LongSupplier clock;
+
+    private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = new CompoundName("connectioncontrol.maxlifetime");
+    private static final String HTTP_CONNECTION_HEADER_NAME = "Connection";
+    private static final String HTTP_CONNECTION_CLOSE_ARGUMENT = "Close";
+
+    public ConnectionControlSearcher() {
+        this(() -> System.currentTimeMillis());
+    }
+
+    private ConnectionControlSearcher(LongSupplier clock) {
+        this.clock = clock;
+    }
+
+    /**
+     * Create a searcher instance suitable for unit tests.
+     *
+     * @param clock a simulated or real clock behaving similarly to System.currentTimeMillis()
+     * @return a fully initialised instance
+     */
+    public static ConnectionControlSearcher createTestInstance(LongSupplier clock) {
+        return new ConnectionControlSearcher(clock);
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Result result = execution.search(query);
+
+        query.trace(false, 3, simpleName, " updating headers.");
+        keepAliveProcessing(query, result);
+        return result;
+    }
+
+    /**
+     * If the HTTP connection has been alive for too long, set the header
+     * "Connection: Close" to tell the client to close the connection after this
+     * request.
+     */
+    private void keepAliveProcessing(Query query, Result result) {
+        int maxLifetimeSeconds = query.properties().getInteger(KEEPALIVE_MAXLIFETIMESECONDS, -1);
+
+        if (maxLifetimeSeconds < 0) {
+            return;
+        } else if (maxLifetimeSeconds == 0) {
+            result.getHeaders(true).put(HTTP_CONNECTION_HEADER_NAME, HTTP_CONNECTION_CLOSE_ARGUMENT);
+            query.trace(false, 5, simpleName, ": Max HTTP connection lifetime set to 0; adding \"", HTTP_CONNECTION_HEADER_NAME,
+                    ": ", HTTP_CONNECTION_CLOSE_ARGUMENT, "\" header");
+        } else {
+            setCloseIfLifetimeExceeded(query, result, maxLifetimeSeconds);
+        }
+    }
+
+    private void setCloseIfLifetimeExceeded(Query query, Result result, int maxLifetimeSeconds) {
+        final HttpRequest httpRequest = query.getHttpRequest();
+        if (httpRequest == null) {
+            query.trace(false, 5, simpleName, " got max lifetime = ", maxLifetimeSeconds,
+                    ", but got no JDisc request. Setting no header.");
+            return;
+        }
+
+        final long connectedAtMillis = httpRequest.getJDiscRequest().getConnectedAt(TimeUnit.MILLISECONDS);
+        final long maxLifeTimeMillis = maxLifetimeSeconds * 1000L;
+        if (connectedAtMillis + maxLifeTimeMillis < clock.getAsLong()) {
+            result.getHeaders(true).put(HTTP_CONNECTION_HEADER_NAME, HTTP_CONNECTION_CLOSE_ARGUMENT);
+            query.trace(false, 5, simpleName, ": Max HTTP connection lifetime (", maxLifetimeSeconds, ") exceeded; adding \"",
+                    HTTP_CONNECTION_HEADER_NAME, ": ", HTTP_CONNECTION_CLOSE_ARGUMENT, "\" header");
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java
new file mode 100644
index 00000000000..d99cb72f5a3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java
@@ -0,0 +1,191 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.metrics.simple.Counter;
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * Check whether the query tree seems to be "well formed". In other words, run heurestics against
+ * the input data to see whether the query should sent to the search backend.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class InputCheckingSearcher extends Searcher {
+
+    private final Counter utfRejections;
+    private final Counter repeatedConsecutiveTermsInPhraseRejections;
+    private final Counter repeatedTermsInPhraseRejections;
+    private static final Logger log = Logger.getLogger(InputCheckingSearcher.class.getName());
+    private final int MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE = 5;
+    private final int MAX_REPEATED_TERMS_IN_PHRASE=10;
+
+    public InputCheckingSearcher(MetricReceiver metrics) {
+        utfRejections = metrics.declareCounter("double_encoded_utf8_rejections");
+        repeatedTermsInPhraseRejections = metrics.declareCounter("repeated_terms_in_phrase_rejections");
+        repeatedConsecutiveTermsInPhraseRejections = metrics.declareCounter("repeated_consecutive_terms_in_phrase_rejections");
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        try {
+            checkQuery(query);
+        } catch (IllegalArgumentException e) {
+            if (log.isLoggable(LogLevel.DEBUG)) {
+                log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage());
+            }
+            return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage()));
+        }
+        return execution.search(query);
+    }
+
+    private void checkQuery(Query query) {
+        doubleEncodedUtf8(query);
+        checkPhrases(query.getModel().getQueryTree().getRoot());
+        // add new heuristics here
+    }
+
+    private void checkPhrases(Item queryItem) {
+        if (queryItem instanceof PhraseItem) {
+            PhraseItem phrase = (PhraseItem) queryItem;
+            repeatedConsecutiveTermsInPhraseCheck(phrase);
+            repeatedTermsInPhraseCheck(phrase);
+        } else  if (queryItem instanceof CompositeItem) {
+            CompositeItem asComposite = (CompositeItem) queryItem;
+            for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) {
+                checkPhrases(i.next());
+            }
+        }
+    }
+
+    private void repeatedConsecutiveTermsInPhraseCheck(PhraseItem phrase) {
+        if (phrase.getItemCount() > MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) {
+            String prev = null;
+            int repeatedCount = 0;
+            for (int i = 0; i < phrase.getItemCount(); ++i) {
+                Item item = phrase.getItem(i);
+                if (item instanceof TermItem) {
+                    TermItem term = (TermItem) item;
+                    String current = term.getIndexedString();
+                    if (prev != null) {
+                        if (prev.equals(current)) {
+                            repeatedCount++;
+                            if (repeatedCount >= MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) {
+                                repeatedConsecutiveTermsInPhraseRejections.add();
+                                throw new IllegalArgumentException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE +
+                                        " ocurrences of term '" + current + "' in a row detected in phrase : " + phrase.toString());
+                            }
+                        } else {
+                            repeatedCount = 0;
+                        }
+                    }
+                    prev = current;
+                } else {
+                    prev = null;
+                    repeatedCount = 0;
+                }
+            }
+        }
+    }
+    private static final class Count {
+        private int v;
+        Count(int initial) { v = initial; }
+        void inc() { v++; }
+        int get() { return v; }
+    }
+    private void repeatedTermsInPhraseCheck(PhraseItem phrase) {
+        if (phrase.getItemCount() > MAX_REPEATED_TERMS_IN_PHRASE) {
+            Map<String, Count> repeatedCount = new HashMap<>();
+            for (int i = 0; i < phrase.getItemCount(); ++i) {
+                Item item = phrase.getItem(i);
+                if (item instanceof TermItem) {
+                    TermItem term = (TermItem) item;
+                    String current = term.getIndexedString();
+                    Count count = repeatedCount.get(current);
+                    if (count != null) {
+                        if (count.get() >= MAX_REPEATED_TERMS_IN_PHRASE) {
+                            repeatedTermsInPhraseRejections.add();
+                            throw new IllegalArgumentException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE +
+                                    " occurrences of term '" + current + "' in phrase : " + phrase.toString());
+                        }
+                        count.inc();
+                    } else {
+                        repeatedCount.put(current, new Count(1));
+                    }
+                }
+            }
+        }
+    }
+
+
+    private void doubleEncodedUtf8(Query query) {
+        int singleCharacterTerms = countSingleCharacterUserTerms(query.getModel().getQueryTree());
+        if (singleCharacterTerms <= 4) {
+            return;
+        }
+        String userInput = query.getModel().getQueryString();
+        ByteBuffer asOctets = ByteBuffer.allocate(userInput.length());
+        boolean asciiOnly = true;
+        for (int i = 0; i < userInput.length(); ++i) {
+            char c = userInput.charAt(i);
+            if (c > 255) {
+                return; // not double (or more) encoded
+            }
+            if (c > 127) {
+                asciiOnly = false;
+            }
+            asOctets.put((byte) c);
+        }
+        if (asciiOnly) {
+            return;
+        }
+        asOctets.flip();
+        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
+                .onUnmappableCharacter(CodingErrorAction.REPORT);
+        // OK, unmappable character is sort of theoretical, but added to be explicit
+        try {
+            decoder.decode(asOctets);
+        } catch (CharacterCodingException e) {
+            return;
+        }
+        utfRejections.add();
+        throw new IllegalArgumentException("The user input has been determined to be double encoded UTF-8."
+                + " Please investigate whether this is a false positive.");
+    }
+
+    private int countSingleCharacterUserTerms(Item queryItem) {
+        if (queryItem instanceof CompositeItem) {
+            int sum = 0;
+            CompositeItem asComposite = (CompositeItem) queryItem;
+            for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) {
+                sum += countSingleCharacterUserTerms(i.next());
+            }
+            return sum;
+        } else if (queryItem instanceof WordItem) {
+            WordItem word = (WordItem) queryItem;
+            return (word.isFromQuery() && word.stringValue().length() == 1) ? 1 : 0;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
new file mode 100755
index 00000000000..95cec1d0960
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
@@ -0,0 +1,219 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ClusterInfoConfig;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.config.RateLimitingConfig;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.yolean.chain.Provides;
+
+import java.time.Clock;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * A simple rate limiter.
+ * <p>
+ * This takes these query parameter arguments:
+ * <ul>
+ *     <li>rate.id - (String) the id of the client from rate limiting perspective
+ *     <li>rate.cost - (Double) the cost Double of this query. This is read after executing the query and hence can be set
+ *     by downstream searchers inspecting the result to allow differencing the cost of various queries. Default is 1.
+ *     <li>rate.quota - (Double) the cost per second a particular id is allowed to consume in this system.
+ *     <li>rate.idDimension - (String) the name of the rate-id dimension used when logging metrics.
+ *                                 If this is not specified, the metric will be logged without dimensions.
+ *     <li>rate.dryRun - (Boolean) emit metrics on rejected requests but don't actually reject them
+ * </ul>
+ * <p>
+ * Whenever quota is exceeded for an id this searcher will reject queries from that id by
+ * returning a result containing a status 429 error.
+ * <p>
+ * If rate.id or rate.quota is not set in Query.properties this searcher will do nothing.
+ * <p>
+ * Metrics: This will emit the count metric requestsOverQuota with the dimension [rate.idDimension=rate.id]
+ * counting rejected requests.
+ * <p>
+ * Ordering: This searcher Provides rateLimiting
+ *
+ * @author bratseth
+ */
+@Provides(RateLimitingSearcher.RATE_LIMITING)
+public class RateLimitingSearcher extends Searcher {
+
+    /** Constant containing the name this Provides - "rateLimiting", for ordering constraints */
+    public static final String RATE_LIMITING = "rateLimiting";
+
+    public static final CompoundName idKey = new CompoundName("rate.id");
+    public static final CompoundName costKey = new CompoundName("rate.cost");
+    public static final CompoundName quotaKey = new CompoundName("rate.quota");
+    public static final CompoundName idDimensionKey = new CompoundName("rate.idDimension");
+    public static final CompoundName dryRunKey = new CompoundName("rate.dryRun");
+
+    private static final String requestsOverQuotaMetricName = "requestsOverQuota";
+
+    /** Used to divide quota by nodes. Assumption: All nodes get the same share of traffic. */
+    private final int nodeCount;
+
+    /** Shared capacity across all threads. Each thread will ask for more capacity from here when they run out. */
+    private final AvailableCapacity availableCapacity;
+
+    /** Capacity already allocated to this thread */
+    private final ThreadLocal<Map<String, Double>> allocatedCapacity = new ThreadLocal<>();
+
+    /** For emitting metrics */
+    private final Metric metric;
+
+    /**
+     * How much capacity to allocate to a thread each time it runs out.
+     * A higher value means less contention and less accuracy.
+     */
+    private final double capacityIncrement;
+
+    /** How often to check for new capacity if we have run out */
+    private final double recheckForCapacityProbability;
+
+    @Inject
+    public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, Metric metric) {
+        this(rateLimitingConfig, clusterInfoConfig, metric, Clock.systemUTC());
+    }
+
+    /** For testing - allows injection of a timer to avoid depending on the system clock */
+    public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, Metric metric, Clock clock) {
+        this.capacityIncrement = rateLimitingConfig.capacityIncrement();
+        this.recheckForCapacityProbability = rateLimitingConfig.recheckForCapacityProbability();
+        this.availableCapacity = new AvailableCapacity(rateLimitingConfig.maxAvailableCapacity(), clock);
+
+        this.nodeCount = clusterInfoConfig.nodeCount();
+
+        this.metric = metric;
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        String id = query.properties().getString(idKey);
+        Double rate = query.properties().getDouble(quotaKey);
+        if (id == null || rate == null) {
+            query.trace(false, 6, "Skipping rate limiting check. Need both " + idKey + " and " + quotaKey + " set");
+            return execution.search(query);
+        }
+
+        rate = rate / nodeCount;
+
+        if (allocatedCapacity.get() == null) // new thread
+            allocatedCapacity.set(new HashMap<>());
+        if (allocatedCapacity.get().get(id) == null) // new id in this thread
+            requestCapacity(id, rate);
+
+        // Check if there is capacity available. Cannot check for exact cost as it may be computed after execution
+        // no capacity means we're over rate. Only recheck occasionally to limit synchronization.
+        if (getAllocatedCapacity(id) <= 0 && ThreadLocalRandom.current().nextDouble() < recheckForCapacityProbability) {
+            requestCapacity(id, rate);
+        }
+
+        if (rate==0 || getAllocatedCapacity(id) <= 0) { // we are still over rate: reject
+            metric.add(requestsOverQuotaMetricName, 1, createContext(query.properties().getString(idDimensionKey, ""), id));
+            if ( ! query.properties().getBoolean(dryRunKey, false))
+                return new Result(query, new ErrorMessage(429, "Too many requests", "Allowed rate: " + rate + "/s"));
+        }
+
+        Result result = execution.search(query);
+        addAllocatedCapacity(id, - query.properties().getDouble(costKey, 1.0));
+
+        if (getAllocatedCapacity(id) <= 0) // make sure we ask for more with 100% probability when first running out
+            requestCapacity(id, rate);
+
+        return result;
+    }
+
+    private Metric.Context createContext(String dimensionName, String dimensionValue) {
+        if (dimensionName.isEmpty())
+            return metric.createContext(Collections.emptyMap());
+        return metric.createContext(Collections.singletonMap(dimensionName, dimensionValue));
+    }
+
+    private double getAllocatedCapacity(String id) {
+        Double value = allocatedCapacity.get().get(id);
+        if (value == null) return 0;
+        return value;
+    }
+
+    private void addAllocatedCapacity(String id, double newCapacity) {
+        Double capacity = allocatedCapacity.get().get(id);
+        if (capacity != null)
+            newCapacity += capacity;
+        allocatedCapacity.get().put(id, newCapacity);
+    }
+
+    private void requestCapacity(String id, double rate) {
+        double minimumRequested = Math.max(0, -getAllocatedCapacity(id)); // If we are below, make sure we reach 0
+        double preferredRequested = Math.max(capacityIncrement, -getAllocatedCapacity(id));
+        addAllocatedCapacity(id, availableCapacity.request(id, minimumRequested, preferredRequested, rate));
+    }
+
+    /**
+     * This keeps track of the current "capacity" (total cost) available to each client (rate id)
+     * across all threads. Capacity is supplied at the rate per second given by the clients quota.
+     * When all the capacity is spent, no further capacity will be handed out, leading to request rejection.
+     * Capacity has a max value it will never exceed to avoid clients saving capacity for future overspending.
+     */
+    private static class AvailableCapacity {
+
+        private final double maxAvailableCapacity;
+        private final Clock clock;
+
+        private final Map<String, CapacityAllocation> available = new HashMap<>();
+
+        public AvailableCapacity(double maxAvailableCapacity, Clock clock) {
+            this.maxAvailableCapacity = maxAvailableCapacity;
+            this.clock = clock;
+        }
+
+        /** Returns an amount of capacity between 0 and the requested amount based on availability for this id */
+        public synchronized double request(String id, double minimumRequested, double preferredRequested, double rate) {
+            CapacityAllocation allocation = available.get(id);
+            if (allocation == null) {
+                allocation = new CapacityAllocation(rate, clock);
+                available.put(id, allocation);
+            }
+            return allocation.request(minimumRequested, preferredRequested, rate, maxAvailableCapacity);
+        }
+
+    }
+
+    private static class CapacityAllocation {
+
+        private double capacity;
+        private final Clock clock;
+        private long lastAllocatedTime;
+
+        public CapacityAllocation(double initialCapacity, Clock clock) {
+            this.capacity = initialCapacity;
+            this.clock = clock;
+            lastAllocatedTime = clock.millis();
+        }
+
+        public double request(double minimumRequested, double preferredRequested, double rate, double maxAvailableCapacity) {
+            if ( preferredRequested > capacity) { // attempt to allocate more
+                // rate is per second so we get rate/1000 per millisecond
+                long currentTime = clock.millis();
+                capacity += Math.min(maxAvailableCapacity, rate/1000d * (Math.max(0, currentTime - lastAllocatedTime)));
+                lastAllocatedTime = currentTime;
+            }
+            double grantedCapacity = Math.min(capacity/10, preferredRequested); // /10 to avoid stealing all capacity when low
+            if (grantedCapacity < minimumRequested)
+                grantedCapacity = Math.min(minimumRequested, capacity);
+            capacity = capacity - grantedCapacity;
+            return grantedCapacity;
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java
new file mode 100644
index 00000000000..ff00c8edb9b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vespa.config.search.AttributesConfig;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Validates that the attribute given as match-phase override is actually a valid numeric attribute
+ * with fast-search enabled.
+ * Created by balder on 1/21/15.
+ */
+public class ValidateMatchPhaseSearcher extends Searcher {
+    private Set<String> validMatchPhaseAttributes = new HashSet<>();
+    private Set<String> validDiversityAttributes = new HashSet<>();
+    public ValidateMatchPhaseSearcher(AttributesConfig attributesConfig) {
+        for (AttributesConfig.Attribute a : attributesConfig.attribute()) {
+            if (a.fastsearch() &&
+                (a.collectiontype() == AttributesConfig.Attribute.Collectiontype.SINGLE) &&
+                isNumeric(a.datatype()))
+            {
+                validMatchPhaseAttributes.add(a.name());
+            }
+        }
+        for (AttributesConfig.Attribute a : attributesConfig.attribute()) {
+            if ((a.collectiontype() == AttributesConfig.Attribute.Collectiontype.SINGLE) &&
+                ((a.datatype() == AttributesConfig.Attribute.Datatype.STRING) || isNumeric(a.datatype())))
+            {
+                validDiversityAttributes.add(a.name());
+            }
+        }
+    }
+    private boolean isNumeric(AttributesConfig.Attribute.Datatype.Enum dt) {
+        return  dt == AttributesConfig.Attribute.Datatype.DOUBLE ||
+                dt == AttributesConfig.Attribute.Datatype.FLOAT ||
+                dt == AttributesConfig.Attribute.Datatype.INT8 ||
+                dt == AttributesConfig.Attribute.Datatype.INT16 ||
+                dt == AttributesConfig.Attribute.Datatype.INT32 ||
+                dt == AttributesConfig.Attribute.Datatype.INT64;
+    }
+    @Override
+    public Result search(Query query, Execution execution) {
+        ErrorMessage e = validate(query);
+        return (e != null)
+                ? new Result(query, e)
+                : execution.search(query);
+    }
+
+    private ErrorMessage validate(Query query) {
+        String attribute = query.getRanking().getMatchPhase().getAttribute();
+        if ( attribute != null && ! validMatchPhaseAttributes.contains(attribute) ) {
+            return ErrorMessage.createInvalidQueryParameter("The attribute '" + attribute + "' is not available for match-phase. " +
+                                                            "It must be a single value numeric attribute with fast-search.");
+        }
+        attribute = query.getRanking().getMatchPhase().getDiversity().getAttribute();
+        if (attribute != null && ! validDiversityAttributes.contains(attribute)) {
+            return ErrorMessage.createInvalidQueryParameter("The attribute '" + attribute + "' is not available for match-phase diversification. " +
+                    "It must be a single value numeric or string attribute.");
+        }
+        return null;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/package-info.java b/container-search/src/main/java/com/yahoo/search/searchers/package-info.java
new file mode 100644
index 00000000000..78f1e5940a6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Various useful searchers
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.searchers;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;