diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /container-search/src/main/java/com/yahoo/search/searchers |
Publish
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search/searchers')
6 files changed, 683 insertions, 0 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java new file mode 100644 index 00000000000..064e38d91fc --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.searchchain.Execution; + +/** + * Searcher that sets cache control HTTP headers in response based on query/GET parameters to + * control caching done by proxy/caches such as YSquid and YTS: + * <ul> + * <li>max-age=XXX - set with &cachecontrol.maxage parameter + * <li>stale-while-revalidate=YYY - set with &cachecontrol.staleage + * <li>no-cache - if Vespa &noCache or &cachecontrol.nocache parameter is set to true + * </ul> + * + * <p>This is controlled through the three query parameters <code>cachecontrol.maxage</code>, + * <code>cachecontrol.staleage</code> and <code>cachecontrol.nocache</code>, with the obvious meanings.</p> + * + * Example: + * <ul> + * <li>Request: "?query=foo&cachecontrol.maxage=60&cachecontrol.staleage=3600" + * <li>Response HTTP header: "Cache-Control: max-age=60, revalidate-while-stale=3600" + * </ul> + * + * Further documentation on use of Cache-Control headers: + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + * + * @author Frode Lundgren + */ +public class CacheControlSearcher extends Searcher { + + private static final CompoundName cachecontrolNocache=new CompoundName("cachecontrol.nocache"); + private static final CompoundName cachecontrolMaxage=new CompoundName("cachecontrol.maxage"); + private static final CompoundName cachecontrolStaleage=new CompoundName("cachecontrol.staleage"); + + public static final String CACHE_CONTROL_HEADER = "Cache-Control"; + + @Override + public Result search(Query query, Execution execution) { + query.trace("CacheControlSearcher: Running version $Revision$", false, 6); + Result result = execution.search(query); + query = result.getQuery(); + + if (result.getHeaders(true) == null) { + query.trace("CacheControlSearcher: No HTTP header map available - skipping searcher.", false, 5); + return result; + } + + // If you specify no-cache, no further cache control headers make sense + if (query.properties().getBoolean(cachecontrolNocache, false) || query.getNoCache()) { + result.getHeaders(true).put(CACHE_CONTROL_HEADER, "no-cache"); + query.trace("CacheControlSearcher: Added no-cache header", false, 4); + return result; + } + + // Handle max-age header + int maxage = query.properties().getInteger(cachecontrolMaxage, -1); + if (maxage > 0) { + result.getHeaders(true).put(CACHE_CONTROL_HEADER, "max-age=" + maxage); + query.trace("CacheControlSearcher: Set max-age header to " + maxage, false, 4); + } + + // Handle stale-while-revalidate header + int staleage = query.properties().getInteger(cachecontrolStaleage, -1); + if (staleage > 0) { + result.getHeaders(true).put(CACHE_CONTROL_HEADER, "stale-while-revalidate=" + staleage); + query.trace("CacheControlSearcher: Set stale-while-revalidate header to " + maxage, false, 4); + } + + return result; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java new file mode 100644 index 00000000000..cdbf864f7fd --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; + +/** + * Searcher which can enforce HTTP connection close based on query properties. + * + * <p> + * This searcher informs the client to close a persistent HTTP connection if the + * connection is older than the configured max lifetime. This is done by adding + * the "Connection" HTTP header with the value "Close" to the result. + * </p> + * + * <p> + * The searcher reads the query property "connectioncontrol.maxlifetime", which + * is an integer number of seconds, to get the value for maximum connection + * lifetime. Setting it to zero will enforce connection close independently of + * the age of the connection. Typical usage would be as follows: + * </p> + * + * <ol> + * <li>Add the ConnectionControlSearcher to the default search chain of your + * application. (It has no special ordering considerations.)</li> + * + * <li>For the default query profile of your application, set a reasonable value + * for "connectioncontrol.maxlifetime". The definition of reasonable will be + * highly application dependent, but it should always be less than the grace + * period when taking the container out of production traffic.</li> + * + * <li>Deploy application. The container will now inform clients to close + * connections/reconnect within the configured time limit. + * </ol> + * + * @author frodelu + * @author Steinar Knutsen + */ +public class ConnectionControlSearcher extends Searcher { + + private final String simpleName = this.getClass().getSimpleName(); + + private final LongSupplier clock; + + private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = new CompoundName("connectioncontrol.maxlifetime"); + private static final String HTTP_CONNECTION_HEADER_NAME = "Connection"; + private static final String HTTP_CONNECTION_CLOSE_ARGUMENT = "Close"; + + public ConnectionControlSearcher() { + this(() -> System.currentTimeMillis()); + } + + private ConnectionControlSearcher(LongSupplier clock) { + this.clock = clock; + } + + /** + * Create a searcher instance suitable for unit tests. + * + * @param clock a simulated or real clock behaving similarly to System.currentTimeMillis() + * @return a fully initialised instance + */ + public static ConnectionControlSearcher createTestInstance(LongSupplier clock) { + return new ConnectionControlSearcher(clock); + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + + query.trace(false, 3, simpleName, " updating headers."); + keepAliveProcessing(query, result); + return result; + } + + /** + * If the HTTP connection has been alive for too long, set the header + * "Connection: Close" to tell the client to close the connection after this + * request. + */ + private void keepAliveProcessing(Query query, Result result) { + int maxLifetimeSeconds = query.properties().getInteger(KEEPALIVE_MAXLIFETIMESECONDS, -1); + + if (maxLifetimeSeconds < 0) { + return; + } else if (maxLifetimeSeconds == 0) { + result.getHeaders(true).put(HTTP_CONNECTION_HEADER_NAME, HTTP_CONNECTION_CLOSE_ARGUMENT); + query.trace(false, 5, simpleName, ": Max HTTP connection lifetime set to 0; adding \"", HTTP_CONNECTION_HEADER_NAME, + ": ", HTTP_CONNECTION_CLOSE_ARGUMENT, "\" header"); + } else { + setCloseIfLifetimeExceeded(query, result, maxLifetimeSeconds); + } + } + + private void setCloseIfLifetimeExceeded(Query query, Result result, int maxLifetimeSeconds) { + final HttpRequest httpRequest = query.getHttpRequest(); + if (httpRequest == null) { + query.trace(false, 5, simpleName, " got max lifetime = ", maxLifetimeSeconds, + ", but got no JDisc request. Setting no header."); + return; + } + + final long connectedAtMillis = httpRequest.getJDiscRequest().getConnectedAt(TimeUnit.MILLISECONDS); + final long maxLifeTimeMillis = maxLifetimeSeconds * 1000L; + if (connectedAtMillis + maxLifeTimeMillis < clock.getAsLong()) { + result.getHeaders(true).put(HTTP_CONNECTION_HEADER_NAME, HTTP_CONNECTION_CLOSE_ARGUMENT); + query.trace(false, 5, simpleName, ": Max HTTP connection lifetime (", maxLifetimeSeconds, ") exceeded; adding \"", + HTTP_CONNECTION_HEADER_NAME, ": ", HTTP_CONNECTION_CLOSE_ARGUMENT, "\" header"); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java new file mode 100644 index 00000000000..d99cb72f5a3 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -0,0 +1,191 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.ListIterator; +import java.util.Map; +import java.util.logging.Logger; + +import com.yahoo.log.LogLevel; +import com.yahoo.metrics.simple.Counter; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.TermItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; + +/** + * Check whether the query tree seems to be "well formed". In other words, run heurestics against + * the input data to see whether the query should sent to the search backend. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class InputCheckingSearcher extends Searcher { + + private final Counter utfRejections; + private final Counter repeatedConsecutiveTermsInPhraseRejections; + private final Counter repeatedTermsInPhraseRejections; + private static final Logger log = Logger.getLogger(InputCheckingSearcher.class.getName()); + private final int MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE = 5; + private final int MAX_REPEATED_TERMS_IN_PHRASE=10; + + public InputCheckingSearcher(MetricReceiver metrics) { + utfRejections = metrics.declareCounter("double_encoded_utf8_rejections"); + repeatedTermsInPhraseRejections = metrics.declareCounter("repeated_terms_in_phrase_rejections"); + repeatedConsecutiveTermsInPhraseRejections = metrics.declareCounter("repeated_consecutive_terms_in_phrase_rejections"); + } + + @Override + public Result search(Query query, Execution execution) { + try { + checkQuery(query); + } catch (IllegalArgumentException e) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); + } + return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); + } + return execution.search(query); + } + + private void checkQuery(Query query) { + doubleEncodedUtf8(query); + checkPhrases(query.getModel().getQueryTree().getRoot()); + // add new heuristics here + } + + private void checkPhrases(Item queryItem) { + if (queryItem instanceof PhraseItem) { + PhraseItem phrase = (PhraseItem) queryItem; + repeatedConsecutiveTermsInPhraseCheck(phrase); + repeatedTermsInPhraseCheck(phrase); + } else if (queryItem instanceof CompositeItem) { + CompositeItem asComposite = (CompositeItem) queryItem; + for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { + checkPhrases(i.next()); + } + } + } + + private void repeatedConsecutiveTermsInPhraseCheck(PhraseItem phrase) { + if (phrase.getItemCount() > MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) { + String prev = null; + int repeatedCount = 0; + for (int i = 0; i < phrase.getItemCount(); ++i) { + Item item = phrase.getItem(i); + if (item instanceof TermItem) { + TermItem term = (TermItem) item; + String current = term.getIndexedString(); + if (prev != null) { + if (prev.equals(current)) { + repeatedCount++; + if (repeatedCount >= MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) { + repeatedConsecutiveTermsInPhraseRejections.add(); + throw new IllegalArgumentException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE + + " ocurrences of term '" + current + "' in a row detected in phrase : " + phrase.toString()); + } + } else { + repeatedCount = 0; + } + } + prev = current; + } else { + prev = null; + repeatedCount = 0; + } + } + } + } + private static final class Count { + private int v; + Count(int initial) { v = initial; } + void inc() { v++; } + int get() { return v; } + } + private void repeatedTermsInPhraseCheck(PhraseItem phrase) { + if (phrase.getItemCount() > MAX_REPEATED_TERMS_IN_PHRASE) { + Map<String, Count> repeatedCount = new HashMap<>(); + for (int i = 0; i < phrase.getItemCount(); ++i) { + Item item = phrase.getItem(i); + if (item instanceof TermItem) { + TermItem term = (TermItem) item; + String current = term.getIndexedString(); + Count count = repeatedCount.get(current); + if (count != null) { + if (count.get() >= MAX_REPEATED_TERMS_IN_PHRASE) { + repeatedTermsInPhraseRejections.add(); + throw new IllegalArgumentException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE + + " occurrences of term '" + current + "' in phrase : " + phrase.toString()); + } + count.inc(); + } else { + repeatedCount.put(current, new Count(1)); + } + } + } + } + } + + + private void doubleEncodedUtf8(Query query) { + int singleCharacterTerms = countSingleCharacterUserTerms(query.getModel().getQueryTree()); + if (singleCharacterTerms <= 4) { + return; + } + String userInput = query.getModel().getQueryString(); + ByteBuffer asOctets = ByteBuffer.allocate(userInput.length()); + boolean asciiOnly = true; + for (int i = 0; i < userInput.length(); ++i) { + char c = userInput.charAt(i); + if (c > 255) { + return; // not double (or more) encoded + } + if (c > 127) { + asciiOnly = false; + } + asOctets.put((byte) c); + } + if (asciiOnly) { + return; + } + asOctets.flip(); + CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + // OK, unmappable character is sort of theoretical, but added to be explicit + try { + decoder.decode(asOctets); + } catch (CharacterCodingException e) { + return; + } + utfRejections.add(); + throw new IllegalArgumentException("The user input has been determined to be double encoded UTF-8." + + " Please investigate whether this is a false positive."); + } + + private int countSingleCharacterUserTerms(Item queryItem) { + if (queryItem instanceof CompositeItem) { + int sum = 0; + CompositeItem asComposite = (CompositeItem) queryItem; + for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { + sum += countSingleCharacterUserTerms(i.next()); + } + return sum; + } else if (queryItem instanceof WordItem) { + WordItem word = (WordItem) queryItem; + return (word.isFromQuery() && word.stringValue().length() == 1) ? 1 : 0; + } else { + return 0; + } + } +} diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java new file mode 100755 index 00000000000..95cec1d0960 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java @@ -0,0 +1,219 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.jdisc.Metric; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.RateLimitingConfig; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.chain.Provides; + +import java.time.Clock; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; + +/** + * A simple rate limiter. + * <p> + * This takes these query parameter arguments: + * <ul> + * <li>rate.id - (String) the id of the client from rate limiting perspective + * <li>rate.cost - (Double) the cost Double of this query. This is read after executing the query and hence can be set + * by downstream searchers inspecting the result to allow differencing the cost of various queries. Default is 1. + * <li>rate.quota - (Double) the cost per second a particular id is allowed to consume in this system. + * <li>rate.idDimension - (String) the name of the rate-id dimension used when logging metrics. + * If this is not specified, the metric will be logged without dimensions. + * <li>rate.dryRun - (Boolean) emit metrics on rejected requests but don't actually reject them + * </ul> + * <p> + * Whenever quota is exceeded for an id this searcher will reject queries from that id by + * returning a result containing a status 429 error. + * <p> + * If rate.id or rate.quota is not set in Query.properties this searcher will do nothing. + * <p> + * Metrics: This will emit the count metric requestsOverQuota with the dimension [rate.idDimension=rate.id] + * counting rejected requests. + * <p> + * Ordering: This searcher Provides rateLimiting + * + * @author bratseth + */ +@Provides(RateLimitingSearcher.RATE_LIMITING) +public class RateLimitingSearcher extends Searcher { + + /** Constant containing the name this Provides - "rateLimiting", for ordering constraints */ + public static final String RATE_LIMITING = "rateLimiting"; + + public static final CompoundName idKey = new CompoundName("rate.id"); + public static final CompoundName costKey = new CompoundName("rate.cost"); + public static final CompoundName quotaKey = new CompoundName("rate.quota"); + public static final CompoundName idDimensionKey = new CompoundName("rate.idDimension"); + public static final CompoundName dryRunKey = new CompoundName("rate.dryRun"); + + private static final String requestsOverQuotaMetricName = "requestsOverQuota"; + + /** Used to divide quota by nodes. Assumption: All nodes get the same share of traffic. */ + private final int nodeCount; + + /** Shared capacity across all threads. Each thread will ask for more capacity from here when they run out. */ + private final AvailableCapacity availableCapacity; + + /** Capacity already allocated to this thread */ + private final ThreadLocal<Map<String, Double>> allocatedCapacity = new ThreadLocal<>(); + + /** For emitting metrics */ + private final Metric metric; + + /** + * How much capacity to allocate to a thread each time it runs out. + * A higher value means less contention and less accuracy. + */ + private final double capacityIncrement; + + /** How often to check for new capacity if we have run out */ + private final double recheckForCapacityProbability; + + @Inject + public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, Metric metric) { + this(rateLimitingConfig, clusterInfoConfig, metric, Clock.systemUTC()); + } + + /** For testing - allows injection of a timer to avoid depending on the system clock */ + public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, Metric metric, Clock clock) { + this.capacityIncrement = rateLimitingConfig.capacityIncrement(); + this.recheckForCapacityProbability = rateLimitingConfig.recheckForCapacityProbability(); + this.availableCapacity = new AvailableCapacity(rateLimitingConfig.maxAvailableCapacity(), clock); + + this.nodeCount = clusterInfoConfig.nodeCount(); + + this.metric = metric; + } + + @Override + public Result search(Query query, Execution execution) { + String id = query.properties().getString(idKey); + Double rate = query.properties().getDouble(quotaKey); + if (id == null || rate == null) { + query.trace(false, 6, "Skipping rate limiting check. Need both " + idKey + " and " + quotaKey + " set"); + return execution.search(query); + } + + rate = rate / nodeCount; + + if (allocatedCapacity.get() == null) // new thread + allocatedCapacity.set(new HashMap<>()); + if (allocatedCapacity.get().get(id) == null) // new id in this thread + requestCapacity(id, rate); + + // Check if there is capacity available. Cannot check for exact cost as it may be computed after execution + // no capacity means we're over rate. Only recheck occasionally to limit synchronization. + if (getAllocatedCapacity(id) <= 0 && ThreadLocalRandom.current().nextDouble() < recheckForCapacityProbability) { + requestCapacity(id, rate); + } + + if (rate==0 || getAllocatedCapacity(id) <= 0) { // we are still over rate: reject + metric.add(requestsOverQuotaMetricName, 1, createContext(query.properties().getString(idDimensionKey, ""), id)); + if ( ! query.properties().getBoolean(dryRunKey, false)) + return new Result(query, new ErrorMessage(429, "Too many requests", "Allowed rate: " + rate + "/s")); + } + + Result result = execution.search(query); + addAllocatedCapacity(id, - query.properties().getDouble(costKey, 1.0)); + + if (getAllocatedCapacity(id) <= 0) // make sure we ask for more with 100% probability when first running out + requestCapacity(id, rate); + + return result; + } + + private Metric.Context createContext(String dimensionName, String dimensionValue) { + if (dimensionName.isEmpty()) + return metric.createContext(Collections.emptyMap()); + return metric.createContext(Collections.singletonMap(dimensionName, dimensionValue)); + } + + private double getAllocatedCapacity(String id) { + Double value = allocatedCapacity.get().get(id); + if (value == null) return 0; + return value; + } + + private void addAllocatedCapacity(String id, double newCapacity) { + Double capacity = allocatedCapacity.get().get(id); + if (capacity != null) + newCapacity += capacity; + allocatedCapacity.get().put(id, newCapacity); + } + + private void requestCapacity(String id, double rate) { + double minimumRequested = Math.max(0, -getAllocatedCapacity(id)); // If we are below, make sure we reach 0 + double preferredRequested = Math.max(capacityIncrement, -getAllocatedCapacity(id)); + addAllocatedCapacity(id, availableCapacity.request(id, minimumRequested, preferredRequested, rate)); + } + + /** + * This keeps track of the current "capacity" (total cost) available to each client (rate id) + * across all threads. Capacity is supplied at the rate per second given by the clients quota. + * When all the capacity is spent, no further capacity will be handed out, leading to request rejection. + * Capacity has a max value it will never exceed to avoid clients saving capacity for future overspending. + */ + private static class AvailableCapacity { + + private final double maxAvailableCapacity; + private final Clock clock; + + private final Map<String, CapacityAllocation> available = new HashMap<>(); + + public AvailableCapacity(double maxAvailableCapacity, Clock clock) { + this.maxAvailableCapacity = maxAvailableCapacity; + this.clock = clock; + } + + /** Returns an amount of capacity between 0 and the requested amount based on availability for this id */ + public synchronized double request(String id, double minimumRequested, double preferredRequested, double rate) { + CapacityAllocation allocation = available.get(id); + if (allocation == null) { + allocation = new CapacityAllocation(rate, clock); + available.put(id, allocation); + } + return allocation.request(minimumRequested, preferredRequested, rate, maxAvailableCapacity); + } + + } + + private static class CapacityAllocation { + + private double capacity; + private final Clock clock; + private long lastAllocatedTime; + + public CapacityAllocation(double initialCapacity, Clock clock) { + this.capacity = initialCapacity; + this.clock = clock; + lastAllocatedTime = clock.millis(); + } + + public double request(double minimumRequested, double preferredRequested, double rate, double maxAvailableCapacity) { + if ( preferredRequested > capacity) { // attempt to allocate more + // rate is per second so we get rate/1000 per millisecond + long currentTime = clock.millis(); + capacity += Math.min(maxAvailableCapacity, rate/1000d * (Math.max(0, currentTime - lastAllocatedTime))); + lastAllocatedTime = currentTime; + } + double grantedCapacity = Math.min(capacity/10, preferredRequested); // /10 to avoid stealing all capacity when low + if (grantedCapacity < minimumRequested) + grantedCapacity = Math.min(minimumRequested, capacity); + capacity = capacity - grantedCapacity; + return grantedCapacity; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java new file mode 100644 index 00000000000..ff00c8edb9b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespa.config.search.AttributesConfig; + +import java.util.HashSet; +import java.util.Set; + +/** + * Validates that the attribute given as match-phase override is actually a valid numeric attribute + * with fast-search enabled. + * Created by balder on 1/21/15. + */ +public class ValidateMatchPhaseSearcher extends Searcher { + private Set<String> validMatchPhaseAttributes = new HashSet<>(); + private Set<String> validDiversityAttributes = new HashSet<>(); + public ValidateMatchPhaseSearcher(AttributesConfig attributesConfig) { + for (AttributesConfig.Attribute a : attributesConfig.attribute()) { + if (a.fastsearch() && + (a.collectiontype() == AttributesConfig.Attribute.Collectiontype.SINGLE) && + isNumeric(a.datatype())) + { + validMatchPhaseAttributes.add(a.name()); + } + } + for (AttributesConfig.Attribute a : attributesConfig.attribute()) { + if ((a.collectiontype() == AttributesConfig.Attribute.Collectiontype.SINGLE) && + ((a.datatype() == AttributesConfig.Attribute.Datatype.STRING) || isNumeric(a.datatype()))) + { + validDiversityAttributes.add(a.name()); + } + } + } + private boolean isNumeric(AttributesConfig.Attribute.Datatype.Enum dt) { + return dt == AttributesConfig.Attribute.Datatype.DOUBLE || + dt == AttributesConfig.Attribute.Datatype.FLOAT || + dt == AttributesConfig.Attribute.Datatype.INT8 || + dt == AttributesConfig.Attribute.Datatype.INT16 || + dt == AttributesConfig.Attribute.Datatype.INT32 || + dt == AttributesConfig.Attribute.Datatype.INT64; + } + @Override + public Result search(Query query, Execution execution) { + ErrorMessage e = validate(query); + return (e != null) + ? new Result(query, e) + : execution.search(query); + } + + private ErrorMessage validate(Query query) { + String attribute = query.getRanking().getMatchPhase().getAttribute(); + if ( attribute != null && ! validMatchPhaseAttributes.contains(attribute) ) { + return ErrorMessage.createInvalidQueryParameter("The attribute '" + attribute + "' is not available for match-phase. " + + "It must be a single value numeric attribute with fast-search."); + } + attribute = query.getRanking().getMatchPhase().getDiversity().getAttribute(); + if (attribute != null && ! validDiversityAttributes.contains(attribute)) { + return ErrorMessage.createInvalidQueryParameter("The attribute '" + attribute + "' is not available for match-phase diversification. " + + "It must be a single value numeric or string attribute."); + } + return null; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/searchers/package-info.java b/container-search/src/main/java/com/yahoo/search/searchers/package-info.java new file mode 100644 index 00000000000..78f1e5940a6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Various useful searchers + */ +@ExportPackage +@PublicApi +package com.yahoo.search.searchers; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; |