diff options
author | Jo Kristian Bergum <bergum@yahoo-inc.com> | 2017-03-22 11:51:12 +0100 |
---|---|---|
committer | Jo Kristian Bergum <bergum@yahoo-inc.com> | 2017-03-22 11:51:12 +0100 |
commit | 3e9a1ac431f8b0b86eb8d0e63b91238f61662d95 (patch) | |
tree | 622a6f3382939cc432d049786835d9b97eeba7cf /vespa-hadoop | |
parent | 8b892a928597adfe9b9b6ba1ff4b78146843d848 (diff) |
Set a higher client queue timeout and reduce client queue, with many mappers/reducers each gets a large 30K queue and timeout in queue is handled as a docment failure.
Diffstat (limited to 'vespa-hadoop')
-rw-r--r-- | vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 8072e99e2ab..e23daee5318 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -126,11 +126,14 @@ public class VespaRecordWriter extends RecordWriter { feedParamsBuilder.setRoute(configuration.route()); feedParamsBuilder.setMaxSleepTimeMs(configuration.maxSleepTimeMs()); feedParamsBuilder.setMaxInFlightRequests(configuration.maxInFlightRequests()); + feedParamsBuilder.setLocalQueueTimeOut(3600*1000); //1 hour queue timeout SessionParams.Builder sessionParams = new SessionParams.Builder(); sessionParams.setThrottlerMinSize(configuration.throttlerMinSize()); sessionParams.setConnectionParams(connParamsBuilder.build()); sessionParams.setFeedParams(feedParamsBuilder.build()); + sessionParams.setClientQueueSize(configuration.maxInFlightRequests()*2); + String endpoints = configuration.endpoint(); StringTokenizer tokenizer = new StringTokenizer(endpoints, ","); |