summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBjørn Christian Seime <bjorncs@verizonmedia.com>2021-07-08 14:40:46 +0200
committerGitHub <noreply@github.com>2021-07-08 14:40:46 +0200
commitc517b78271ff5018eba68aecf52b5000a92a1d13 (patch)
tree64db5273b4a7b0dfd0b117e484b67b7b9fadd881
parentaf4d2b77b83d76ef90ee52ff1d3fab165fdcfdf2 (diff)
parentd0b7efca909067296a7d5c9c23ca5d6ea1baa053 (diff)
Merge pull request #18571 from vespa-engine/bjorncs/vespa-hadoop
Bjorncs/vespa hadoop
-rw-r--r--vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java2
-rw-r--r--vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java8
2 files changed, 8 insertions, 2 deletions
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java
index 97bc7dc838e..f80c28424f2 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java
@@ -35,7 +35,7 @@ public class VespaOutputFormat extends OutputFormat {
public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
VespaCounters counters = VespaCounters.get(context);
VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride);
- return configuration.useLegacyClient()
+ return configuration.useLegacyClient().orElse(true)
? new LegacyVespaRecordWriter(configuration, counters)
: new VespaRecordWriter(configuration, counters);
}
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java
index 7219e621486..1be794f8e11 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java
@@ -6,6 +6,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.io.StringReader;
+import java.util.Optional;
import java.util.Properties;
public class VespaConfiguration {
@@ -131,7 +132,11 @@ public class VespaConfiguration {
return getInt(PROGRESS_REPORT, 1000);
}
- public boolean useLegacyClient() { return getBoolean(USE_LEGACY_CLIENT, true); }
+ public Optional<Boolean> useLegacyClient() {
+ String raw = getString(USE_LEGACY_CLIENT);
+ if (raw == null || raw.trim().isEmpty()) return Optional.empty();
+ return Optional.of(Boolean.parseBoolean(raw));
+ }
public String getString(String name) {
if (override != null && override.containsKey(name)) {
@@ -191,6 +196,7 @@ public class VespaConfiguration {
sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n");
sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n");
sb.append(NUM_RETRIES + ": " + numRetries() +"\n");
+ sb.append(USE_LEGACY_CLIENT + ": " + useLegacyClient().map(Object::toString).orElse("<empty>") +"\n");
return sb.toString();
}