diff options
author | Rong-En Fan <rafan@verizonmedia.com> | 2020-04-01 17:30:03 +0800 |
---|---|---|
committer | Rong-En Fan <rafan@verizonmedia.com> | 2020-04-01 17:32:28 +0800 |
commit | f4c622836569b015fff25fc2adb2826134d60a1d (patch) | |
tree | 9dce8e0df30a6ff490f98e128212b93acd3f6de5 /vespa-hadoop/src | |
parent | ede7a4ac09e169813087b05d8b98818332b6578c (diff) |
Improve error handling
- Fix NoClassDefFoundError on shaded/vespa/commons/lang/exception/ExceptionUtils
The ExceptionUtils from vespajlib would be a good replacement if it was
client library (JDK 8 compatible). To minimize dependency, simply grab
the implementation.
- Emit UDF error logs to stderr so it can be examined
- Use different PigWarning to make triage easier
Diffstat (limited to 'vespa-hadoop/src')
-rw-r--r-- | vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java | 30 | ||||
-rw-r--r-- | vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java | 21 |
2 files changed, 45 insertions, 6 deletions
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java index 32cdbf9af5c..57b01085460 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.vespa.hadoop.mapreduce.util.TupleTools; import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; -import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.pig.EvalFunc; import org.apache.pig.PigWarning; import org.apache.pig.data.DataBag; @@ -18,6 +17,9 @@ import org.joda.time.DateTime; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.UncheckedIOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.*; @@ -89,11 +91,11 @@ public class VespaDocumentOperation extends EvalFunc<String> { return null; } if (template == null || template.length() == 0) { - warn("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); + warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { - warn("No valid operation found. Skipping.", PigWarning.UDF_WARNING_1); + warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } @@ -111,7 +113,7 @@ public class VespaDocumentOperation extends EvalFunc<String> { // create json json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { - warn("No valid document operation could be created.", PigWarning.UDF_WARNING_1); + warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } @@ -121,8 +123,8 @@ public class VespaDocumentOperation extends EvalFunc<String> { sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); - sb.append(ExceptionUtils.getStackTrace(e)); - warn(sb.toString(), PigWarning.UDF_WARNING_1); + sb.append(getStackTraceAsString(e)); + warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } @@ -449,4 +451,20 @@ public class VespaDocumentOperation extends EvalFunc<String> { g.writeEndArray(); } + // copied from vespajlib for reducing dependency and building with JDK 8 + private static String getStackTraceAsString(Throwable throwable) { + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true)) { + throwable.printStackTrace(printWriter); + return stringWriter.getBuffer().toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + // wrapper to emit logs + private void warnLog(String msg, PigWarning warning) { + warn(msg, warning); + System.err.println(msg); + } } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java index 3c6805019b8..c1ff925697d 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java @@ -85,6 +85,27 @@ public class VespaDocumentOperationTest { @Test + public void requireThatUDFReturnsNullWhenExceptionHappens() throws IOException { + Schema schema = new Schema(); + Tuple tuple = TupleFactory.getInstance().newTuple(); + + // broken DELTA format that would throw internally + Map<String, Double> tensor = new HashMap<String, Double>() {{ + put("xlabel1", 2.0); // missing : between 'x' and 'label1' + }}; + + addToTuple("id", DataType.CHARARRAY, "123", schema, tuple); + addToTuple("tensor", DataType.MAP, tensor, schema, tuple); + + VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "create-tensor-fields=tensor"); + docOp.setInputSchema(schema); + String json = docOp.exec(tuple); + + assertNull(json); + } + + + @Test public void requireThatUDFCorrectlyGeneratesRemoveOperation() throws Exception { String json = getDocumentOperationJson("operation=remove", "docid=id:<application>:metrics::<name>-<date>"); ObjectMapper m = new ObjectMapper(); |