Improve error handling

- Fix NoClassDefFoundError on shaded/vespa/commons/lang/exception/ExceptionUtils
  The ExceptionUtils from vespajlib would be a good replacement if it was
  client library (JDK 8 compatible). To minimize dependency, simply grab
  the implementation.
- Emit UDF error logs to stderr so it can be examined
- Use different PigWarning to make triage easier

2 files changed, 45 insertions, 6 deletions

diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
index 32cdbf9af5c..57b01085460 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.yahoo.vespa.hadoop.mapreduce.util.TupleTools;
 import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.PigWarning;
 import org.apache.pig.data.DataBag;
@@ -18,6 +17,9 @@ import org.joda.time.DateTime;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.*;
@@ -89,11 +91,11 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             return null;
         }
         if (template == null || template.length() == 0) {
-            warn("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1);
+            warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1);
             return null;
         }
         if (operation == null) {
-            warn("No valid operation found. Skipping.", PigWarning.UDF_WARNING_1);
+            warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2);
             return null;
         }
 
@@ -111,7 +113,7 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             // create json
             json = create(operation, docId, fields, properties, inputSchema);
             if (json == null || json.length() == 0) {
-                warn("No valid document operation could be created.", PigWarning.UDF_WARNING_1);
+                warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3);
                 return null;
             }
 
@@ -121,8 +123,8 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             sb.append("Caught exception processing input row: \n");
             sb.append(tuple.toString());
             sb.append("\nException: ");
-            sb.append(ExceptionUtils.getStackTrace(e));
-            warn(sb.toString(), PigWarning.UDF_WARNING_1);
+            sb.append(getStackTraceAsString(e));
+            warnLog(sb.toString(), PigWarning.UDF_WARNING_4);
             return null;
         }
 
@@ -449,4 +451,20 @@ public class VespaDocumentOperation extends EvalFunc<String> {
         g.writeEndArray();
     }
 
+    // copied from vespajlib for reducing dependency and building with JDK 8
+    private static String getStackTraceAsString(Throwable throwable) {
+        try (StringWriter stringWriter = new StringWriter();
+             PrintWriter printWriter = new PrintWriter(stringWriter, true)) {
+            throwable.printStackTrace(printWriter);
+            return stringWriter.getBuffer().toString();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    // wrapper to emit logs
+    private void warnLog(String msg, PigWarning warning) {
+        warn(msg, warning);
+        System.err.println(msg);
+    }
 }
diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
index 3c6805019b8..c1ff925697d 100644
--- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
+++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
@@ -85,6 +85,27 @@ public class VespaDocumentOperationTest {
 
 
     @Test
+    public void requireThatUDFReturnsNullWhenExceptionHappens() throws IOException {
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+
+        // broken DELTA format that would throw internally
+        Map<String, Double> tensor = new HashMap<String, Double>() {{
+            put("xlabel1", 2.0); // missing : between 'x' and 'label1'
+        }};
+
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+        addToTuple("tensor", DataType.MAP, tensor, schema, tuple);
+
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "create-tensor-fields=tensor");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        assertNull(json);
+    }
+
+
+    @Test
     public void requireThatUDFCorrectlyGeneratesRemoveOperation() throws Exception {
         String json = getDocumentOperationJson("operation=remove", "docid=id:<application>:metrics::<name>-<date>");
         ObjectMapper m = new ObjectMapper();