Merge branch 'master' into partial_update_bag_as_map_and_tensor

2 files changed, 45 insertions, 10 deletions

diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
index 219996ee9aa..94176bbb658 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.yahoo.vespa.hadoop.mapreduce.util.TupleTools;
 import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.PigWarning;
 import org.apache.pig.data.DataBag;
@@ -19,6 +18,9 @@ import org.joda.time.DateTime;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.*;
@@ -125,14 +127,11 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             if (statusReporter != null) {
                 statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1);
             }
-            warn("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1);
+            warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1);
             return null;
         }
         if (operation == null) {
-            if (statusReporter != null) {
-                statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1);
-            }
-            warn("No valid operation found. Skipping.", PigWarning.UDF_WARNING_1);
+            warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2);
             return null;
         }
 
@@ -149,7 +148,7 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             // create json
             json = create(operation, docId, fields, properties, inputSchema);
             if (json == null || json.length() == 0) {
-                warn("No valid document operation could be created.", PigWarning.UDF_WARNING_1);
+                warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3);
                 return null;
             }
 
@@ -162,8 +161,8 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             sb.append("Caught exception processing input row: \n");
             sb.append(tuple.toString());
             sb.append("\nException: ");
-            sb.append(ExceptionUtils.getStackTrace(e));
-            warn(sb.toString(), PigWarning.UDF_WARNING_1);
+            sb.append(getStackTraceAsString(e));
+            warnLog(sb.toString(), PigWarning.UDF_WARNING_4);
             return null;
         }
         if (statusReporter != null) {
@@ -644,4 +643,21 @@ public class VespaDocumentOperation extends EvalFunc<String> {
         }
         g.writeEndArray();
     }
+
+    // copied from vespajlib for reducing dependency and building with JDK 8
+    private static String getStackTraceAsString(Throwable throwable) {
+        try (StringWriter stringWriter = new StringWriter();
+             PrintWriter printWriter = new PrintWriter(stringWriter, true)) {
+            throwable.printStackTrace(printWriter);
+            return stringWriter.getBuffer().toString();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    // wrapper to emit logs
+    private void warnLog(String msg, PigWarning warning) {
+        warn(msg, warning);
+        System.err.println(msg);
+    }
 }
diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
index 72d0a2ec069..67003273cac 100644
--- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
+++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
@@ -86,7 +86,6 @@ public class VespaDocumentOperationTest {
 
     @Test
     public void requireThatUDFCorrectlyGeneratesRemoveBagAsMapOperation() throws Exception {
-
         DataBag bag = BagFactory.getInstance().newDefaultBag();
 
         Schema innerObjectSchema = new Schema();
@@ -249,6 +248,26 @@ public class VespaDocumentOperationTest {
     }
 
     @Test
+    public void requireThatUDFReturnsNullWhenExceptionHappens() throws IOException {
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+
+        // broken DELTA format that would throw internally
+        Map<String, Double> tensor = new HashMap<String, Double>() {{
+            put("xlabel1", 2.0); // missing : between 'x' and 'label1'
+        }};
+
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+        addToTuple("tensor", DataType.MAP, tensor, schema, tuple);
+  
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "create-tensor-fields=tensor");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        assertNull(json);
+    }
+
+    @Test
     public void requireThatUDFCorrectlyGeneratesRemoveOperation() throws Exception {
         String json = getDocumentOperationJson("operation=remove", "docid=id:<application>:metrics::<name>-<date>");
         ObjectMapper m = new ObjectMapper();