From 1c0ae314ffc19ad9f7a1e7d2e08b14df329c16ca Mon Sep 17 00:00:00 2001 From: jingtinghuang Date: Wed, 8 Apr 2020 10:54:23 +0800 Subject: add property verbose --- .../vespa/hadoop/pig/VespaDocumentOperation.java | 15 ++++------ .../hadoop/pig/VespaDocumentOperationTest.java | 33 ++++++++++++++++++++-- 2 files changed, 36 insertions(+), 12 deletions(-) (limited to 'vespa-hadoop/src') diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java index f433ec9a35e..d0d9790c07e 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java @@ -67,7 +67,7 @@ public class VespaDocumentOperation extends EvalFunc { private static final String PROPERTY_CREATE_IF_NON_EXISTENT = "create-if-non-existent"; private static final String PROPERTY_ID_TEMPLATE = "docid"; private static final String PROPERTY_OPERATION = "operation"; - private static final String PROPERTY_PRINT_DOCID_WHEN_ERROR = "print-docid-when-error"; + private static final String PROPERTY_VERBOSE = "verbose"; private static final String BAG_AS_MAP_FIELDS = "bag-as-map-fields"; private static final String SIMPLE_ARRAY_FIELDS = "simple-array-fields"; private static final String SIMPLE_OBJECT_FIELDS = "simple-object-fields"; @@ -81,7 +81,7 @@ public class VespaDocumentOperation extends EvalFunc { private static final String PARTIAL_UPDATE_ASSIGN = "assign"; private static final String PARTIAL_UPDATE_ADD = "add"; private static final String PARTIAL_UPDATE_REMOVE = "remove"; - private static boolean printIdOnError; + private static boolean printId; private static Map mapPartialOperationMap; @@ -115,7 +115,7 @@ public class VespaDocumentOperation extends EvalFunc { properties = VespaConfiguration.loadProperties(params); template = properties.getProperty(PROPERTY_ID_TEMPLATE); operation = Operation.fromString(properties.getProperty(PROPERTY_OPERATION, "put")); - printIdOnError = Boolean.parseBoolean(properties.getProperty(PROPERTY_PRINT_DOCID_WHEN_ERROR, "false")); + printId = Boolean.parseBoolean(properties.getProperty(PROPERTY_VERBOSE, "false")); } @Override @@ -148,6 +148,9 @@ public class VespaDocumentOperation extends EvalFunc { Schema inputSchema = getInputSchema(); Map fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); + if (printId) { + System.out.println("Processing docId: "+ docId); + } // create json json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { @@ -160,12 +163,6 @@ public class VespaDocumentOperation extends EvalFunc { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } - if (printIdOnError) { - Schema inputSchema = getInputSchema(); - Map fields = TupleTools.tupleMap(inputSchema, tuple); - String docId = TupleTools.toString(fields, template); - System.out.println("Error occur when processing document with docID: " +docId); - } StringBuilder sb = new StringBuilder(); sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java index 31f68807736..e278026b00d 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java @@ -558,7 +558,7 @@ public class VespaDocumentOperationTest { } @Test - public void requireThatUDFPrintIdWhenError() throws IOException { + public void requireThatUDFPrintIdWhenVerbose() throws IOException { DataBag bag = BagFactory.getInstance().newDefaultBag(); Schema objectSchema = new Schema(); @@ -577,11 +577,38 @@ public class VespaDocumentOperationTest { Tuple tuple = TupleFactory.getInstance().newTuple(); addToTuple("bag", DataType.BAG, bag, objectSchema, schema, tuple); - VespaDocumentOperation docOp = new VespaDocumentOperation("docid=7654321", "simple-object-fields=bag","print-docid-when-error=true"); + VespaDocumentOperation docOp = new VespaDocumentOperation("docid=7654321", "bag-as-map-fields=bag","verbose=true"); docOp.setInputSchema(schema); String json = docOp.exec(tuple); - assertThat(outContent.toString(), CoreMatchers.containsString("Error occur when processing document with docID: 7654321")); + assertThat(outContent.toString(), CoreMatchers.containsString("Processing docId: 7654321")); + } + + @Test + public void requireThatUDFVerboseSetToFalseByDefault() throws IOException { + DataBag bag = BagFactory.getInstance().newDefaultBag(); + + Schema objectSchema = new Schema(); + Tuple objectTuple = TupleFactory.getInstance().newTuple(); + addToTuple("key", DataType.CHARARRAY, "123456", objectSchema, objectTuple); + addToTuple("value", DataType.INTEGER, 123456, objectSchema, objectTuple); + bag.add(objectTuple); + + objectSchema = new Schema(); + objectTuple = TupleFactory.getInstance().newTuple(); + addToTuple("key", DataType.CHARARRAY, "234567", objectSchema, objectTuple); + addToTuple("value", DataType.INTEGER, 234567, objectSchema, objectTuple); + bag.add(objectTuple); + + Schema schema = new Schema(); + Tuple tuple = TupleFactory.getInstance().newTuple(); + addToTuple("bag", DataType.BAG, bag, objectSchema, schema, tuple); + + VespaDocumentOperation docOp = new VespaDocumentOperation("docid=7654321", "bag-as-map-fields=bag"); + docOp.setInputSchema(schema); + String json = docOp.exec(tuple); + + assertEquals("", outContent.toString()); } private void addToTuple(String alias, byte type, Object value, Schema schema, Tuple tuple) { -- cgit v1.2.3