diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
-rw-r--r-- | container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java | 334 | ||||
-rw-r--r-- | container-search/src/main/java/com/yahoo/search/result/Hit.java | 28 |
2 files changed, 211 insertions, 151 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 6c7018317c3..55c846ccb5b 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -68,6 +68,7 @@ import java.util.function.LongSupplier; * JSON renderer for search results. * * @author Steinar Knutsen + * @author bratseth */ // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc. public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @@ -75,18 +76,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); - private enum RenderDecision { - YES, NO, DO_NOT_KNOW; - - boolean booleanValue() { - switch (this) { - case YES: return true; - case NO: return false; - default: throw new IllegalStateException(); - } - } - } - // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; private static final String BUCKET_TO = "to"; @@ -133,6 +122,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private final JsonFactory generatorFactory; private JsonGenerator generator; + private FieldConsumer fieldConsumer; private Deque<Integer> renderedChildren; private boolean debugRendering; private LongSupplier timeSource; @@ -304,9 +294,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void init() { super.init(); - generator = null; - renderedChildren = null; debugRendering = false; + setGenerator(null, debugRendering); + renderedChildren = null; timeSource = System::currentTimeMillis; stream = null; } @@ -314,9 +304,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); - generator = generatorFactory.createGenerator(stream, JsonEncoding.UTF8); - renderedChildren = new ArrayDeque<>(); debugRendering = getDebugRendering(getResult().getQuery()); + setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); + renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); renderTrace(getExecution().trace()); renderTiming(); @@ -473,17 +463,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return ! (hit instanceof DefaultErrorHit); } - private void fieldsStart(MutableBoolean hasFieldsField) throws IOException { - if (hasFieldsField.get()) return; - generator.writeObjectFieldStart(FIELDS); - hasFieldsField.set(true); - } - - private void fieldsEnd(MutableBoolean hasFieldsField) throws IOException { - if ( ! hasFieldsField.get()) return; - generator.writeEndObject(); - } - private void renderHitContents(Hit hit) throws IOException { String id = hit.getDisplayId(); if (id != null) @@ -509,39 +488,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } private void renderAllFields(Hit hit) throws IOException { - MutableBoolean hasFieldsField = new MutableBoolean(false); - renderTotalHitCount(hit, hasFieldsField); - renderStandardFields(hit, hasFieldsField); - fieldsEnd(hasFieldsField); - } - - private void renderStandardFields(Hit hit, MutableBoolean hasFieldsField) { - hit.forEachField((name, value) -> { - try { - if (shouldRender(name, value)) { - fieldsStart(hasFieldsField); - renderField(name, value); - } - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + fieldConsumer.startHitFields(); + renderTotalHitCount(hit); + renderStandardFields(hit); + fieldConsumer.endHitFields(); } - private boolean shouldRender(String name, Object value) { - if (debugRendering) return true; - - if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; - - if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; - - // StringFieldValue cannot hold a null, so checking length directly is OK: - if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false; - - if (value instanceof NanNumber) return false; - - return true; + private void renderStandardFields(Hit hit) { + hit.forEachFieldAsRaw(fieldConsumer); } private void renderSpecialCasesForGrouping(Hit hit) throws IOException { @@ -606,96 +560,13 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id).getTo()) : id.getTo()).toString(); } - private void renderTotalHitCount(Hit hit, MutableBoolean hasFieldsField) throws IOException { + private void renderTotalHitCount(Hit hit) throws IOException { if ( ! (getRecursionLevel() == 1 && hit instanceof HitGroup)) return; - fieldsStart(hasFieldsField); + fieldConsumer.ensureFieldsField(); generator.writeNumberField(TOTAL_COUNT, getResult().getTotalHitCount()); - } - - private void renderField(String name, Object value) throws IOException { - generator.writeFieldName(name); - renderFieldContents(value); - } - - private void renderFieldContents(Object field) throws IOException { - if (field == null) { - generator.writeNull(); - } else if (field instanceof Number) { - renderNumberField((Number) field); - } else if (field instanceof TreeNode) { - generator.writeTree((TreeNode) field); - } else if (field instanceof Tensor) { - renderTensor(Optional.of((Tensor)field)); - } else if (field instanceof JsonProducer) { - generator.writeRawValue(((JsonProducer) field).toJson()); - } else if (field instanceof Inspectable) { - StringBuilder intermediate = new StringBuilder(); - JsonRender.render((Inspectable) field, intermediate, true); - generator.writeRawValue(intermediate.toString()); - } else if (field instanceof StringFieldValue) { - // This needs special casing as JsonWriter hides empty strings now - generator.writeString(((StringFieldValue)field).getString()); - } else if (field instanceof TensorFieldValue) { - renderTensor(((TensorFieldValue)field).getTensor()); - } else if (field instanceof FieldValue) { - // the null below is the field which has already been written - ((FieldValue) field).serialize(null, new JsonWriter(generator)); - } else if (field instanceof JSONArray || field instanceof JSONObject) { - // org.json returns null if the object would not result in - // syntactically correct JSON - String s = field.toString(); - if (s == null) { - generator.writeNull(); - } else { - generator.writeRawValue(s); - } - } else { - generator.writeString(field.toString()); - } - } - - private void renderNumberField(Number field) throws IOException { - if (field instanceof Integer) { - generator.writeNumber(field.intValue()); - } else if (field instanceof Float) { - generator.writeNumber(field.floatValue()); - } else if (field instanceof Double) { - generator.writeNumber(field.doubleValue()); - } else if (field instanceof Long) { - generator.writeNumber(field.longValue()); - } else if (field instanceof Byte || field instanceof Short) { - generator.writeNumber(field.intValue()); - } else if (field instanceof BigInteger) { - generator.writeNumber((BigInteger) field); - } else if (field instanceof BigDecimal) { - generator.writeNumber((BigDecimal) field); - } else { - generator.writeNumber(field.doubleValue()); - } - } - - private void renderTensor(Optional<Tensor> tensor) throws IOException { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - if (tensor.isPresent()) { - for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { - Tensor.Cell cell = i.next(); - - generator.writeStartObject(); - - generator.writeObjectFieldStart("address"); - for (int d = 0; d < cell.getKey().size(); d++) - generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); - generator.writeEndObject(); - - generator.writeObjectField("value", cell.getValue()); - - generator.writeEndObject(); - } - } - generator.writeEndArray(); - generator.writeEndObject(); + // alternative for the above two lines: + // fieldConsumer.accept(TOTAL_COUNT, getResult().getTotalHitCount()); } @Override @@ -774,11 +645,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return null; } - /** - * Only for testing. Never to be used in any other context. - */ - void setGenerator(JsonGenerator generator) { + private void setGenerator(JsonGenerator generator, boolean debugRendering) { this.generator = generator; + this.fieldConsumer = generator == null ? null : new FieldConsumer(generator, debugRendering); } /** @@ -787,5 +656,170 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { void setTimeSource(LongSupplier timeSource) { this.timeSource = timeSource; } - + + /** + * Received callbacks when fields of hits are encountered. + * This instance is reused for all hits of a Result since we are in a single-threaded context + * and want to limit object creation. + */ + private static class FieldConsumer implements Hit.RawUtf8Consumer { + + private final JsonGenerator generator; + private final boolean debugRendering; + + private MutableBoolean hasFieldsField; + + public FieldConsumer(JsonGenerator generator, boolean debugRendering) { + this.generator = generator; + this.debugRendering = debugRendering; + } + + /** + * Call before using this for a hit to track whether we + * have created the "fields" field of the JSON object + */ + void startHitFields() { + this.hasFieldsField = new MutableBoolean(false); + } + + /** Call before rendering a field to the generator */ + void ensureFieldsField() throws IOException { + if (hasFieldsField.get()) return; + generator.writeObjectFieldStart(FIELDS); + hasFieldsField.set(true); + } + + /** Call after all fields in a hit to close the "fields" field of the JSON object */ + void endHitFields() throws IOException { + if ( ! hasFieldsField.get()) return; + generator.writeEndObject(); + this.hasFieldsField = null; + } + + @Override + public void accept(String name, Object value) { + try { + if (shouldRender(name, value)) { + ensureFieldsField(); + generator.writeFieldName(name); + renderFieldContents(value); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void accept(String name, byte[] utf8Data, int offset, int length) { + try { + if (shouldRenderUtf8Value(name, length)) { + ensureFieldsField(); + generator.writeFieldName(name); + generator.writeUTF8String(utf8Data, offset, length); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private boolean shouldRender(String name, Object value) { + if (debugRendering) return true; + if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; + if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; + // StringFieldValue cannot hold a null, so checking length directly is OK: + if (value instanceof StringFieldValue && ((StringFieldValue) value).getString().isEmpty()) return false; + if (value instanceof NanNumber) return false; + return true; + } + + private boolean shouldRenderUtf8Value(String name, int length) { + if (debugRendering) return true; + if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; + if (length == 0) return false; + return true; + } + + private void renderFieldContents(Object field) throws IOException { + if (field == null) { + generator.writeNull(); + } else if (field instanceof Number) { + renderNumberField((Number) field); + } else if (field instanceof TreeNode) { + generator.writeTree((TreeNode) field); + } else if (field instanceof Tensor) { + renderTensor(Optional.of((Tensor)field)); + } else if (field instanceof JsonProducer) { + generator.writeRawValue(((JsonProducer) field).toJson()); + } else if (field instanceof Inspectable) { + StringBuilder intermediate = new StringBuilder(); + JsonRender.render((Inspectable) field, intermediate, true); + generator.writeRawValue(intermediate.toString()); + } else if (field instanceof StringFieldValue) { + generator.writeString(((StringFieldValue)field).getString()); + } else if (field instanceof TensorFieldValue) { + renderTensor(((TensorFieldValue)field).getTensor()); + } else if (field instanceof FieldValue) { + // the null below is the field which has already been written + ((FieldValue) field).serialize(null, new JsonWriter(generator)); + } else if (field instanceof JSONArray || field instanceof JSONObject) { + // org.json returns null if the object would not result in + // syntactically correct JSON + String s = field.toString(); + if (s == null) { + generator.writeNull(); + } else { + generator.writeRawValue(s); + } + } else { + generator.writeString(field.toString()); + } + } + + private void renderNumberField(Number field) throws IOException { + if (field instanceof Integer) { + generator.writeNumber(field.intValue()); + } else if (field instanceof Float) { + generator.writeNumber(field.floatValue()); + } else if (field instanceof Double) { + generator.writeNumber(field.doubleValue()); + } else if (field instanceof Long) { + generator.writeNumber(field.longValue()); + } else if (field instanceof Byte || field instanceof Short) { + generator.writeNumber(field.intValue()); + } else if (field instanceof BigInteger) { + generator.writeNumber((BigInteger) field); + } else if (field instanceof BigDecimal) { + generator.writeNumber((BigDecimal) field); + } else { + generator.writeNumber(field.doubleValue()); + } + } + + private void renderTensor(Optional<Tensor> tensor) throws IOException { + generator.writeStartObject(); + generator.writeArrayFieldStart("cells"); + if (tensor.isPresent()) { + for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { + Tensor.Cell cell = i.next(); + + generator.writeStartObject(); + + generator.writeObjectFieldStart("address"); + for (int d = 0; d < cell.getKey().size(); d++) + generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); + generator.writeEndObject(); + + generator.writeObjectField("value", cell.getValue()); + + generator.writeEndObject(); + } + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index f68916c8a68..74c31aa33c5 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -404,13 +404,25 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** * Receive a callback on the given object for each field in this hit. - * This is the most resource efficient way of traversing all the fields of a hit. + * This is more efficient than accessing the fields as a map or iterator. */ public void forEachField(BiConsumer<String, Object> consumer) { if (fields == null) return; fields.forEach(consumer); } + /** + * Receive a callback on the given object for each field in this hit, + * where the callback will provide raw utf-8 byte data for strings whose data + * is already available at this form. + * This is the most resource efficient way of traversing all the fields of a hit + * in renderers which produces utf-8. + */ + public void forEachFieldAsRaw(RawUtf8Consumer consumer) { + if (fields == null) return; + fields.forEach(consumer); // No utf-8 fields available in Hit + } + /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ public Map<String, Object> fields() { return getUnmodifiableFieldMap(); } @@ -800,4 +812,18 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return "hit " + getId() + " (relevance " + getRelevance() + ")"; } + public interface RawUtf8Consumer extends BiConsumer<String, Object> { + + /** + * Called for fields which are available as UTF-8 instead of accept(String, Object). + * + * @param fieldName the name of the field + * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data + * @param offset the start index of the data to accept into the utf8Data array + * @param length the length of the data to accept into the utf8Data array + */ + void accept(String fieldName, byte[] utf8Data, int offset, int length); + + } + } |