From 59e1468eb5fefed6983c479c12fe8397db770860 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 21 Jun 2019 13:58:19 +0200 Subject: Nonfunctional changes only --- .../src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java | 1 - .../src/main/java/com/yahoo/search/rendering/Renderer.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 3f45e8e8f00..5130cf7ff34 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -145,7 +145,6 @@ public class ClusterSearcher extends Searcher { documentDbConfig); addBackendSearcher(searcher); } else { - System.out.println("Dispatchers: " + searchClusterConfig.dispatcher().size()); for (int dispatcherIndex = 0; dispatcherIndex < searchClusterConfig.dispatcher().size(); dispatcherIndex++) { try { if ( ! isRemote(searchClusterConfig.dispatcher(dispatcherIndex).host())) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java b/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java index c9b890e64f5..a5b51e60861 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java @@ -40,7 +40,7 @@ abstract public class Renderer extends com.yahoo.processing.rendering.Renderer render(OutputStream stream, Result response, Execution execution, Request request) { Writer writer = null; try { - writer = createWriter(stream,response); + writer = createWriter(stream, response); render(writer, response); } catch (IOException e) { @@ -50,7 +50,7 @@ abstract public class Renderer extends com.yahoo.processing.rendering.Renderer completed=SettableFuture.create(); + SettableFuture completed = SettableFuture.create(); completed.set(true); return completed; } -- cgit v1.2.3 From 488f61ee81794912357b5b1ea97af69df7da0d9b Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 27 Jun 2019 05:28:49 +0200 Subject: Add test --- .../java/com/yahoo/search/result/FeatureData.java | 31 +++++++++--- .../prelude/fastsearch/SlimeSummaryTestCase.java | 57 +++++++++++++++++++--- .../java/com/yahoo/prelude/fastsearch/summary.cfg | 4 +- .../search/rendering/JsonRendererTestCase.java | 2 +- 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 7e5d6b12f30..1fd8f6e7e17 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -18,7 +18,7 @@ import java.util.Set; /** * A wrapper for structured data representing feature values: A map of floats and tensors. - * This class is not thread safe even when it is only consumed. + * This class is immutable but not thread safe. */ public class FeatureData implements Inspectable, JsonProducer { @@ -26,6 +26,8 @@ public class FeatureData implements Inspectable, JsonProducer { private Set featureNames = null; + private String jsonForm = null; + public FeatureData(Inspector value) { this.value = value; } @@ -38,15 +40,12 @@ public class FeatureData implements Inspectable, JsonProducer { @Override public Inspector inspect() { return value; } - @Override - public String toString() { - if (value.type() == Type.EMPTY) return ""; - return toJson(); - } - @Override public String toJson() { - return writeJson(new StringBuilder()).toString(); + if (jsonForm != null) return jsonForm; + + jsonForm = writeJson(new StringBuilder()).toString(); + return jsonForm; } @Override @@ -95,6 +94,22 @@ public class FeatureData implements Inspectable, JsonProducer { return featureNames; } + @Override + public String toString() { + if (value.type() == Type.EMPTY) return ""; + return toJson(); + } + + @Override + public int hashCode() { return toJson().hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof FeatureData)) return false; + return ((FeatureData)other).toJson().equals(this.toJson()); + } + /** A JSON encoder which encodes DATA as a tensor */ private static class Encoder extends JsonRender.StringEncoder { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index ad4d0cf82e5..e6cc3ac9e54 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -3,9 +3,11 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableSet; import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; @@ -17,9 +19,11 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -62,6 +66,7 @@ public class SlimeSummaryTestCase { assertNull(hit.getField("jsonstring_field")); assertNull(hit.getField("tensor_field1")); assertNull(hit.getField("tensor_field2")); + assertNull(hit.getField("summaryfeatures")); } @Test @@ -75,7 +80,7 @@ public class SlimeSummaryTestCase { @Test public void testDecoding() { Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); - Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}"); + Tensor tensor2 = Tensor.from("tensor(x[1],y[1]):{{x:0,y:0}:-0.3}"); DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); assertNull(docsum.lazyDecode("default", fullSummary(tensor1, tensor2), hit)); @@ -111,6 +116,12 @@ public class SlimeSummaryTestCase { } assertEquals(tensor1, hit.getField("tensor_field1")); assertEquals(tensor2, hit.getField("tensor_field2")); + FeatureData featureData = (FeatureData)hit.getField("summaryfeatures"); + assertEquals("double_feature,tensor1_feature,tensor2_feature", + featureData.featureNames().stream().sorted().collect(Collectors.joining(","))); + assertEquals(0.5, featureData.getDouble("double_feature"), 0.00000001); + assertEquals(tensor1, featureData.getTensor("tensor1_feature")); + assertEquals(tensor2, featureData.getTensor("tensor2_feature")); } @Test @@ -238,7 +249,9 @@ public class SlimeSummaryTestCase { assertFields(expected, hit); // --- Add full summary - assertNull(fullDocsum.lazyDecode("default", fullishSummary(), hit)); + Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}"); + Tensor tensor2 = Tensor.from("tensor(x[1],y[1]):{{x:0,y:0}:-0.3}"); + assertNull(fullDocsum.lazyDecode("default", fullishSummary(tensor1, tensor2), hit)); expected.put("integer_field", 4); expected.put("short_field", (short)2); expected.put("byte_field", (byte)1); @@ -247,7 +260,15 @@ public class SlimeSummaryTestCase { expected.put("int64_field", 8L); expected.put("string_field", "string_value"); expected.put("longstring_field", "longstring_value"); - assertFields(expected, hit); + expected.put("tensor_field1", tensor1); + expected.put("tensor_field2", tensor2); + + Slime slime = new Slime(); + Cursor summaryFeatures = slime.setObject(); + summaryFeatures.setDouble("double_feature", 0.5); + summaryFeatures.setData("tensor1_feature", TypedBinaryFormat.encode(tensor1)); + summaryFeatures.setData("tensor2_feature", TypedBinaryFormat.encode(tensor2)); + expected.put("summaryfeatures", new FeatureData(new SlimeAdapter(slime.get()))); hit.removeField("string_field"); hit.removeField("integer_field"); @@ -272,7 +293,7 @@ public class SlimeSummaryTestCase { fail("Multiple callbacks for " + name); traversed.put(name, value); }); - assertEquals(expected, traversed); + assertEqualMaps(expected, traversed); // raw utf8 field traverser Map traversedUtf8 = new HashMap<>(); hit.forEachFieldAsRaw(new Utf8FieldTraverser(traversedUtf8)); @@ -288,7 +309,7 @@ public class SlimeSummaryTestCase { // fieldKeys assertEquals(expected.keySet(), hit.fieldKeys()); // fields - assertEquals(expected, hit.fields()); + assertEqualMaps(expected, hit.fields()); // fieldIterator int fieldIteratorFieldCount = 0; for (Iterator> i = hit.fieldIterator(); i.hasNext(); ) { @@ -302,6 +323,15 @@ public class SlimeSummaryTestCase { assertEquals(field.getValue(), hit.getField(field.getKey())); } + private void assertEqualMaps(Map expected, Map actual) { + assertEquals("Map sizes", expected.size(), actual.size()); + assertEquals("Keys", expected.keySet(), actual.keySet()); + for (var expectedEntry : expected.entrySet()) { + assertEquals("Key '" + expectedEntry.getKey() + "'", + expectedEntry.getValue(), actual.get(expectedEntry.getKey())); + } + } + private byte[] emptySummary() { Slime slime = new Slime(); slime.setObject(); @@ -339,7 +369,7 @@ public class SlimeSummaryTestCase { return encode((slime)); } - private byte[] fullishSummary() { + private byte[] fullishSummary(Tensor tensor1, Tensor tensor2) { Slime slime = new Slime(); Cursor docsum = slime.setObject(); docsum.setLong("integer_field", 4); @@ -352,6 +382,7 @@ public class SlimeSummaryTestCase { //docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8)); docsum.setString("longstring_field", "longstring_value"); //docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8)); + addTensors(tensor1, tensor2, docsum); return encode((slime)); } @@ -374,11 +405,23 @@ public class SlimeSummaryTestCase { field.setLong("foo", 1); field.setLong("bar", 2); } + + addTensors(tensor1, tensor2, docsum); + return encode((slime)); + } + + private void addTensors(Tensor tensor1, Tensor tensor2, Cursor docsum) { if (tensor1 != null) docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1)); if (tensor2 != null) docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2)); - return encode((slime)); + + if (tensor1 !=null && tensor2 != null) { + Cursor summaryFeatures = docsum.setObject("summaryfeatures"); + summaryFeatures.setDouble("double_feature", 0.5); + summaryFeatures.setData("tensor1_feature", TypedBinaryFormat.encode(tensor1)); + summaryFeatures.setData("tensor2_feature", TypedBinaryFormat.encode(tensor2)); + } } private byte[] encode(Slime slime) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg index e46904b17d0..e074eadcbc2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg @@ -3,7 +3,7 @@ documentdb[0].name test documentdb[0].summaryclass[1] documentdb[0].summaryclass[0].name default documentdb[0].summaryclass[0].id 0 -documentdb[0].summaryclass[0].fields[14] +documentdb[0].summaryclass[0].fields[15] documentdb[0].summaryclass[0].fields[0].name integer_field documentdb[0].summaryclass[0].fields[0].type integer documentdb[0].summaryclass[0].fields[1].name short_field @@ -32,3 +32,5 @@ documentdb[0].summaryclass[0].fields[12].name tensor_field1 documentdb[0].summaryclass[0].fields[12].type tensor documentdb[0].summaryclass[0].fields[13].name tensor_field2 documentdb[0].summaryclass[0].fields[13].type tensor +documentdb[0].summaryclass[0].fields[14].name summaryfeatures +documentdb[0].summaryclass[0].fields[14].type featuredata diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 9fb2e627e9c..16f7b0d7a57 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -123,7 +123,7 @@ public class JsonRendererTestCase { } @Test - public void testDataTypes() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testDataTypes() throws IOException, InterruptedException, ExecutionException { String expected = "{" + " \"root\": {" + " \"children\": [" -- cgit v1.2.3 From a6f5bf09c0aeb0d4678d57c4d3fe0045d5155c53 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 27 Jun 2019 07:06:57 +0200 Subject: Override rendering of FeatureData json --- .../com/yahoo/search/rendering/JsonRenderer.java | 9 ++++-- .../search/rendering/JsonRendererTestCase.java | 32 +++++++++++++++++----- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 2f241f9c7a3..4bcb48447db 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -18,6 +18,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.JsonWriter; import com.yahoo.lang.MutableBoolean; +import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; @@ -39,6 +40,7 @@ import com.yahoo.search.result.Coverage; import com.yahoo.search.result.DefaultErrorHit; import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; @@ -781,7 +783,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer { } private void renderFieldContents(Object field) throws IOException { - if (field instanceof Inspectable) { + if (field instanceof Inspectable && ! (field instanceof FeatureData)) { renderInspector(((Inspectable)field).inspect()); } else { renderFieldContentsDirect(field); @@ -799,6 +801,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer { generator.writeTree((TreeNode) field); } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); + } else if (field instanceof FeatureData) { + generator.writeRawValue(((FeatureData)field).toJson()); } else if (field instanceof Inspectable) { renderInspectorDirect(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { @@ -811,8 +815,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer { // the null below is the field which has already been written ((FieldValue) field).serialize(null, new JsonWriter(generator)); } else if (field instanceof JSONArray || field instanceof JSONObject) { - // org.json returns null if the object would not result in - // syntactically correct JSON + // org.json returns null if the object would not result in syntactically correct JSON String s = field.toString(); if (s == null) { generator.writeNull(); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 16f7b0d7a57..a245d61bafb 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -36,6 +36,7 @@ import com.yahoo.search.grouping.result.RootGroup; import com.yahoo.search.grouping.result.StringId; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; @@ -51,6 +52,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import com.yahoo.yolean.trace.TraceNode; import org.json.JSONArray; @@ -139,7 +141,13 @@ public class JsonRendererTestCase { + " \"predicate\": \"a in [b]\"," + " \"tensor1\": { \"cells\": [ { \"address\": {\"x\": \"a\"}, \"value\":2.0 } ] }," + " \"tensor2\": { \"cells\": [] }," - + " \"tensor3\": { \"cells\": [ { \"address\": {\"x\": \"a\", \"y\": \"0\"}, \"value\":2.0 }, { \"address\": {\"x\": \"a\", \"y\": \"1\"}, \"value\":-1.0 } ] }" + + " \"tensor3\": { \"cells\": [ { \"address\": {\"x\": \"a\", \"y\": \"0\"}, \"value\":2.0 }, { \"address\": {\"x\": \"a\", \"y\": \"1\"}, \"value\":-1.0 } ] }," + + " \"summaryfeatures\": {" + + " \"scalar1\":1.5," + + " \"scalar2\":2.5," + + " \"tensor1\":{\"type\":\"tensor(x[3])\",\"cells\":[{\"address\":{\"x\":\"0\"},\"value\":1.5},{\"address\":{\"x\":\"1\"},\"value\":2.0},{\"address\":{\"x\":\"2\"},\"value\":2.5}]}," + + " \"tensor2\":{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":0.5}]}" + + " }" + " }," + " \"id\": \"datatypestuff\"," + " \"relevance\": 1.0" @@ -166,12 +174,24 @@ public class JsonRendererTestCase { h.setField("tensor2", new TensorFieldValue(TensorType.empty)); h.setField("tensor3", Tensor.from("{ {x:a, y:0}: 2.0, {x:a, y:1}: -1 }")); h.setField("object", new Thingie()); + h.setField("summaryfeatures", createSummaryFeatures()); r.hits().add(h); r.setTotalHitCount(1L); String summary = render(r); assertEqualJson(expected, summary); } + private FeatureData createSummaryFeatures() { + Slime slime = new Slime(); + Cursor features = slime.setObject(); + features.setDouble("scalar1", 1.5); + features.setDouble("scalar2", 2.5); + Tensor tensor1 = Tensor.from("tensor(x[3]):[1.5, 2, 2.5]"); + features.setData("tensor1", TypedBinaryFormat.encode(tensor1)); + Tensor tensor2 = Tensor.from(0.5); + features.setData("tensor2", TypedBinaryFormat.encode(tensor2)); + return new FeatureData(new SlimeAdapter(slime.get())); + } @Test public void testTracing() throws IOException, InterruptedException, ExecutionException { @@ -679,12 +699,10 @@ public class JsonRendererTestCase { + "}"; Result r = newEmptyResult(); Hit h = new Hit("moredatatypestuff"); - h.setField("byte", Byte.valueOf((byte) 8)); - h.setField("short", Short.valueOf((short) 16)); - h.setField("bigInteger", new BigInteger( - "340282366920938463463374607431768211455")); - h.setField("bigDecimal", new BigDecimal( - "340282366920938463463374607431768211456.5")); + h.setField("byte", (byte)8); + h.setField("short", (short)16); + h.setField("bigInteger", new BigInteger("340282366920938463463374607431768211455")); + h.setField("bigDecimal", new BigDecimal("340282366920938463463374607431768211456.5")); h.setField("nanNumber", NanNumber.NaN); r.hits().add(h); r.setTotalHitCount(1L); -- cgit v1.2.3 From a86c4b38eb673e46c55684388ea6580f50353206 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 27 Jun 2019 09:32:03 +0200 Subject: Update ABI spec --- container-search/abi-spec.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 06713d14d88..8f9dc33e944 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -7146,12 +7146,14 @@ "methods": [ "public void (com.yahoo.data.access.Inspector)", "public com.yahoo.data.access.Inspector inspect()", - "public java.lang.String toString()", "public java.lang.String toJson()", "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)", "public java.lang.Double getDouble(java.lang.String)", "public com.yahoo.tensor.Tensor getTensor(java.lang.String)", - "public java.util.Set featureNames()" + "public java.util.Set featureNames()", + "public java.lang.String toString()", + "public int hashCode()", + "public boolean equals(java.lang.Object)" ], "fields": [] }, -- cgit v1.2.3