summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGeir Storli <geirstorli@yahoo.no>2017-02-23 13:53:09 +0100
committerGitHub <noreply@github.com>2017-02-23 13:53:09 +0100
commitbc717833cccf027b017b2d901ffa9f0bf27eccdc (patch)
tree5e01663f0d4b0af0a024015ebde419ad03c29f9f
parenteca5ec7b8cee9dcc3f4f1bac7c7bb1cea76e9a51 (diff)
parent8b797b2efe94f0d7b84e5594a29b83e7a10b1d8d (diff)
Merge pull request #1844 from yahoo/arnej/remove-nonslime-summary-code
Arnej/remove nonslime summary code
-rw-r--r--container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java7
-rw-r--r--container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java1
-rw-r--r--container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java15
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java7
-rw-r--r--searchcore/src/tests/proton/docsummary/docsummary.cpp103
-rw-r--r--searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp33
-rw-r--r--searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp17
-rw-r--r--searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h5
-rw-r--r--searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/common/transport.h2
-rw-r--r--searchsummary/src/tests/docsummary/positionsdfw_test.cpp12
-rw-r--r--searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp4
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp162
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp116
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h15
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h3
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp209
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h9
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp17
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp81
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h2
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h5
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp102
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h4
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp38
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h5
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp44
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h5
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp192
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h2
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp38
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h5
-rw-r--r--streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp4
-rw-r--r--streamingvisitors/src/vespa/searchvisitor/searchvisitor.h1
-rw-r--r--vsm/src/tests/docsum/docsum.cpp92
-rw-r--r--vsm/src/vespa/vsm/vsm/CMakeLists.txt1
-rw-r--r--vsm/src/vespa/vsm/vsm/docsumfilter.cpp41
-rw-r--r--vsm/src/vespa/vsm/vsm/docsumfilter.h5
-rw-r--r--vsm/src/vespa/vsm/vsm/jsondocsumwriter.cpp174
-rw-r--r--vsm/src/vespa/vsm/vsm/jsondocsumwriter.h51
-rw-r--r--vsm/src/vespa/vsm/vsm/slimefieldwriter.h6
42 files changed, 192 insertions, 1452 deletions
diff --git a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java
index b5476ed6ba3..105c1d49392 100644
--- a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java
+++ b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java
@@ -69,7 +69,9 @@ public class GetDocSumsPacket extends Packet {
* definition of enum getdocsums_flags
*/
public static final int GDFLAG_IGNORE_ROW = 0x00000001;
- public static final int GDFLAG_ALLOW_SLIME = 0x00000002;
+
+ // TODO: now always assumed true; remove in Vespa 7
+ public static final int GDFLAG_ALLOW_SLIME_NOTUSED = 0x00000002;
public void encodeBody(ByteBuffer buffer) {
setFieldsFromHits();
@@ -80,9 +82,6 @@ public class GetDocSumsPacket extends Packet {
query.getRanking().getProperties().put(sessionIdKey, query.getSessionId(false).asUtf8String().toString());
}
- // always allow slime docsums
- flags |= GDFLAG_ALLOW_SLIME;
-
// set the default features
long features = GDF_MLD;
if (sendQuery)
diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java
index 9fe64e6977d..ca2bd9c4b05 100644
--- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java
+++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java
@@ -181,7 +181,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor {
}
params.setLibraryParameter("summarycount", String.valueOf(query.getOffset() + query.getHits()));
params.setLibraryParameter("rankprofile", query.getRanking().getProfile());
- params.setLibraryParameter("allowslimedocsums", "true");
params.setLibraryParameter("queryflags", String.valueOf(getQueryFlags(query)));
ByteBuffer buf = ByteBuffer.allocate(1024);
diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java
index 76323b7139d..a5c538d235a 100644
--- a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java
+++ b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java
@@ -37,13 +37,13 @@ public class GetDocSumsPacketTestCase {
hit.setIgnoreRowBits(true);
assertPacket(true, hit, new byte[] { 0, 0, 0, 57, 0, 0, 0, -37, 0, 0, 40, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE,
IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7,
- 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 3 });
+ 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 1 });
hit = new FastHit();
hit.setIgnoreRowBits(false);
- assertPacket(true, hit, new byte[] {0, 0, 0, 57, 0, 0, 0, -37, 0, 0, 40, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE,
+ assertPacket(true, hit, new byte[] {0, 0, 0, 53, 0, 0, 0, -37, 0, 0, 8, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE,
IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7,
- 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111, 0, 0, 0, 2});
+ 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111});
}
@Test
@@ -52,13 +52,13 @@ public class GetDocSumsPacketTestCase {
hit.setIgnoreRowBits(true);
assertPacket(false, hit, new byte[] { 0, 0, 0, 43, 0, 0, 0, -37, 0, 0, 40, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE,
IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7,
- 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 3
+ 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1
});
hit = new FastHit();
hit.setIgnoreRowBits(false);
- assertPacket(false, hit, new byte[] { 0, 0, 0, 43, 0, 0, 0, -37, 0, 0, 40, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE,
- IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 2
+ assertPacket(false, hit, new byte[] { 0, 0, 0, 39, 0, 0, 0, -37, 0, 0, 8, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE,
+ IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x01, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116
});
}
@@ -68,6 +68,7 @@ public class GetDocSumsPacketTestCase {
result.getQuery().getSessionId(true); // create session id.
result.getQuery().getRanking().setQueryCache(true);
FastHit hit = new FastHit();
+ hit.setIgnoreRowBits(true);
result.hits().add(hit);
assertPacket(false, result, new byte[] { 0, 0, 0, -123, 0, 0, 0, -37, 0, 0, 56, 17, 0, 0, 0, 0,
// query timeout
@@ -84,7 +85,7 @@ public class GetDocSumsPacketTestCase {
// caches: features => true
0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e',
// flags
- 0, 0, 0, 2
+ 0, 0, 0, 1
});
}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
index 9787cf2838f..194de116baf 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
@@ -262,7 +262,7 @@ public class FastSearcherTestCase {
buf.get(actual);
byte IGNORE = 69;
- byte[] expected = new byte[] { 0, 0, 0, -85, 0, 0, 0, -37, 0, 0, 48, 17, 0, 0, 0, 0,
+ byte[] expected = new byte[] { 0, 0, 0, -89, 0, 0, 0, -37, 0, 0, 16, 17, 0, 0, 0, 0,
// query timeout
IGNORE, IGNORE, IGNORE, IGNORE,
// "default" - rank profile
@@ -275,9 +275,8 @@ public class FastSearcherTestCase {
// match: documentdb.searchdoctype => test
0, 0, 0, 5, 'm', 'a', 't', 'c', 'h', 0, 0, 0, 1, 0, 0, 0, 24, 'd', 'o', 'c', 'u', 'm', 'e', 'n', 't', 'd', 'b', '.', 's', 'e', 'a', 'r', 'c', 'h', 'd', 'o', 'c', 't', 'y', 'p', 'e', 0, 0, 0, 4, 't', 'e', 's', 't',
// sessionId => qrserver.0.XXXXXXXXXXXXX.0
- 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e',
- // flags
- 0, 0, 0, 2
+ 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e'
+ // no flags
};
assertEquals(expected.length, actual.length);
for (int i = 0; i < expected.length; ++i) {
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index bf44e62f3ef..c5ba2b3b04e 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -128,6 +128,26 @@ Tensor::UP createTensor(const TensorCells &cells,
return TensorFactory::create(cells, dimensions, builder);
}
+vespalib::string asVstring(vespalib::Memory str) {
+ return vespalib::string(str.data, str.size);
+}
+vespalib::string asVstring(const vespalib::slime::Inspector &value) {
+ return asVstring(value.asString());
+}
+
+void decode(const ResEntry *entry, vespalib::Slime &slime) {
+ vespalib::Memory mem(entry->_dataval,
+ entry->_datalen);
+ size_t decodeRes = vespalib::slime::BinaryFormat::decode(mem, slime);
+ ASSERT_EQUAL(decodeRes, mem.size);
+}
+
+std::string b64encode(const vespalib::slime::Inspector &value) {
+ vespalib::Memory mem = value.asData();
+ std::string str(mem.data, mem.size);
+ return vespalib::Base64::encode(str);
+}
+
} // namespace
@@ -357,7 +377,7 @@ public:
GeneralResultPtr
Test::getResult(DocumentStoreAdapter & dsa, uint32_t docId)
{
- DocsumStoreValue docsum = dsa.getMappedDocsum(docId, false);
+ DocsumStoreValue docsum = dsa.getMappedDocsum(docId);
ASSERT_TRUE(docsum.pt() != NULL);
GeneralResultPtr retval(new GeneralResult(dsa.getResultClass(),
0, 0, 0));
@@ -419,7 +439,7 @@ Test::assertTensor(const Tensor::UP & exp, const std::string & fieldName,
ASSERT_EQUAL(::search::fs4transport::SLIME_MAGIC_ID, classId);
vespalib::Slime slime;
vespalib::Memory serialized(docsum.data.c_str() + sizeof(classId),
- docsum.data.size() - sizeof(classId));
+ docsum.data.size() - sizeof(classId));
size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized,
slime);
ASSERT_EQUAL(decodeRes, serialized.size);
@@ -553,7 +573,7 @@ Test::requireThatAdapterHandlesMultipleDocuments()
EXPECT_EQUAL(2000u, res->GetEntry("a")->_intval);
}
{ // doc 2
- DocsumStoreValue docsum = dsa.getMappedDocsum(2, false);
+ DocsumStoreValue docsum = dsa.getMappedDocsum(2);
EXPECT_TRUE(docsum.pt() == NULL);
}
{ // doc 0 (again)
@@ -634,7 +654,6 @@ Test::requireThatDocsumRequestIsProcessed()
DocsumRequest req;
req.resultClassName = "class1";
- req._flags = ::search::fs4transport::GDFLAG_ALLOW_SLIME;
req.hits.push_back(DocsumRequest::Hit(gid2));
req.hits.push_back(DocsumRequest::Hit(gid4));
req.hits.push_back(DocsumRequest::Hit(gid9));
@@ -643,10 +662,10 @@ Test::requireThatDocsumRequestIsProcessed()
EXPECT_EQUAL(3u, rep->docsums.size());
EXPECT_EQUAL(2u, rep->docsums[0].docid);
EXPECT_EQUAL(gid2, rep->docsums[0].gid);
- EXPECT_TRUE(assertSlime("{a:20}", *rep, 0, true));
+ EXPECT_TRUE(assertSlime("{a:20}", *rep, 0, false));
EXPECT_EQUAL(4u, rep->docsums[1].docid);
EXPECT_EQUAL(gid4, rep->docsums[1].gid);
- EXPECT_TRUE(assertSlime("{a:40}", *rep, 1, true));
+ EXPECT_TRUE(assertSlime("{a:40}", *rep, 1, false));
EXPECT_EQUAL(search::endDocId, rep->docsums[2].docid);
EXPECT_EQUAL(gid9, rep->docsums[2].gid);
EXPECT_TRUE(rep->docsums[2].data.get() == NULL);
@@ -674,11 +693,10 @@ Test::requireThatRewritersAreUsed()
DocsumRequest req;
req.resultClassName = "class2";
- req._flags = ::search::fs4transport::GDFLAG_ALLOW_SLIME;
req.hits.push_back(DocsumRequest::Hit(gid1));
DocsumReply::UP rep = dc._ddb->getDocsums(req);
EXPECT_EQUAL(1u, rep->docsums.size());
- EXPECT_TRUE(assertSlime("{aa:20}", *rep, 0, true));
+ EXPECT_TRUE(assertSlime("{aa:20}", *rep, 0, false));
}
@@ -782,7 +800,6 @@ Test::requireThatAttributesAreUsed()
DocsumRequest req;
req.resultClassName = "class3";
- req._flags = ::search::fs4transport::GDFLAG_ALLOW_SLIME;
req.hits.push_back(DocsumRequest::Hit(gid2));
req.hits.push_back(DocsumRequest::Hit(gid3));
DocsumReply::UP rep = dc._ddb->getDocsums(req);
@@ -808,7 +825,7 @@ Test::requireThatAttributesAreUsed()
"bf:[],"
"bg:[],"
"bh:[],"
- "bi:[]}", *rep, 1, true));
+ "bi:[]}", *rep, 1, false));
TEST_DO(assertTensor(Tensor::UP(), "bj", *rep, 1, rclass));
proton::IAttributeManager::SP attributeManager =
@@ -833,7 +850,6 @@ Test::requireThatAttributesAreUsed()
DocsumRequest req3;
req3.resultClassName = "class3";
- req3._flags = ::search::fs4transport::GDFLAG_ALLOW_SLIME;
req3.hits.push_back(DocsumRequest::Hit(gid3));
DocsumReply::UP rep3 = dc._ddb->getDocsums(req3);
@@ -1080,17 +1096,28 @@ Test::requireThatUrisAreUsed()
DocumentStoreAdapter dsa(store, *bc._repo, getResultConfig(), "class0",
bc.createFieldCacheRepo(getResultConfig())->getFieldCache("class0"),
getMarkupFields());
+
EXPECT_TRUE(assertString("http://www.yahoo.com:81/fluke?ab=2#4",
"urisingle", dsa, 1));
- EXPECT_TRUE(assertString("[\"http://www.yahoo.com:82/fluke?ab=2#8\","
- "\"http://www.flickr.com:82/fluke?ab=2#9\"]",
- "uriarray", dsa, 1));
- EXPECT_TRUE(assertString("["
- "{\"item\":\"http://www.yahoo.com:83/fluke?ab=2#12\",\"weight\":4}"
- ","
- "{\"item\":\"http://www.flickr.com:85/fluke?ab=2#13\",\"weight\":7}"
- "]",
- "uriwset", dsa, 1));
+ GeneralResultPtr res = getResult(dsa, 1);
+ {
+ vespalib::Slime slime;
+ decode(res->GetEntry("uriarray"), slime);
+ EXPECT_TRUE(slime.get().valid());
+ EXPECT_EQUAL("http://www.yahoo.com:82/fluke?ab=2#8", asVstring(slime.get()[0]));
+ EXPECT_EQUAL("http://www.flickr.com:82/fluke?ab=2#9", asVstring(slime.get()[1]));
+ }
+ {
+ vespalib::Slime slime;
+ decode(res->GetEntry("uriwset"), slime);
+ EXPECT_TRUE(slime.get().valid());
+ EXPECT_EQUAL(4L, slime.get()[0]["weight"].asLong());
+ EXPECT_EQUAL(7L, slime.get()[1]["weight"].asLong());
+ vespalib::string arr0s = asVstring(slime.get()[0]["item"]);
+ vespalib::string arr1s = asVstring(slime.get()[1]["item"]);
+ EXPECT_EQUAL("http://www.yahoo.com:83/fluke?ab=2#12", arr0s);
+ EXPECT_EQUAL("http://www.flickr.com:85/fluke?ab=2#13", arr1s);
+ }
}
@@ -1127,7 +1154,6 @@ Test::requireThatPositionsAreUsed()
DocsumRequest req;
req.resultClassName = "class5";
- req._flags = ::search::fs4transport::GDFLAG_ALLOW_SLIME;
req.hits.push_back(DocsumRequest::Hit(gid1));
DocsumReply::UP rep = dc._ddb->getDocsums(req);
// uint32_t rclass = 5;
@@ -1142,7 +1168,8 @@ Test::requireThatPositionsAreUsed()
"<position x=\"1008\" y=\"1009\" latlong=\"N0.001009;E0.001008\" />'"
",wp2:[{item:1048370,weight:43},{item:1048382,weight:44}]"
",wp2x:'<position x=\"1012\" y=\"1013\" latlong=\"N0.001013;E0.001012\" />"
- "<position x=\"1014\" y=\"1015\" latlong=\"N0.001015;E0.001014\" />'}", *rep, 0, true));
+ "<position x=\"1014\" y=\"1015\" latlong=\"N0.001015;E0.001014\" />'}",
+ *rep, 0, false));
}
@@ -1211,18 +1238,26 @@ Test::requireThatRawFieldsWorks()
ASSERT_TRUE(assertString(raw1s,
"i", dsa, 1));
- ASSERT_TRUE(assertString(empty + "[\"" +
- vespalib::Base64::encode(raw1a0) +
- "\",\"" +
- vespalib::Base64::encode(raw1a1) +
- "\"]",
- "araw", dsa, 1));
- ASSERT_TRUE(assertString(empty + "[{\"item\":\"" +
- vespalib::Base64::encode(raw1w1) +
- "\",\"weight\":46},{\"item\":\"" +
- vespalib::Base64::encode(raw1w0) +
- "\",\"weight\":45}]",
- "wraw", dsa, 1));
+
+ GeneralResultPtr res = getResult(dsa, 1);
+ {
+ vespalib::Slime slime;
+ decode(res->GetEntry("araw"), slime);
+ EXPECT_TRUE(slime.get().valid());
+ EXPECT_EQUAL(vespalib::Base64::encode(raw1a0), b64encode(slime.get()[0]));
+ EXPECT_EQUAL(vespalib::Base64::encode(raw1a1), b64encode(slime.get()[1]));
+ }
+ {
+ vespalib::Slime slime;
+ decode(res->GetEntry("wraw"), slime);
+ EXPECT_TRUE(slime.get().valid());
+ EXPECT_EQUAL(46L, slime.get()[0]["weight"].asLong());
+ EXPECT_EQUAL(45L, slime.get()[1]["weight"].asLong());
+ std::string arr0s = b64encode(slime.get()[0]["item"]);
+ std::string arr1s = b64encode(slime.get()[1]["item"]);
+ EXPECT_EQUAL(vespalib::Base64::encode(raw1w1), arr0s);
+ EXPECT_EQUAL(vespalib::Base64::encode(raw1w0), arr1s);
+ }
}
diff --git a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
index 113113af195..5159f5b6061 100644
--- a/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
+++ b/searchcore/src/tests/proton/docsummary/summaryfieldconverter_test.cpp
@@ -432,7 +432,7 @@ T
Test::cvtSummaryAs(bool markup, const FieldValue::UP &fv)
{
ASSERT_TRUE(fv.get() != NULL);
- FieldValue::UP r = SFC::convertSummaryField(markup, *fv, false);
+ FieldValue::UP r = SFC::convertSummaryField(markup, *fv);
return cvtValueAs<T>(r);
}
@@ -487,8 +487,7 @@ void Test::requireThatSummaryIsAnUnmodifiedString() {
setSummaryField("string");
Document summary = makeDocument();
checkString("Foo Bar Baz", SFC::convertSummaryField(false,
- *summary.getValue("string"),
- false).get());
+ *summary.getValue("string")).get());
}
void Test::requireThatAttributeIsAnUnmodifiedString() {
@@ -502,28 +501,18 @@ void Test::requireThatArrayIsFlattenedInSummaryField() {
setSummaryField("string_array");
Document summary = makeDocument();
FieldBlock expect("[\"\\\"foO\\\"\",\"ba\\\\R\"]");
- checkString(expect.json,
- SFC::convertSummaryField(false,
- *summary.getValue("string_array"),
- false).get());
checkData(expect.binary,
SFC::convertSummaryField(false,
- *summary.getValue("string_array"),
- true).get());
+ *summary.getValue("string_array")).get());
}
void Test::requireThatWeightedSetIsFlattenedInSummaryField() {
setSummaryField("string_wset");
Document summary = makeDocument();
FieldBlock expect("[{\"item\":\"\\\"foo\\\"\",\"weight\":2},{\"item\":\"ba\\\\r\",\"weight\":4}]");
- checkString(expect.json,
- SFC::convertSummaryField(false,
- *summary.getValue("string_wset"),
- false).get());
checkData(expect.binary,
SFC::convertSummaryField(false,
- *summary.getValue("string_wset"),
- true).get());
+ *summary.getValue("string_wset")).get());
}
void Test::requireThatPositionsAreTransformedInSummary() {
@@ -672,7 +661,7 @@ Test::requireThatPredicateIsPrinted()
doc.setValue("predicate", PredicateFieldValue(std::move(input)));
checkString("'foo' in ['bar']\n",
- SFC::convertSummaryField(false, *doc.getValue("predicate"), false).get());
+ SFC::convertSummaryField(false, *doc.getValue("predicate")).get());
}
@@ -695,21 +684,17 @@ Test::requireThatTensorIsNotConverted()
TEST_CALL(checkTensor(createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} },
{"x", "y"}),
SFC::convertSummaryField(false,
- *doc.getValue("tensor"),
- true).get()));
+ *doc.getValue("tensor")).get()));
doc.setValue("tensor", TensorFieldValue());
TEST_CALL(checkTensor(Tensor::UP(),
SFC::convertSummaryField(false,
- *doc.getValue("tensor"),
- true).get()));
+ *doc.getValue("tensor")).get()));
}
void Test::checkStringForAllConversions(const string& expected, const FieldValue* fv) {
ASSERT_TRUE(fv != nullptr);
- for (bool use_slime : {true, false}) {
- checkString(expected, SFC::convertSummaryField(false, *fv, use_slime).get());
- }
+ checkString(expected, SFC::convertSummaryField(false, *fv).get());
}
const ReferenceDataType& Test::getAsRefType(const string& name) const {
@@ -751,7 +736,7 @@ void Test::requireThatReferenceInCompositeTypeEmitsSlimeData() {
FieldBlock expect(R"({"inner_ref":"id:ns:target_dummy_document::foo"})");
checkData(expect.binary,
- SFC::convertSummaryField(false, *doc.getValue("nested"), true).get());
+ SFC::convertSummaryField(false, *doc.getValue("nested")).get());
}
} // namespace
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
index 508dafe4d0a..85bbb1cff75 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
@@ -67,19 +67,14 @@ DocsumContext::createReply()
uint32_t docId = _docsumState._docsumbuf[i];
reply->docsums[i].docid = docId;
if (docId != search::endDocId && !rci.mustSkip) {
- if ((_docsumState._args.getFlags() & ::search::fs4transport::GDFLAG_ALLOW_SLIME) != 0) {
- Slime slime(Slime::Params(std::move(symbols)));
- vespalib::slime::SlimeInserter inserter(slime);
- _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, slime, inserter);
- uint32_t docsumLen = (slime.get().type().getId() != NIX::ID)
+ Slime slime(Slime::Params(std::move(symbols)));
+ vespalib::slime::SlimeInserter inserter(slime);
+ _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, slime, inserter);
+ uint32_t docsumLen = (slime.get().type().getId() != NIX::ID)
? IDocsumWriter::slime2RawBuf(slime, buf)
: 0;
- reply->docsums[i].setData(buf.GetDrainPos(), docsumLen);
- symbols = Slime::reclaimSymbols(std::move(slime));
- } else {
- uint32_t docsumLen = _docsumWriter.WriteDocsum(docId, &_docsumState, &_docsumStore, &buf);
- reply->docsums[i].setData(buf.GetDrainPos(), docsumLen);
- }
+ reply->docsums[i].setData(buf.GetDrainPos(), docsumLen);
+ symbols = Slime::reclaimSymbols(std::move(slime));
}
}
return reply;
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
index fc5f1288f00..3b4a72eb4fd 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
@@ -104,7 +104,7 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type)
void
-DocumentStoreAdapter::convertFromSearchDoc(Document &doc, uint32_t docId, bool useSlimeInsideFields)
+DocumentStoreAdapter::convertFromSearchDoc(Document &doc, uint32_t docId)
{
for (size_t i = 0; i < _resultClass->GetNumEntries(); ++i) {
const ResConfigEntry * entry = _resultClass->GetEntry(i);
@@ -141,7 +141,7 @@ DocumentStoreAdapter::convertFromSearchDoc(Document &doc, uint32_t docId, bool u
fieldName.c_str(), fieldValue->toString().c_str(),
entry->_type);
FieldValue::UP convertedFieldValue =
- SummaryFieldConverter::convertSummaryField(markup, *fieldValue, useSlimeInsideFields);
+ SummaryFieldConverter::convertSummaryField(markup, *fieldValue);
if (convertedFieldValue.get() != NULL) {
if (!writeField(*convertedFieldValue, entry->_type)) {
LOG(warning,
@@ -181,7 +181,7 @@ DocumentStoreAdapter(const search::IDocumentStore & docStore,
}
DocsumStoreValue
-DocumentStoreAdapter::getMappedDocsum(uint32_t docId, bool useSlimeInsideFields)
+DocumentStoreAdapter::getMappedDocsum(uint32_t docId)
{
if (!_resultPacker.Init(getSummaryClassId())) {
LOG(warning,
@@ -201,7 +201,7 @@ DocumentStoreAdapter::getMappedDocsum(uint32_t docId, bool useSlimeInsideFields)
"getMappedDocSum(%u): document={\n%s\n}",
docId,
document->toString(true).c_str());
- convertFromSearchDoc(*document, docId, useSlimeInsideFields);
+ convertFromSearchDoc(*document, docId);
const char * buf;
uint32_t buflen;
if (!_resultPacker.GetDocsumBlob(&buf, &buflen)) {
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h
index d58117ace96..26c6b7683fc 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.h
@@ -32,8 +32,7 @@ private:
search::docsummary::ResType type);
void
- convertFromSearchDoc(document::Document &doc, uint32_t docId,
- bool useSlimeInsideFields);
+ convertFromSearchDoc(document::Document &doc, uint32_t docId);
public:
DocumentStoreAdapter(const search::IDocumentStore &docStore,
@@ -48,7 +47,7 @@ public:
}
uint32_t getNumDocs() const override { return _docStore.nextId(); }
- search::docsummary::DocsumStoreValue getMappedDocsum(uint32_t docId, bool useSlimeInsideFields) override;
+ search::docsummary::DocsumStoreValue getMappedDocsum(uint32_t docId) override;
uint32_t getSummaryClassId() const override { return _resultClass->GetClassID(); }
};
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
index 132856ef755..06abf5e65bc 100644
--- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp
@@ -60,7 +60,6 @@ DocsumBySlime::slimeToRequest(const Inspector & request)
{
DocsumRequest::UP docsumRequest(std::make_unique<DocsumRequest>(true));
- docsumRequest->_flags = search::fs4transport::GDFLAG_ALLOW_SLIME;
docsumRequest->resultClassName = request[SUMMARYCLASS].asString().make_string();
Inspector & gids = request[GIDS];
docsumRequest->hits.reserve(gids.entries());
diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h
index 2625b3f0635..d41b636ec8d 100644
--- a/searchlib/src/vespa/searchlib/common/transport.h
+++ b/searchlib/src/vespa/searchlib/common/transport.h
@@ -104,7 +104,7 @@ enum getdocsums_features {
enum getdocsums_flags {
GDFLAG_IGNORE_ROW = 0x00000001,
- GDFLAG_ALLOW_SLIME = 0x00000002
+ GDFLAG_ALLOW_SLIME_NOTUSED = 0x00000002 // TODO: remove in Vespa 7
};
// docsum class for slime tunneling
diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
index 2e756a21d2b..1892fa96179 100644
--- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
+++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
@@ -13,6 +13,7 @@ LOG_SETUP("positionsdfw_test");
#include <vespa/searchsummary/docsummary/docsumstate.h>
#include <vespa/searchlib/util/rawbuf.h>
#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/slime/slime.h>
#include <vespa/juniper/rpinterface.h>
using search::RawBuf;
@@ -116,16 +117,17 @@ void checkWritePositionField(Test &test, AttrType &attr,
createPositionsDFW(attr.getName().c_str(), &attribute_man);
ASSERT_TRUE(writer.get());
ResType res_type = RES_LONG_STRING;
- RawBuf target(1024);
MyGetDocsumsStateCallback callback;
GetDocsumsState state(callback);
state._attributes.push_back(&attr);
- writer->WriteField(doc_id, 0, &state, res_type, &target);
+ vespalib::Slime target;
+ vespalib::slime::SlimeInserter inserter(target);
+ writer->insertField(doc_id, nullptr, &state, res_type, inserter);
- test.EXPECT_EQUAL(expected.size(), *(const uint32_t *)(target.GetDrainPos()));
- const char *p = target.GetDrainPos() + 4;
- test.EXPECT_EQUAL(expected, string(p, p + expected.size()));
+ vespalib::Memory got = target.get().asString();
+ test.EXPECT_EQUAL(expected.size(), got.size);
+ test.EXPECT_EQUAL(expected, string(got.data, got.size));
}
void Test::requireThat2DPositionFieldIsWritten() {
diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp
index f5f37468b79..7a17e803ea8 100644
--- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp
+++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp
@@ -56,7 +56,6 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback {
config->CreateEnumMaps();
writer.reset(new DynamicDocsumWriter(config, 0));
packer.reset(new ResultPacker(writer->GetResultConfig()));
- state._args.setFlags(search::fs4transport::GDFLAG_ALLOW_SLIME);
}
void getDocsum(Slime &slime) {
uint32_t classId;
@@ -70,8 +69,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback {
::decode(Memory(buf.GetDrainPos(), buf.GetUsedLen()), slime), 0u);
}
uint32_t getNumDocs() const override { return 2; }
- DocsumStoreValue getMappedDocsum(uint32_t docid, bool useSlimeInsideFields) override {
- EXPECT_EQUAL(true, useSlimeInsideFields);
+ DocsumStoreValue getMappedDocsum(uint32_t docid) override {
EXPECT_EQUAL(1u, docid);
EXPECT_TRUE(packer->Init(0));
EXPECT_TRUE(packer->AddInteger(4));
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
index 71f15b0d894..869cffb7661 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
@@ -78,11 +78,6 @@ public:
SingleAttrDFW(const vespalib::string & attrName) :
AttrDFW(attrName)
{ }
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- RawBuf *target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
@@ -91,92 +86,6 @@ public:
virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const;
};
-uint32_t
-SingleAttrDFW::WriteField(uint32_t docid,
- GeneralResult *,
- GetDocsumsState * state,
- ResType type,
- RawBuf *target)
-{
- const char *s="";
- const IAttributeVector & v = vec(*state);
- switch (type) {
- case RES_INT: {
- uint32_t val = v.getInt(docid);
- target->append(&val, sizeof(val));
- return sizeof(val);
- break; }
- case RES_SHORT: {
- uint16_t val = v.getInt(docid);
- target->append(&val, sizeof(val));
- return sizeof(val);
- break; }
- case RES_BYTE: {
- uint8_t val = v.getInt(docid);
- target->append(&val, sizeof(val));
- return sizeof(val);
- break; }
- case RES_FLOAT: {
- float val = v.getFloat(docid);
- target->append(&val, sizeof(val));
- return sizeof(val);
- break; }
- case RES_DOUBLE: {
- double val = v.getFloat(docid);
- target->append(&val, sizeof(val));
- return sizeof(val);
- break; }
- case RES_INT64: {
- uint64_t val = v.getInt(docid);
- target->append(&val, sizeof(val));
- return sizeof(val);
- break; }
- case RES_STRING:
- case RES_DATA: {
- s = v.getString(docid, NULL, 0); // no need to pass in a buffer, this attribute has a string storage.
- uint32_t len = strlen(s);
- uint16_t slen = (len < 0xffff) ? len : 0xffff;
- target->append(&slen, sizeof(slen));
- target->append(s, slen);
- return (sizeof(slen) + slen);
- break; }
- case RES_TENSOR: {
- vespalib::nbostream str;
- BasicType::Type t = v.getBasicType();
- switch (t) {
- case BasicType::TENSOR: {
- const tensor::TensorAttribute &tv =
- static_cast<const tensor::TensorAttribute &>(v);
- const auto tensor = tv.getTensor(docid);
- if (tensor) {
- vespalib::tensor::TypedBinaryFormat::serialize(str, *tensor);
- }
- }
- default:
- break;
- }
- uint32_t slen = str.size();
- target->append(&slen, sizeof(slen));
- target->append(str.peek(), slen);
- return (sizeof(slen) + slen);
- }
- case RES_JSONSTRING:
- case RES_XMLSTRING:
- case RES_FEATUREDATA:
- case RES_LONG_STRING:
- case RES_LONG_DATA: {
- s = v.getString(docid, NULL, 0); // no need to pass in a buffer, this attribute has a string storage.
- uint32_t slen = strlen(s);
- target->append(&slen, sizeof(slen));
- target->append(s, slen);
- return (sizeof(slen) + slen);
- break; }
- default:
- return 0;
- }
- return 0;
-}
-
bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state) const
{
return vec(*state).isUndefined(docid);
@@ -268,11 +177,6 @@ class MultiAttrDFW : public AttrDFW
{
public:
MultiAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) {}
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- RawBuf *target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
@@ -281,72 +185,6 @@ public:
};
-uint32_t
-MultiAttrDFW::WriteField(uint32_t docid,
- GeneralResult *,
- GetDocsumsState * state,
- ResType type,
- RawBuf *target)
-{
- bool isLong = IsBinaryCompatible(type, RES_LONG_STRING);
- uint32_t written = 0;
- uint16_t str_len_16 = 0;
- uint32_t str_len_32 = 0;
- int str_len_ofs = target->GetUsedLen();
- vespalib::JSONStringer & jsonStr = state->_jsonStringer;
-
- if (isLong) {
- target->append(&str_len_32, sizeof(str_len_32));
- } else {
- target->append(&str_len_16, sizeof(str_len_16));
- }
- const IAttributeVector & v = vec(*state);
- uint32_t entries = v.getValueCount(docid);
- {
- std::vector<IAttributeVector::WeightedString> elements(entries);
- entries = std::min(entries, v.get(docid, &elements[0], entries));
- jsonStr.clear();
- jsonStr.beginArray();
- for (uint32_t i = 0; i < entries; ++i) {
- if (v.hasWeightedSetType()) {
- jsonStr.beginArray();
- jsonStr.appendString(elements[i].getValue());
- jsonStr.appendInt64(elements[i].getWeight());
- jsonStr.endArray();
- } else {
- jsonStr.appendString(elements[i].getValue());
- }
- }
- jsonStr.endArray();
- (*target) += jsonStr.toString().c_str();
- jsonStr.clear();
- }
-
- // calculate number of bytes written
- written = target->GetUsedLen() - str_len_ofs;
-
- // patch in correct field length
- if (isLong) {
- str_len_32 = written - sizeof(str_len_32);
- memcpy(target->GetWritableDrainPos(str_len_ofs),
- &str_len_32, sizeof(str_len_32));
- } else {
- str_len_16 = written - sizeof(str_len_16);
- if (str_len_16 != written - sizeof(str_len_16)) {
- target->truncate(str_len_ofs);
- str_len_16 = 0;
- target->append(&str_len_16, sizeof(uint16_t));
- *target += "***OVERFLOW***";
- written = target->GetUsedLen() - str_len_ofs;
- str_len_16 = written - sizeof(uint16_t);
- assert(str_len_16 == written - sizeof(uint16_t));
- }
- memcpy(target->GetWritableDrainPos(str_len_ofs),
- &str_len_16, sizeof(str_len_16));
- }
- return written;
-}
-
void
MultiAttrDFW::insertField(uint32_t docid,
GeneralResult *,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
index e71b4d0fe69..e4c5181d05f 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp
@@ -44,19 +44,6 @@ EmptyDFW::insertField(uint32_t /*docid*/,
return;
}
-uint32_t
-EmptyDFW::WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target)
-{
- (void) docid;
- (void) gres;
- (void) state;
- return DocsumFormat::addEmpty(type, *target);
-}
-
//--------------------------------------------------------------------------
CopyDFW::CopyDFW()
@@ -172,109 +159,6 @@ CopyDFW::insertField(uint32_t /*docid*/,
}
}
-uint32_t
-CopyDFW::WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target)
-{
- (void) docid;
-
- uint32_t written = 0;
-
- int idx = gres->GetClass()->GetIndexFromEnumValue(_inputFieldEnumValue);
- ResEntry *entry = gres->GetEntry(idx);
-
- DocsumFormat::Appender appender(*target);
-
- if (entry != NULL &&
- IsRuntimeCompatible(entry->_type, type)) {
-
- // copy field
-
- switch (type) {
-
- case RES_INT: {
- written += appender.addInt32(entry->_intval);
- break; }
-
- case RES_SHORT: {
- written += appender.addShort(entry->_intval);
- break; }
-
- case RES_BYTE: {
- written += appender.addByte(entry->_intval);
- break; }
-
- case RES_FLOAT: {
- written += appender.addFloat(entry->_doubleval);
- break; }
-
- case RES_DOUBLE: {
- written += appender.addDouble(entry->_doubleval);
- break; }
-
- case RES_INT64: {
- written += appender.addInt64(entry->_int64val);
- break; }
-
- case RES_STRING: {
- uint32_t len;
- const char *spt;
- // resolve field
- entry->_resolve_field(&spt, &len,
- &state->_docSumFieldSpace);
- written += appender.addShortData(spt, len);
- break; }
-
- case RES_DATA: {
- uint32_t len;
- const char *dpt;
- // resolve field
- entry->_resolve_field(&dpt, &len,
- &state->_docSumFieldSpace);
- written += appender.addShortData(dpt, len);
- break; }
-
- case RES_XMLSTRING:
- case RES_JSONSTRING:
- case RES_FEATUREDATA:
- case RES_LONG_STRING: {
-
- uint32_t flen = entry->_len;
- uint32_t slen = entry->_get_length();
-
- // preserve compression flag
- target->append(&flen, sizeof(flen));
- written += sizeof(flen);
- target->append(entry->_stringval, slen);
- written += slen;
-
- break; }
-
- case RES_TENSOR:
- case RES_LONG_DATA: {
-
- uint32_t flen = entry->_len;
- uint32_t dlen = entry->_get_length();
-
- // preserve compression flag
- target->append(&flen, sizeof(flen));
- written += sizeof(flen);
- target->append(entry->_dataval, dlen);
- written += dlen;
-
- break; }
- }
- } else {
- // insert empty field
- written += appender.addEmpty(type);
- }
-
- return written;
-}
-
//--------------------------------------------------------------------------
} // namespace docsummary
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h
index 2c276402cbe..2f86e8ef5f1 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h
@@ -29,11 +29,6 @@ public:
{ return ResultConfig::IsRuntimeCompatible(a, b); }
virtual bool IsGenerated() const = 0;
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target) = 0;
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
@@ -61,11 +56,6 @@ public:
virtual ~EmptyDFW();
virtual bool IsGenerated() const { return true; }
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
@@ -87,11 +77,6 @@ public:
bool Init(const ResultConfig & config, const char *inputField);
virtual bool IsGenerated() const { return false; }
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h
index ae611cd71a1..fe4b7c990ca 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h
@@ -39,9 +39,8 @@ public:
*
* @return docsum blob location and size
* @param docid local document id
- * @param useSlimeInsideFields use serialized slime instead of json for structured fields
**/
- virtual DocsumStoreValue getMappedDocsum(uint32_t docid, bool useSlimeInsideFields) = 0;
+ virtual DocsumStoreValue getMappedDocsum(uint32_t docid) = 0;
/**
* Will return default input class used.
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
index 01ebf333fe2..e0d0a9fab42 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp
@@ -28,151 +28,6 @@ IDocsumWriter::slime2RawBuf(const Slime & slime, RawBuf & buf)
return (buf.GetUsedLen() - preUsed);
}
-uint32_t
-DynamicDocsumWriter::WriteClassID(uint32_t classID, search::RawBuf *target)
-{
- uint32_t written = 0;
-
- target->append(&classID, sizeof(classID));
- written = sizeof(classID);
-
- return written;
-}
-
-
-uint32_t
-DynamicDocsumWriter::GenerateDocsum(uint32_t docid,
- GetDocsumsState *state,
- const ResultClass *outputClass,
- search::RawBuf *target)
-{
- uint32_t written = 0;
-
- written += WriteClassID(outputClass->GetClassID(), target);
-
- for (uint32_t i = 0; i < outputClass->GetNumEntries(); i++) {
-
- const ResConfigEntry *outCfg = outputClass->GetEntry(i);
- IDocsumFieldWriter *writer = _overrideTable[outCfg->_enumValue];
- LOG_ASSERT(writer != NULL);
-
- written += writer->WriteField(docid, NULL, state, outCfg->_type, target);
- }
-
- return written;
-}
-
-
-uint32_t
-DynamicDocsumWriter::RepackDocsum(GeneralResult *gres,
- GetDocsumsState *state,
- const ResultClass *outputClass,
- search::RawBuf *target)
-{
- uint32_t written = 0;
-
- written += WriteClassID(outputClass->GetClassID(), target);
-
- DocsumFormat::Appender appender(*target);
-
- for (uint32_t i = 0; i < outputClass->GetNumEntries(); i++) {
-
- const ResConfigEntry *outCfg = outputClass->GetEntry(i);
- IDocsumFieldWriter *writer = _overrideTable[outCfg->_enumValue];
-
- if (writer != NULL) {
-
- written += writer->WriteField(gres->GetDocID(), gres, state,
- outCfg->_type, target);
-
- } else {
-
- int inIdx = gres->GetClass()->GetIndexFromEnumValue(outCfg->_enumValue);
- const ResConfigEntry *inCfg = gres->GetClass()->GetEntry(inIdx);
-
- if (inCfg != NULL && inCfg->_type == outCfg->_type) {
-
- // copy field
-
- const ResEntry *entry = gres->GetEntry(inIdx);
- LOG_ASSERT(entry != NULL);
-
- switch (outCfg->_type) {
-
- case RES_INT: {
- written += appender.addInt32(entry->_intval);
- break; }
-
- case RES_SHORT: {
- written += appender.addShort(entry->_intval);
- break; }
-
- case RES_BYTE: {
- written += appender.addByte(entry->_intval);
- break; }
-
- case RES_FLOAT: {
- written += appender.addFloat(entry->_doubleval);
- break; }
-
- case RES_DOUBLE: {
- written += appender.addDouble(entry->_doubleval);
- break; }
-
- case RES_INT64: {
- written += appender.addInt64(entry->_int64val);
- break; }
-
- case RES_STRING: {
- uint32_t slen = entry->_stringlen;
- const char *sval = entry->_stringval;
- written += appender.addShortData(sval, slen);
- break; }
-
- case RES_DATA: {
- uint32_t dlen = entry->_datalen;
- const char *dval = entry->_dataval;
- written += appender.addShortData(dval, dlen);
- break; }
-
- case RES_XMLSTRING:
- case RES_JSONSTRING:
- case RES_FEATUREDATA:
- case RES_LONG_STRING: {
- uint32_t flen = entry->_len;
- uint32_t slen = entry->_get_length();
-
- // preserve compression flag
- target->append(&flen, sizeof(flen));
- written += sizeof(flen);
- target->append(entry->_stringval, slen);
- written += slen;
- break; }
-
- case RES_TENSOR:
- case RES_LONG_DATA: {
- uint32_t flen = entry->_len;
- uint32_t dlen = entry->_get_length();
-
- // preserve compression flag
- target->append(&flen, sizeof(flen));
- written += sizeof(flen);
- target->append(entry->_dataval, dlen);
- written += dlen;
- break; }
- }
-
- } else {
- // insert empty field
- written += appender.addEmpty(outCfg->_type);
- }
- }
- } // END for loop
-
- return written;
-}
-
-
DynamicDocsumWriter::ResolveClassInfo
DynamicDocsumWriter::resolveClassInfo(vespalib::stringref outputClassName, uint32_t inputClassId) const
{
@@ -223,51 +78,6 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const
rci.outputClass = rci.inputClass;
rci.outputClassInfo = rci.inputClass->getDynamicInfo();
}
- if ((rci.inputClass == rci.outputClass) && (rci.outputClassInfo->_overrideCnt == 0)) {
- rci.mustRepack = false;
- }
-}
-
-void
-DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, DocsumStoreValue blob) const
-{
- uint32_t id = _resultConfig->GetClassID(blob.pt(), blob.len());
- resolveInputClass(rci, id);
-}
-
-
-uint32_t
-DynamicDocsumWriter::oldWriteDocsum(uint32_t docid,
- GetDocsumsState *state,
- IDocsumStore *docinfos,
- search::RawBuf *target)
-{
- ResolveClassInfo rci = resolveOutputClass(state->_args.getResultClassName());
- if (rci.mustSkip) {
- return 0;
- } else if (rci.allGenerated) {
- // generate docsum entry on-the-fly
- return GenerateDocsum(docid, state, rci.outputClass, target);
- }
- // look up docsum entry
- DocsumStoreValue value = docinfos->getMappedDocsum(docid, false);
- resolveInputClass(rci, value);
- if (rci.mustSkip) {
- return 0;
- } else if (rci.mustRepack) {
- // re-pack docsum blob
- GeneralResult gres(rci.inputClass, 0, docid, 0);
- if (gres.inplaceUnpack(value)) {
- return RepackDocsum(&gres, state, rci.outputClass, target);
- } else { // unpack failed
- LOG(error, "Unpack failed: illegal docsum entry for document %d", docid);
- }
- } else {
- // pass-through docsum blob
- target->append(value.pt(), value.len());
- return value.len();
- }
- return 0;
}
@@ -311,9 +121,7 @@ static void convertEntry(GetDocsumsState *state,
case RES_JSONSTRING:
entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace);
if (len != 0) {
- // note: 'JSONSTRING' really means 'structured data',
- // and in this code path we depend on calling the
- // getMappedDocsum api with flag useSlimeInsideFields=true
+ // note: 'JSONSTRING' really means 'structured data'
size_t d = BinaryFormat::decode_into(Memory(ptr, len), slime, inserter);
if (d != len) {
LOG(warning, "could not decode %u bytes: %zu bytes decoded", len, d);
@@ -347,7 +155,7 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci,
}
} else {
// look up docsum entry
- DocsumStoreValue value = docinfos->getMappedDocsum(docid, true);
+ DocsumStoreValue value = docinfos->getMappedDocsum(docid);
// re-pack docsum blob
GeneralResult gres(rci.inputClass, 0, docid, 0);
if (! gres.inplaceUnpack(value)) {
@@ -500,13 +308,12 @@ DynamicDocsumWriter::WriteDocsum(uint32_t docid,
IDocsumStore *docinfos,
search::RawBuf *target)
{
- if ((state->_args.getFlags() & ::search::fs4transport::GDFLAG_ALLOW_SLIME) != 0) {
- vespalib::Slime slime;
- vespalib::slime::SlimeInserter inserter(slime);
- insertDocsum(resolveClassInfo(state->_args.getResultClassName(), docinfos->getSummaryClassId()), docid, state, docinfos, slime, inserter);
- return slime2RawBuf(slime, *target);
- }
- return oldWriteDocsum(docid, state, docinfos, target);
+ vespalib::Slime slime;
+ vespalib::slime::SlimeInserter inserter(slime);
+ ResolveClassInfo rci = resolveClassInfo(state->_args.getResultClassName(),
+ docinfos->getSummaryClassId());
+ insertDocsum(rci, docid, state, docinfos, slime, inserter);
+ return slime2RawBuf(slime, *target);
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h
index 66e64bc7d10..a773c704536 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h
@@ -25,13 +25,12 @@ public:
struct ResolveClassInfo {
bool mustSkip;
bool allGenerated;
- bool mustRepack;
uint32_t outputClassId;
const ResultClass *outputClass;
const ResultClass::DynamicInfo *outputClassInfo;
const ResultClass *inputClass;
ResolveClassInfo()
- : mustSkip(false), allGenerated(false), mustRepack(true),
+ : mustSkip(false), allGenerated(false),
outputClassId(ResultConfig::NoClassID()),
outputClass(NULL), outputClassInfo(NULL), inputClass(NULL)
{ }
@@ -85,12 +84,8 @@ private:
search::RawBuf *target);
void resolveInputClass(ResolveClassInfo &rci, uint32_t id) const;
- void resolveInputClass(ResolveClassInfo &rci, DocsumStoreValue blob) const;
- ResolveClassInfo resolveOutputClass(vespalib::stringref outputClassName) const;
- uint32_t oldWriteDocsum(uint32_t docid, GetDocsumsState *state,
- IDocsumStore *docinfos,
- search::RawBuf *target);
+ ResolveClassInfo resolveOutputClass(vespalib::stringref outputClassName) const;
public:
DynamicDocsumWriter(ResultConfig *config, KeywordExtractor *extractor);
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
index bd83f821ff1..c903c7cbb78 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
@@ -449,23 +449,6 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid,
}
}
-uint32_t
-DynamicTeaserDFW::WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target)
-{
- vespalib::string teaser = makeDynamicTeaser(docid, gres, state);
-
- bool isLong = IsBinaryCompatible(type, RES_LONG_STRING);
- if (isLong) {
- return DocsumFormat::addLongData(*target, teaser.c_str(), teaser.size());
- } else {
- return DocsumFormat::addShortData(*target, teaser.c_str(), teaser.size());
- }
-}
-
void
DynamicTeaserDFW::insertField(uint32_t docid,
GeneralResult *gres,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp
index b1c68c4ffff..0922b231b10 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp
@@ -36,61 +36,6 @@ void fmtZcurve(int64_t zval, vespalib::slime::Inserter &target)
}
}
-void fmtZcurve(int64_t zval, vespalib::JSONWriter json)
-{
- int32_t docx = 0;
- int32_t docy = 0;
- vespalib::geo::ZCurve::decode(zval, &docx, &docy);
- json.beginObject();
- json.appendKey("y"); json.appendInt64(docy);
- json.appendKey("x"); json.appendInt64(docx);
- json.endObject();
-}
-
-vespalib::asciistream
-formatField(const IAttributeVector & attribute, uint32_t docid)
-{
- vespalib::asciistream target;
- vespalib::JSONWriter json(target);
-
- if (attribute.hasMultiValue()) {
- uint32_t entries = attribute.getValueCount(docid);
- LOG(debug, "docid=%d, entries=%d", docid, entries);
- json.beginArray();
- if (attribute.hasWeightedSetType()) {
- std::vector<IAttributeVector::WeightedInt> elements(entries);
- entries = attribute.get(docid, &elements[0], entries);
- for (uint32_t i = 0; i < entries; ++i) {
- json.beginObject();
- int64_t pos = elements[i].getValue();
- json.appendKey("item");
- fmtZcurve(pos, json);
- json.appendKey("weight");
- json.appendInt64(elements[i].getWeight());
- json.endObject();
- }
- } else {
- std::vector<IAttributeVector::largeint_t> elements(16);
- uint32_t numValues = attribute.get(docid, &elements[0], elements.size());
- if (numValues > elements.size()) {
- elements.resize(numValues);
- numValues = attribute.get(docid, &elements[0], elements.size());
- assert(numValues <= elements.size());
- }
- LOG(debug, "docid=%d, numValues=%d", docid, numValues);
- for (uint32_t i = 0; i < numValues; i++) {
- int64_t pos = elements[i];
- fmtZcurve(pos, json);
- }
- }
- } else {
- int64_t pos = attribute.getInt(docid);
- LOG(debug, "docid=%d, pos=%ld", docid, pos);
- fmtZcurve(pos, json);
- }
- return target;
-}
-
}
void
@@ -135,32 +80,6 @@ GeoPositionDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState * d
}
}
-uint32_t
-GeoPositionDFW::WriteField(uint32_t docid,
- GeneralResult *,
- GetDocsumsState * dsState,
- ResType type,
- search::RawBuf * target)
-{
- int str_len_ofs = target->GetUsedLen();
-
- vespalib::asciistream val(formatField(vec(*dsState), docid));
-
- bool isLong = IsBinaryCompatible(type, RES_LONG_STRING);
- if (isLong) {
- uint32_t str_len_32 = val.size();
- target->append(&str_len_32, sizeof(str_len_32));
- target->append(val.c_str(), str_len_32);
- } else {
- uint16_t str_len_16 = val.size();
- target->append(&str_len_16, sizeof(str_len_16));
- target->append(val.c_str(), str_len_16);
- }
- // calculate number of bytes written
- uint32_t written = target->GetUsedLen() - str_len_ofs;
- return written;
-}
-
GeoPositionDFW::UP
GeoPositionDFW::create(const char *attribute_name,
IAttributeManager *attribute_manager)
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h
index a36045f0268..72ec905b5a7 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h
@@ -15,8 +15,6 @@ class GeoPositionDFW : public AttrDFW
public:
typedef std::unique_ptr<GeoPositionDFW> UP;
GeoPositionDFW(const vespalib::string & attrName);
- uint32_t WriteField(uint32_t docid, GeneralResult * gres, GetDocsumsState * state,
- ResType type, search::RawBuf * target) override;
void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
ResType type, vespalib::slime::Inserter &target) override;
static UP create(const char *attribute_name, IAttributeManager *attribute_manager);
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h
index 44431a146bb..981885754bd 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h
@@ -57,11 +57,6 @@ public:
GeneralResult *gres,
GetDocsumsState *state);
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult *gres,
- GetDocsumsState *state,
- ResType type,
- search::RawBuf *target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
index 723238b1b8b..841840d8349 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp
@@ -105,84 +105,6 @@ AbsDistanceDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *st
}
}
-
-uint32_t
-AbsDistanceDFW::WriteField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
- ResType type, search::RawBuf *target)
-{
- (void) gres;
-
- bool forceEmpty = true;
-
- const vespalib::string &locationStr = state->_args.getLocation();
- if (locationStr.size() > 0) {
- if (state->_parsedLocation.get() == NULL) {
- state->_callback.ParseLocation(state);
- }
- assert(state->_parsedLocation.get() != NULL);
- if (state->_parsedLocation->getParseError() == NULL) {
- forceEmpty = false;
- }
- }
-
- uint32_t written = 0;
- if (!forceEmpty) {
- uint64_t absdist = findMinDistance(docid, state);
-
- if (type != RES_INT) {
- bool isLong = IsBinaryCompatible(type, RES_LONG_STRING);
- uint16_t str_len_16 = 0;
- uint32_t str_len_32 = 0;
- int str_len_ofs = target->GetUsedLen();
-
- if (isLong)
- target->append(&str_len_32, sizeof(str_len_32));
- else
- target->append(&str_len_16, sizeof(str_len_16));
-
- target->addNum64(absdist, 1, ' ');
-
- // calculate number of bytes written
- written = target->GetUsedLen() - str_len_ofs;
-
- // patch in correct field length
- if (isLong) {
- str_len_32 = written - sizeof(str_len_32);
- memcpy(target->GetWritableDrainPos(str_len_ofs), &str_len_32,
- sizeof(str_len_32));
- } else {
- str_len_16 = written - sizeof(str_len_16);
- memcpy(target->GetWritableDrainPos(str_len_ofs), &str_len_16,
- sizeof(str_len_16));
- }
- } else {
- uint32_t val32 = (uint32_t) absdist;
- target->append(&val32, sizeof(val32));
- written = sizeof(val32);
- }
- } else {
- if (type != RES_INT) {
- bool isLong = IsBinaryCompatible(type, RES_LONG_STRING);
- uint16_t str_len_16 = 0;
- uint32_t str_len_32 = 0;
- int str_len_ofs = target->GetUsedLen();
-
- if (isLong)
- target->append(&str_len_32, sizeof(str_len_32));
- else
- target->append(&str_len_16, sizeof(str_len_16));
-
- // calculate number of bytes written
- written = target->GetUsedLen() - str_len_ofs;
- } else {
- uint32_t val32 = 0u;
- target->append(&val32, sizeof(val32));
- written = sizeof(val32);
- }
- }
- return written;
-}
-
//--------------------------------------------------------------------------
PositionsDFW::PositionsDFW(const vespalib::string & attrName) :
@@ -243,30 +165,6 @@ formatField(const attribute::IAttributeVector &attribute, uint32_t docid, ResTyp
}
}
-uint32_t
-PositionsDFW::WriteField(uint32_t docid, GeneralResult *, GetDocsumsState * dsState,
- ResType type, search::RawBuf *target)
-{
- int str_len_ofs = target->GetUsedLen();
-
- vespalib::asciistream val(formatField(vec(*dsState), docid, type));
-
- bool isLong = IsBinaryCompatible(type, RES_LONG_STRING);
- if (isLong) {
- uint32_t str_len_32 = val.size();
- target->append(&str_len_32, sizeof(str_len_32));
- target->append(val.c_str(), str_len_32);
- } else {
- uint16_t str_len_16 = val.size();
- target->append(&str_len_16, sizeof(str_len_16));
- target->append(val.c_str(), str_len_16);
- }
- // calculate number of bytes written
- uint32_t written = target->GetUsedLen() - str_len_ofs;
- return written;
-}
-
-
void
PositionsDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState * dsState,
ResType type, vespalib::slime::Inserter &target)
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h
index 2140ea52a63..0aad5b2550c 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h
@@ -15,8 +15,6 @@ public:
AbsDistanceDFW(const vespalib::string & attrName);
bool IsGenerated() const override { return true; }
- uint32_t WriteField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
- ResType type, search::RawBuf *target) override;
void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
ResType type, vespalib::slime::Inserter &target) override;
};
@@ -31,8 +29,6 @@ public:
PositionsDFW(const vespalib::string & attrName);
bool IsGenerated() const override { return true; }
- uint32_t WriteField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
- ResType type, search::RawBuf *target) override;
void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state,
ResType type, vespalib::slime::Inserter &target) override ;
};
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp
index 629bd37b155..e1378ba1882 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp
@@ -23,44 +23,6 @@ RankFeaturesDFW::init(IDocsumEnvironment * env)
_env = env;
}
-uint32_t
-RankFeaturesDFW::WriteField(uint32_t docid,
- GeneralResult * gres,
- GetDocsumsState * state,
- ResType type,
- search::RawBuf * target)
-{
- (void) gres;
-
- if (state->_rankFeatures.get() == NULL) {
- state->_callback.FillRankFeatures(state, _env);
- if (state->_rankFeatures.get() == NULL) { // still no rank features to write
- return DocsumFormat::addEmpty(type, *target);
- }
- }
-
- uint32_t written = 0;
-
- const FeatureSet::StringVector & names = state->_rankFeatures->getNames();
- const feature_t * values = state->_rankFeatures->getFeaturesByDocId(docid);
- vespalib::JSONStringer & json(state->_jsonStringer);
- if (values != NULL) {
- json.clear();
- json.beginObject();
- for (uint32_t i = 0; i < names.size(); ++i) {
- featureDump(json, names[i], values[i]);
- }
- json.endObject();
- written += SummaryFeaturesDFW::writeString(json.toString(), type, target);
- json.clear();
- } else {
- written += DocsumFormat::addEmpty(type, *target);
- }
-
- return written;
-}
-
-
void
RankFeaturesDFW::insertField(uint32_t docid,
GeneralResult *,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h
index a04271a16c1..3c1382c1eda 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h
@@ -22,11 +22,6 @@ public:
virtual ~RankFeaturesDFW();
void init(IDocsumEnvironment * env);
virtual bool IsGenerated() const { return true; }
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult * gres,
- GetDocsumsState * state,
- ResType type,
- search::RawBuf * target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
index 758162f5e97..d78f684501f 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
@@ -87,50 +87,6 @@ SummaryFeaturesDFW::insertField(uint32_t docid,
}
}
-uint32_t
-SummaryFeaturesDFW::WriteField(uint32_t docid,
- GeneralResult * gres,
- GetDocsumsState * state,
- ResType type,
- search::RawBuf * target)
-{
- (void) gres;
-
- if (state->_summaryFeatures.get() == 0) {
- state->_callback.FillSummaryFeatures(state, _env);
- if (state->_summaryFeatures.get() == 0) { // still no summary features to write
- return DocsumFormat::addEmpty(type, *target);
- }
- }
-
- uint32_t written = 0;
-
- const FeatureSet::StringVector &names = state->_summaryFeatures->getNames();
- vespalib::JSONStringer & json(state->_jsonStringer);
- const feature_t *values = state->_summaryFeatures->getFeaturesByDocId(docid);
- if (values != NULL) {
- json.clear();
- json.beginObject();
- for (uint32_t i = 0; i < names.size(); ++i) {
- featureDump(json, names[i], values[i]);
- }
- json.appendKey(_G_cached);
- if (state->_summaryFeaturesCached) {
- json.appendDouble(1.0);
- } else {
- json.appendDouble(0.0);
- }
- json.endObject();
-
- written += writeString(json.toString(), type, target);
- json.clear();
- } else {
- written += DocsumFormat::addEmpty(type, *target);
- }
-
- return written;
-}
-
void FeaturesDFW::featureDump(vespalib::JSONStringer & json, const vespalib::stringref & name, double feature)
{
json.appendKey(name);
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h
index 20e2624c7b7..7da574fe1fd 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h
@@ -31,11 +31,6 @@ public:
virtual ~SummaryFeaturesDFW();
void init(IDocsumEnvironment * env);
virtual bool IsGenerated() const { return true; }
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult * gres,
- GetDocsumsState * state,
- ResType type,
- search::RawBuf * target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
index 92378ed83ce..6f545dfe88e 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
@@ -36,12 +36,10 @@
#include <vespa/vespalib/geo/zcurve.h>
#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/jsonwriter.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/data/slime/slime.h>
#include <vespa/vespalib/data/slime/convenience.h>
#include <vespa/vespalib/data/slime/binary_format.h>
-#include <vespa/vespalib/data/slime/json_format.h>
#include <vespa/eval/tensor/serialization/slime_binary_format.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -86,7 +84,6 @@ using search::util::URL;
using std::make_pair;
using std::pair;
using std::vector;
-using vespalib::JSONWriter;
using vespalib::asciistream;
using vespalib::geo::ZCurve;
using vespalib::make_string;
@@ -236,178 +233,6 @@ struct SummaryHandler {
};
-
-class JsonFiller : public ConstFieldValueVisitor {
- JSONWriter &_json;
- bool _tokenize;
-
- virtual void visit(const AnnotationReferenceFieldValue & v ) {
- (void)v;
- _json.beginObject();
- _json.appendKey("error");
- _json.appendString("cannot convert from annotation reference field");
- _json.endObject();
- }
- virtual void visit(const Document & v) {
- (void)v;
- _json.beginObject();
- _json.appendKey("error");
- _json.appendString("cannot convert from field of type document");
- _json.endObject();
- }
-
- virtual void visit(const MapFieldValue & v) {
- _json.beginArray();
- for (const auto & entry : v) {
- _json.beginObject();
-
- _json.appendKey("key");
- const FieldValue &key = *(entry.first);
- key.accept(*this);
-
- const FieldValue &val = *(entry.second);
- _json.appendKey("value");
- val.accept(*this);
-
- _json.endObject();
- }
- _json.endArray();
- }
-
- virtual void visit(const ArrayFieldValue &value) {
- _json.beginArray();
- if (value.size() > 0) {
- for (const FieldValue &fv : value) {
- fv.accept(*this);
- }
- }
- _json.endArray();
- }
-
- virtual void visit(const StringFieldValue &value) {
- if (_tokenize) {
- asciistream tmp;
- SummaryHandler handler(value.getValue(), tmp);
- handleIndexingTerms(handler, value);
- _json.appendString(tmp.str());
- } else {
- _json.appendString(value.getValue());
- }
- }
-
- virtual void visit(const IntFieldValue &value) {
- int32_t v = value.getValue(); _json.appendInt64(v);
- }
- virtual void visit(const LongFieldValue &value) {
- int64_t v = value.getValue(); _json.appendInt64(v);
- }
- virtual void visit(const ShortFieldValue &value) {
- int16_t v = value.getValue(); _json.appendInt64(v);
- }
- virtual void visit(const ByteFieldValue &value) {
- int8_t v = value.getAsByte(); _json.appendInt64(v);
- }
- virtual void visit(const DoubleFieldValue &value) {
- double v = value.getValue(); _json.appendDouble(v);
- }
- virtual void visit(const FloatFieldValue &value) {
- float v = value.getValue(); _json.appendFloat(v);
- }
-
- virtual void
- visit(const PredicateFieldValue &value)
- {
- _json.appendJSON(value.toString());
- }
-
- virtual void
- visit(const RawFieldValue &value)
- {
- // Use base64 coding to represent raw values in json strings.
- std::pair<const char *, size_t> buf = value.getAsRaw();
- vespalib::string rawVal(buf.first, buf.first + buf.second);
- _json.appendString(vespalib::Base64::encode(rawVal));
- }
-
- virtual void visit(const StructFieldValue &value) {
- // stringref type_name = value.getDataType()->getName();
- if (*value.getDataType() == *SearchDataType::URI) {
- FieldValue::UP uriAllValue = value.getValue("all");
- if (uriAllValue.get() != NULL &&
- uriAllValue->inherits(IDENTIFIABLE_CLASSID(StringFieldValue)))
- {
- uriAllValue->accept(*this);
- return;
- }
- }
- _json.beginObject();
- for (StructFieldValue::const_iterator itr = value.begin(); itr != value.end(); ++itr) {
- _json.appendKey(itr.field().getName());
- FieldValue::UP nextValue(value.getValue(itr.field()));
- (*nextValue).accept(*this);
- }
- _json.endObject();
- }
-
- virtual void visit(const WeightedSetFieldValue &value) {
- _json.beginArray();
- if ( value.size() > 0) {
- for (const auto & entry : value) {
- _json.beginObject();
- _json.appendKey("item");
- entry.first->accept(*this);
- _json.appendKey("weight");
- int weight = static_cast<const IntFieldValue &>(*entry.second).getValue();
- _json.appendInt64(weight);
- _json.endObject();
- }
- }
- _json.endArray();
- }
-
- virtual void visit(const TensorFieldValue &value) override {
- const auto &tensor = value.getAsTensorPtr();
- if (tensor) {
- auto slime =
- vespalib::tensor::SlimeBinaryFormat::serialize(*tensor);
- vespalib::SimpleBuffer buf;
- vespalib::slime::JsonFormat::encode(*slime, buf, true);
- _json.appendJSON(buf.get().make_string());
- } else {
- // No tensor value => empty object
- _json.beginObject();
- _json.endObject();
- }
- }
-
- void visit(const ReferenceFieldValue& value) override {
- _json.appendString(value.hasValidDocumentId()
- ? value.getDocumentId().toString()
- : string());
- }
-
-public:
- JsonFiller(bool markup, JSONWriter &json)
- : _json(json), _tokenize(markup) {}
-};
-
-class JsonConverter : public FieldValueConverter {
- bool _tokenize;
-public:
- JsonConverter(bool tokenize)
- : _tokenize(tokenize)
- {}
-
- FieldValue::UP convert(const FieldValue &input) {
- asciistream target;
- JSONWriter json(target);
- JsonFiller visitor(_tokenize, json);
- input.accept(visitor);
- return FieldValue::UP(new StringFieldValue(target.str()));
- }
-
-};
-
class SummaryFieldValueConverter : protected ConstFieldValueVisitor
{
asciistream _str;
@@ -604,11 +429,8 @@ class SlimeFiller : public ConstFieldValueVisitor {
virtual void
visit(const RawFieldValue &value)
{
- // Use base64 coding to represent raw values
std::pair<const char *, size_t> buf = value.getAsRaw();
- vespalib::string rawVal(buf.first, buf.first + buf.second);
- vespalib::string encVal(vespalib::Base64::encode(rawVal));
- _inserter.insertString(Memory(encVal.c_str()));
+ _inserter.insertData(Memory(buf.first, buf.second));
}
virtual void visit(const StructFieldValue &value) {
@@ -691,16 +513,10 @@ public:
FieldValue::UP
SummaryFieldConverter::convertSummaryField(bool markup,
- const FieldValue &value,
- bool useSlimeInsideFields)
+ const FieldValue &value)
{
- if (useSlimeInsideFields) {
- SlimeConverter subConv(markup);
- return SummaryFieldValueConverter(markup, subConv).convert(value);
- } else {
- JsonConverter subConv(markup);
- return SummaryFieldValueConverter(markup, subConv).convert(value);
- }
+ SlimeConverter subConv(markup);
+ return SummaryFieldValueConverter(markup, subConv).convert(value);
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h
index 9d6f54755cc..ebf723f2111 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h
@@ -14,7 +14,7 @@ class SummaryFieldConverter
{
public:
static document::FieldValue::UP
- convertSummaryField(bool markup, const document::FieldValue &value, bool useSlimeInsideFields);
+ convertSummaryField(bool markup, const document::FieldValue &value);
};
} // namespace search::docsummary
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp
index 67a7f9e8da6..3f657efb6c6 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp
@@ -52,44 +52,6 @@ TextExtractorDFW::insertField(uint32_t,
target.insertString(vespalib::Memory(extracted.c_str(), extracted.size()));
}
-uint32_t
-TextExtractorDFW::WriteField(uint32_t docid,
- GeneralResult * gres,
- GetDocsumsState * state,
- ResType type,
- search::RawBuf * target)
-{
- (void) docid;
- (void) type;
- uint32_t slen = 0;
- uint32_t begin = target->GetUsedLen();
- // write text length
- target->append(&slen, sizeof(slen));
-
- ResEntry * entry = gres->GetEntryFromEnumValue(_inputFieldEnum);
- if (entry != NULL) {
- const char * buf = NULL;
- uint32_t buflen = 0;
- entry->_resolve_field(&buf, &buflen, &state->_docSumFieldSpace);
- // extract the text
- Tokenizer tokenizer(buf, buflen);
- while (tokenizer.hasMoreTokens()) {
- Tokenizer::Token token = tokenizer.getNextToken();
- target->append(token.getText().c_str(), token.getText().size());
- }
- } else {
- LOG(warning, "Did not find input entry using field enum %d. Write an empty field", _inputFieldEnum);
- }
-
- // calculate number of bytes written
- uint32_t written = target->GetUsedLen() - begin;
- // patch in correct text length
- slen = written - sizeof(slen);
- memcpy(target->GetWritableDrainPos(begin), &slen, sizeof(slen));
-
- return written;
-}
-
}
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h
index f22d5b3daa4..a12455e61a3 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.h
@@ -25,11 +25,6 @@ public:
// Inherit doc
virtual bool IsGenerated() const { return false; }
// Inherit doc
- virtual uint32_t WriteField(uint32_t docid,
- GeneralResult * gres,
- GetDocsumsState * state,
- ResType type,
- search::RawBuf * target);
virtual void insertField(uint32_t docid,
GeneralResult *gres,
GetDocsumsState *state,
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
index c2bac2f6820..8418dc81372 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
@@ -198,10 +198,6 @@ void SearchVisitor::init(const Parameters & params)
static_cast<unsigned>(valueRef.size()));
LOG(debug, "Received summary class: %s", _summaryClass.c_str());
}
- if (params.get("allowslimedocsums", valueRef)) {
- _summaryGenerator.getDocsumState()._args.setFlags(search::fs4transport::GDFLAG_ALLOW_SLIME);
- LOG(debug, "Set flag: GDFLAG_ALLOW_SLIME");
- }
size_t wantedSummaryCount(10);
if (params.get("summarycount", valueRef) ) {
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h
index f2025e8c161..101fd91c5af 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h
@@ -25,6 +25,7 @@
#include <vespa/searchlib/attribute/extendableattributes.h>
#include <vespa/searchlib/common/sortspec.h>
#include <vespa/storage/visiting/visitor.h>
+#include <vespa/document/fieldvalue/fieldvalues.h>
#include <vespa/documentapi/messagebus/messages/queryresultmessage.h>
using namespace search::aggregation;
diff --git a/vsm/src/tests/docsum/docsum.cpp b/vsm/src/tests/docsum/docsum.cpp
index 4ebd7061b69..51894ec3eb1 100644
--- a/vsm/src/tests/docsum/docsum.cpp
+++ b/vsm/src/tests/docsum/docsum.cpp
@@ -5,7 +5,7 @@
#include <vespa/document/fieldvalue/fieldvalues.h>
#include <vespa/vsm/common/docsum.h>
#include <vespa/vsm/vsm/flattendocsumwriter.h>
-#include <vespa/vsm/vsm/jsondocsumwriter.h>
+#include <vespa/vsm/vsm/slimefieldwriter.h>
using namespace document;
@@ -56,15 +56,15 @@ private:
assertFlattenDocsumWriter(fdw, fv, exp);
}
void assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValue & fv, const std::string & exp);
- void assertJSONDocsumWriter(const FieldValue & fv, const std::string & exp) {
- JSONDocsumWriter jdw;
- assertJSONDocsumWriter(jdw, fv, exp);
+ void assertSlimeFieldWriter(const FieldValue & fv, const std::string & exp) {
+ SlimeFieldWriter sfw;
+ assertSlimeFieldWriter(sfw, fv, exp);
}
- void assertJSONDocsumWriter(JSONDocsumWriter & jdw, const FieldValue & fv, const std::string & exp);
+ void assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp);
void testFlattenDocsumWriter();
- void testJSONDocsumWriter();
- void requireThatJSONDocsumWriterHandlesMap();
+ void testSlimeFieldWriter();
+ void requireThatSlimeFieldWriterHandlesMap();
void testDocSumCache();
public:
@@ -104,10 +104,19 @@ DocsumTest::assertFlattenDocsumWriter(FlattenDocsumWriter & fdw, const FieldValu
}
void
-DocsumTest::assertJSONDocsumWriter(JSONDocsumWriter & jdw, const FieldValue & fv, const std::string & exp)
+DocsumTest::assertSlimeFieldWriter(SlimeFieldWriter & sfw, const FieldValue & fv, const std::string & exp)
{
- jdw.write(fv);
- EXPECT_EQUAL(jdw.getResult(), exp);
+ sfw.convert(fv);
+
+ vespalib::Slime gotSlime;
+ vespalib::Memory serialized(sfw.out());
+ size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized, gotSlime);
+ ASSERT_EQUAL(decodeRes, serialized.size);
+
+ vespalib::Slime expSlime;
+ size_t used = vespalib::slime::JsonFormat::decode(exp, expSlime);
+ EXPECT_EQUAL(exp.size(), used);
+ EXPECT_EQUAL(expSlime, gotSlime);
}
void
@@ -147,20 +156,20 @@ DocsumTest::testFlattenDocsumWriter()
}
void
-DocsumTest::testJSONDocsumWriter()
+DocsumTest::testSlimeFieldWriter()
{
{ // basic types
- assertJSONDocsumWriter(LongFieldValue(123456789), "123456789");
- assertJSONDocsumWriter(FloatFieldValue(12.34), "12.34");
- assertJSONDocsumWriter(StringFieldValue("foo bar"), "\"foo bar\"");
+ assertSlimeFieldWriter(LongFieldValue(123456789), "123456789");
+ assertSlimeFieldWriter(DoubleFieldValue(12.34), "12.34");
+ assertSlimeFieldWriter(StringFieldValue("foo bar"), "\"foo bar\"");
}
{ // collection field values
- assertJSONDocsumWriter(createFieldValue(StringList().add("foo").add("bar").add("baz")),
+ assertSlimeFieldWriter(createFieldValue(StringList().add("foo").add("bar").add("baz")),
"[\"foo\",\"bar\",\"baz\"]");
- assertJSONDocsumWriter(createFieldValue(WeightedStringList().add(std::make_pair("bar", 20)).
+ assertSlimeFieldWriter(createFieldValue(WeightedStringList().add(std::make_pair("bar", 20)).
add(std::make_pair("baz", 30)).
add(std::make_pair("foo", 10))),
- "[[\"bar\",20],[\"baz\",30],[\"foo\",10]]");
+ "[{item:\"bar\",weight:20},{item:\"baz\",weight:30},{item:\"foo\",weight:10}]");
}
{ // struct field value
StructDataType subType("substruct");
@@ -186,36 +195,39 @@ DocsumTest::testJSONDocsumWriter()
{ // select a subset and then all
- JSONDocsumWriter jdw;
+ SlimeFieldWriter sfw;
DocsumFieldSpec::FieldIdentifierVector fields;
fields.push_back(DocsumFieldSpec::FieldIdentifier(
0, *type.buildFieldPath("a")));
fields.push_back(DocsumFieldSpec::FieldIdentifier(
0, *type.buildFieldPath("c.e")));
- jdw.setInputFields(fields);
- assertJSONDocsumWriter(jdw, value, "{\"a\":\"foo\",\"c\":{\"e\":\"qux\"}}");
- jdw.clear();
- assertJSONDocsumWriter(jdw, value, "{\"a\":\"foo\",\"b\":\"bar\",\"c\":{\"d\":\"baz\",\"e\":\"qux\"}}");
+ sfw.setInputFields(fields);
+ assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"c\":{\"e\":\"qux\"}}");
+ sfw.clear();
+ assertSlimeFieldWriter(sfw, value, "{\"a\":\"foo\",\"b\":\"bar\",\"c\":{\"d\":\"baz\",\"e\":\"qux\"}}");
}
- }
+
{ // multiple invocations
- JSONDocsumWriter jdw;
- assertJSONDocsumWriter(jdw, StringFieldValue("foo"), "\"foo\"");
- assertJSONDocsumWriter(jdw, StringFieldValue("bar"), "\"foo\"\"bar\"");
- jdw.clear();
- assertJSONDocsumWriter(jdw, StringFieldValue("baz"), "\"baz\"");
+ SlimeFieldWriter sfw;
+ assertSlimeFieldWriter(sfw, StringFieldValue("foo"), "\"foo\"");
+ sfw.clear();
+ assertSlimeFieldWriter(sfw, StringFieldValue("bar"), "\"bar\"");
+ sfw.clear();
+ assertSlimeFieldWriter(sfw, StringFieldValue("baz"), "\"baz\"");
+ }
+
}
}
void
-DocsumTest::requireThatJSONDocsumWriterHandlesMap()
+DocsumTest::requireThatSlimeFieldWriterHandlesMap()
{
{ // map<string, string>
MapDataType mapType(*DataType::STRING, *DataType::STRING);
MapFieldValue mapfv(mapType);
EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), StringFieldValue("v1")));
EXPECT_TRUE(mapfv.put(StringFieldValue("k2"), StringFieldValue("v2")));
- assertJSONDocsumWriter(mapfv, "[{\"key\":\"k1\",\"value\":\"v1\"},{\"key\":\"k2\",\"value\":\"v2\"}]");
+ assertSlimeFieldWriter(mapfv, "[{\"key\":\"k1\",\"value\":\"v1\"},{\"key\":\"k2\",\"value\":\"v2\"}]");
}
{ // map<string, struct>
StructDataType structType("struct");
@@ -230,17 +242,17 @@ DocsumTest::requireThatJSONDocsumWriterHandlesMap()
MapFieldValue mapfv(mapType);
EXPECT_TRUE(mapfv.put(StringFieldValue("k1"), structValue));
{ // select a subset and then all
- JSONDocsumWriter jdw;
+ SlimeFieldWriter sfw;
DocsumFieldSpec::FieldIdentifierVector fields;
fields.push_back(DocsumFieldSpec::FieldIdentifier(0, *mapType.buildFieldPath("value.b")));
- jdw.setInputFields(fields);
- assertJSONDocsumWriter(jdw, mapfv, "[{\"key\":\"k1\",\"value\":{\"b\":\"bar\"}}]");
+ sfw.setInputFields(fields);
+ assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"b\":\"bar\"}}]");
fields[0] = DocsumFieldSpec::FieldIdentifier(0, *mapType.buildFieldPath("{k1}.a"));
- jdw.clear();
- jdw.setInputFields(fields);
- assertJSONDocsumWriter(jdw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\"}}]");
- jdw.clear(); // all fields implicit
- assertJSONDocsumWriter(jdw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\",\"b\":\"bar\"}}]");
+ sfw.clear();
+ sfw.setInputFields(fields);
+ assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\"}}]");
+ sfw.clear(); // all fields implicit
+ assertSlimeFieldWriter(sfw, mapfv, "[{\"key\":\"k1\",\"value\":{\"a\":\"foo\",\"b\":\"bar\"}}]");
}
}
}
@@ -251,8 +263,8 @@ DocsumTest::Main()
TEST_INIT("docsum_test");
testFlattenDocsumWriter();
- testJSONDocsumWriter();
- requireThatJSONDocsumWriterHandlesMap();
+ testSlimeFieldWriter();
+ requireThatSlimeFieldWriterHandlesMap();
TEST_DONE();
}
diff --git a/vsm/src/vespa/vsm/vsm/CMakeLists.txt b/vsm/src/vespa/vsm/vsm/CMakeLists.txt
index 237b0771c95..9df5f1c2957 100644
--- a/vsm/src/vespa/vsm/vsm/CMakeLists.txt
+++ b/vsm/src/vespa/vsm/vsm/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_library(vsm_vsmbase OBJECT
docsumfilter.cpp
fieldsearchspec.cpp
flattendocsumwriter.cpp
- jsondocsumwriter.cpp
slimefieldwriter.cpp
snippetmodifier.cpp
vsm-adapter.cpp
diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
index eb4e0bf532f..e858c8ca8a6 100644
--- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
+++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
@@ -1,11 +1,9 @@
// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "docsumfilter.h"
-#include "jsondocsumwriter.h"
#include "slimefieldwriter.h"
#include <vespa/searchsummary/docsummary/resultclass.h>
#include <vespa/searchsummary/docsummary/summaryfieldconverter.h>
-#include <vespa/vespalib/util/jsonwriter.h>
#include <vespa/document/fieldvalue/literalfieldvalue.h>
#include <vespa/document/base/exceptions.h>
#include <vespa/log/log.h>
@@ -216,7 +214,6 @@ DocsumFilter::DocsumFilter(const DocsumToolsPtr &tools, const IDocSumCache & doc
_fields(),
_highestFieldNo(0),
_packer(tools.get() ? tools->getResultConfig() : NULL),
- _jsonWriter(),
_flattenWriter(),
_snippetModifiers(NULL),
_cachedValue(),
@@ -316,33 +313,6 @@ DocsumFilter::writeField(const document::FieldValue & fv, const FieldPath & path
}
}
-void
-DocsumFilter::writeJSONField(const DocsumFieldSpec & fieldSpec,
- const Document & docsum,
- ResultPacker & packer)
-{
- if (fieldSpec.getCommand() == VsmsummaryConfig::Fieldmap::NONE) {
- const DocsumFieldSpec::FieldIdentifier & fieldId = fieldSpec.getOutputField();
- const document::FieldValue * fv = docsum.getField(fieldId.getId());
- if (fv != NULL) {
- LOG(debug, "writeJSONField: About to write field '%d' as JSONSTRING: field value = '%s'",
- fieldId.getId(), fv->toString().c_str());
- if (!fieldSpec.hasIdentityMapping()) {
- _jsonWriter.setInputFields(fieldSpec.getInputFields());
- }
- _jsonWriter.write(*fv);
- vespalib::string s = _jsonWriter.getResult();
- packer.AddLongString(s.c_str(), s.size());
- _jsonWriter.clear();
- } else {
- LOG(debug, "writeJSONField: Field value not set for field '%d'", fieldId.getId());
- packer.AddEmpty();
- }
- } else {
- LOG(debug, "writeJSONField: Cannot handle this command");
- packer.AddEmpty();
- }
-}
void
DocsumFilter::writeSlimeField(const DocsumFieldSpec & fieldSpec,
@@ -467,7 +437,7 @@ DocsumFilter::getSummaryClassId() const
}
DocsumStoreValue
-DocsumFilter::getMappedDocsum(uint32_t id, bool useSlimeInsideFields)
+DocsumFilter::getMappedDocsum(uint32_t id)
{
const ResultClass *resClass = _tools->getResultClass();
if (resClass == NULL) {
@@ -476,17 +446,12 @@ DocsumFilter::getMappedDocsum(uint32_t id, bool useSlimeInsideFields)
const Document & doc = _docsumCache->getDocSum(id);
- LOG(spam, "getMappedDocsum %u [useSlimeInsideFields=%s]", id, useSlimeInsideFields ? "true" : "false");
-
_packer.Init(resClass->GetClassID());
for (FieldSpecList::iterator it(_fields.begin()), end = _fields.end(); it != end; ++it) {
ResType type = it->getResultType();
if (type == RES_JSONSTRING || type == RES_XMLSTRING) {
- if (useSlimeInsideFields) {
- writeSlimeField(*it, doc, _packer);
- } else {
- writeJSONField(*it, doc, _packer);
- }
+ // this really means 'structured data'
+ writeSlimeField(*it, doc, _packer);
} else {
if (it->getInputFields().size() == 1 && it->getCommand() == VsmsummaryConfig::Fieldmap::NONE) {
const DocsumFieldSpec::FieldIdentifier & fieldId = it->getInputFields()[0];
diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.h b/vsm/src/vespa/vsm/vsm/docsumfilter.h
index 370843a3789..d39526a2575 100644
--- a/vsm/src/vespa/vsm/vsm/docsumfilter.h
+++ b/vsm/src/vespa/vsm/vsm/docsumfilter.h
@@ -9,7 +9,6 @@
#include <vespa/vsm/vsm/docsumfieldspec.h>
#include <vespa/vsm/vsm/fieldsearchspec.h>
#include <vespa/vsm/vsm/flattendocsumwriter.h>
-#include <vespa/vsm/vsm/jsondocsumwriter.h>
#include <vespa/vsm/vsm/vsm-adapter.h>
#include <vespa/searchsummary/docsummary/resultpacker.h>
#include <vespa/searchsummary/docsummary/docsumstore.h>
@@ -38,7 +37,6 @@ private:
FieldSpecList _fields; // list of summary fields to generate
size_t _highestFieldNo;
ResultPacker _packer;
- JSONDocsumWriter _jsonWriter;
FlattenDocsumWriter _flattenWriter;
const FieldModifierMap * _snippetModifiers;
document::FieldValue::UP _cachedValue;
@@ -52,7 +50,6 @@ private:
VsmsummaryConfig::Fieldmap::Command command,
const Document & docsum, bool & modified);
void writeField(const document::FieldValue & fv, const FieldPath & path, ResType type, ResultPacker & packer);
- void writeJSONField(const DocsumFieldSpec & fieldSpec, const Document & docsum, ResultPacker & packer);
void writeSlimeField(const DocsumFieldSpec & fieldSpec, const Document & docsum, ResultPacker & packer);
void writeFlattenField(const DocsumFieldSpec & fieldSpec, const Document & docsum, ResultPacker & packer);
void writeEmpty(ResType type, ResultPacker & packer);
@@ -86,7 +83,7 @@ public:
void setDocSumStore(const IDocSumCache & docsumCache) { _docsumCache = &docsumCache; }
// Inherit doc from IDocsumStore
- DocsumStoreValue getMappedDocsum(uint32_t id, bool useSlimeInsideFields) override;
+ DocsumStoreValue getMappedDocsum(uint32_t id) override;
uint32_t getNumDocs() const override;
uint32_t getSummaryClassId() const override;
};
diff --git a/vsm/src/vespa/vsm/vsm/jsondocsumwriter.cpp b/vsm/src/vespa/vsm/vsm/jsondocsumwriter.cpp
deleted file mode 100644
index e05f69d572a..00000000000
--- a/vsm/src/vespa/vsm/vsm/jsondocsumwriter.cpp
+++ /dev/null
@@ -1,174 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "jsondocsumwriter.h"
-#include <vespa/vespalib/stllike/asciistream.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".vsm.jsondocsumwriter");
-
-namespace {
-
-vespalib::string
-toString(const vsm::FieldPath & fp)
-{
- vespalib::asciistream oss;
- for (size_t i = 0; i < fp.size(); ++i) {
- if (i > 0) {
- oss << ".";
- }
- oss << fp[i].getName();
- }
- return oss.str();
-}
-
-}
-
-namespace vsm {
-
-void
-JSONDocsumWriter::traverseRecursive(const document::FieldValue & fv)
-{
- LOG(debug, "traverseRecursive: class(%s), fieldValue(%s), currentPath(%s)",
- fv.getClass().name(), fv.toString().c_str(), toString(_currPath).c_str());
-
- if (fv.getClass().inherits(document::CollectionFieldValue::classId)) {
- const document::CollectionFieldValue & cfv = static_cast<const document::CollectionFieldValue &>(fv);
- if (cfv.inherits(document::ArrayFieldValue::classId)) {
- const document::ArrayFieldValue & afv = static_cast<const document::ArrayFieldValue &>(cfv);
- _output.beginArray();
- for (size_t i = 0; i < afv.size(); ++i) {
- const document::FieldValue & nfv = afv[i];
- traverseRecursive(nfv);
- }
- _output.endArray();
- } else if (cfv.inherits(document::WeightedSetFieldValue::classId)) {
- const document::WeightedSetFieldValue & wsfv = static_cast<const document::WeightedSetFieldValue &>(cfv);
- _output.beginArray();
- for (document::WeightedSetFieldValue::const_iterator itr = wsfv.begin(); itr != wsfv.end(); ++itr) {
- const document::FieldValue & nfv = *itr->first;
- _output.beginArray();
- traverseRecursive(nfv);
- _output.appendInt64(static_cast<const document::IntFieldValue&>(*itr->second).getValue());
- _output.endArray();
- }
- _output.endArray();
- } else {
- LOG(warning, "traverseRecursive: Cannot handle collection field value of type '%s'",
- fv.getClass().name());
- }
-
- } else if (fv.getClass().inherits(document::MapFieldValue::classId)) {
- const document::MapFieldValue & mfv = static_cast<const document::MapFieldValue &>(fv);
- _output.beginArray();
- for (document::MapFieldValue::const_iterator itr = mfv.begin(); itr != mfv.end(); ++itr) {
- _output.beginObject();
- _output.appendKey("key");
- traverseRecursive(*itr->first);
- _output.appendKey("value");
- const document::MapDataType& mapType = static_cast<const document::MapDataType &>(*mfv.getDataType());
- document::FieldPathEntry valueEntry(
- mapType, mapType.getKeyType(), mapType.getValueType(),
- false, true);
- _currPath.push_back(valueEntry);
- traverseRecursive(*itr->second);
- _currPath.pop_back();
- _output.endObject();
- }
- _output.endArray();
- } else if (fv.getClass().inherits(document::StructuredFieldValue::classId)) {
- const document::StructuredFieldValue & sfv = static_cast<const document::StructuredFieldValue &>(fv);
- _output.beginObject();
- for (document::StructuredFieldValue::const_iterator itr = sfv.begin(); itr != sfv.end(); ++itr) {
- // TODO: Why do we have to iterate like this?
- document::FieldPathEntry fi(sfv.getField(itr.field().getName()));
- _currPath.push_back(fi);
- if (explorePath()) {
- _output.appendKey(itr.field().getName());
- document::FieldValue::UP fval(sfv.getValue(itr.field()));
- traverseRecursive(*fval);
- }
- _currPath.pop_back();
- }
- _output.endObject();
-
- } else {
- if (fv.getClass().inherits(document::LiteralFieldValueB::classId)) {
- const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv);
- _output.appendString(lfv.getValueRef());
- } else if (fv.getClass().inherits(document::NumericFieldValueBase::classId)) {
- switch (fv.getDataType()->getId()) {
- case document::DataType::T_BYTE:
- case document::DataType::T_SHORT:
- case document::DataType::T_INT:
- case document::DataType::T_LONG:
- _output.appendInt64(fv.getAsLong());
- break;
- case document::DataType::T_FLOAT:
- _output.appendFloat(fv.getAsFloat());
- break;
- case document::DataType::T_DOUBLE:
- _output.appendDouble(fv.getAsFloat());
- break;
- default:
- _output.appendString(fv.getAsString());
- }
- } else {
- _output.appendString(fv.toString());
- }
- }
-}
-
-bool
-JSONDocsumWriter::explorePath()
-{
- if (_inputFields == NULL) {
- return true;
- }
- // find out if we should explore the current path
- for (size_t i = 0; i < _inputFields->size(); ++i) {
- const FieldPath & fp = (*_inputFields)[i].getPath();
- if (_currPath.size() <= fp.size()) {
- bool equal = true;
- for (size_t j = 0; j < _currPath.size() && equal; ++j) {
- equal = (fp[j].getName() == _currPath[j].getName());
- }
- if (equal) {
- // the current path matches one of the input field paths
- return true;
- }
- }
- }
- return false;
-}
-
-JSONDocsumWriter::JSONDocsumWriter() :
- _output(),
- _inputFields(NULL),
- _currPath()
-{
-}
-
-void
-JSONDocsumWriter::write(const document::FieldValue & fv)
-{
- if (LOG_WOULD_LOG(debug)) {
- if (_inputFields != NULL) {
- for (size_t i = 0; i < _inputFields->size(); ++i) {
- LOG(debug, "write: input field path [%zd] '%s'", i, toString((*_inputFields)[i].getPath()).c_str());
- }
- } else {
- LOG(debug, "write: no input fields");
- }
- }
- traverseRecursive(fv);
-}
-
-void
-JSONDocsumWriter::clear()
-{
- _output.clear();
- _inputFields = NULL;
- _currPath.clear();
-}
-
-}
diff --git a/vsm/src/vespa/vsm/vsm/jsondocsumwriter.h b/vsm/src/vespa/vsm/vsm/jsondocsumwriter.h
deleted file mode 100644
index 569d3a088d7..00000000000
--- a/vsm/src/vespa/vsm/vsm/jsondocsumwriter.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <vespa/document/fieldvalue/fieldvalues.h>
-#include <vespa/vespalib/util/jsonwriter.h>
-#include <vespa/vsm/common/storagedocument.h>
-#include <vespa/vsm/vsm/docsumfieldspec.h>
-
-namespace vsm {
-
-/**
- * This class is used to write a field value as a json string.
- * If only a subset of the field value should be written this subset
- * is specified using the setInputFields() function.
- **/
-class JSONDocsumWriter
-{
-private:
- vespalib::JSONStringer _output;
- const DocsumFieldSpec::FieldIdentifierVector * _inputFields;
- FieldPath _currPath;
-
- void traverseRecursive(const document::FieldValue & fv);
- bool explorePath();
-
-public:
- JSONDocsumWriter();
-
- /**
- * Specifies the subset of the field value that should be written.
- **/
- void setInputFields(const DocsumFieldSpec::FieldIdentifierVector & inputFields) { _inputFields = &inputFields; }
-
- /**
- * Writes the given field value using the underlying JSONStringer.
- **/
- void write(const document::FieldValue & fv);
-
- /**
- * Returns the result as a string.
- **/
- vespalib::string getResult() { return _output.toString(); }
-
- /**
- * Clears this instance such that it is ready to write a new field value.
- **/
- void clear();
-};
-
-}
-
diff --git a/vsm/src/vespa/vsm/vsm/slimefieldwriter.h b/vsm/src/vespa/vsm/vsm/slimefieldwriter.h
index 0d2856e3e9b..ac540b2ccfe 100644
--- a/vsm/src/vespa/vsm/vsm/slimefieldwriter.h
+++ b/vsm/src/vespa/vsm/vsm/slimefieldwriter.h
@@ -44,6 +44,12 @@ public:
vespalib::stringref out() const {
return vespalib::stringref(_rbuf.GetDrainPos(), _rbuf.GetUsedLen());
}
+
+ void clear() {
+ _rbuf.Reuse();
+ _inputFields = NULL;
+ _currPath.clear();
+ }
};
}