diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-09-30 15:01:39 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-09-30 15:01:39 +0200 |
| commit | 61fea291708fb74113d5f98480a628b0bc7695f3 (patch) | |
| tree | 91fd393a4678cedb1e7edbc589a78b0eeb61755a /searchlib | |
| parent | 4e4ec099edd99775066ab416b35d6716479e3dde (diff) | |
| parent | 082ed671d24b1197580bc265712ae9821a65e481 (diff) | |
Merge pull request #10804 from vespa-engine/balder/remove-leftover-fs4-code
Balder/remove leftover fs4 code
Diffstat (limited to 'searchlib')
32 files changed, 29 insertions, 5321 deletions
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 449580e577b..b78f4fc89ac 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -101,7 +101,6 @@ vespa_define_module( src/tests/common/foregroundtaskexecutor src/tests/common/location src/tests/common/matching_elements - src/tests/common/packets src/tests/common/resultset src/tests/common/sequencedtaskexecutor src/tests/common/struct_field_mapper @@ -119,12 +118,8 @@ vespa_define_module( src/tests/docstore/lid_info src/tests/docstore/logdatastore src/tests/docstore/store_by_bucket - src/tests/engine/docsumapi - src/tests/engine/monitorapi src/tests/engine/proto_converter src/tests/engine/proto_rpc_adapter - src/tests/engine/searchapi - src/tests/engine/transportserver src/tests/expression/attributenode src/tests/features src/tests/features/beta diff --git a/searchlib/src/tests/common/packets/.gitignore b/searchlib/src/tests/common/packets/.gitignore deleted file mode 100644 index e3dcf5376d5..00000000000 --- a/searchlib/src/tests/common/packets/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -packets_test -searchlib_packets_test_app diff --git a/searchlib/src/tests/common/packets/CMakeLists.txt b/searchlib/src/tests/common/packets/CMakeLists.txt deleted file mode 100644 index 12c4b2cd6f5..00000000000 --- a/searchlib/src/tests/common/packets/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_packets_test_app TEST - SOURCES - packets_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_packets_test_app COMMAND searchlib_packets_test_app) diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp deleted file mode 100644 index 35ca7bc1dd9..00000000000 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ /dev/null @@ -1,663 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchlib/common/mapnames.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/controlpacket.h> - -using namespace search::fs4transport; -using vespalib::compression::CompressionConfig; - -#define PCODE_BEGIN PCODE_EOL -#define PCODE_END PCODE_LastCode - -class MyPersistentPacketStreamer : public FS4PersistentPacketStreamer { -public: - MyPersistentPacketStreamer() : - FS4PersistentPacketStreamer(FS4PacketFactory::CreateFS4Packet) { - // empty - } - - uint32_t getChannelId(uint32_t pcode, uint32_t chid) { - return HasChannelID(pcode) ? chid : -1u; - } -}; - -FNET_Packet * -testEncodeDecode(FS4PersistentPacketStreamer &streamer, FNET_Packet &packet) -{ - FNET_Context ctx; - FNET_DataBuffer buf; - buf.WriteInt32(0xdeadbeef); // buffers can have extra data at the front. - streamer.Encode(&packet, 1u, &buf); - buf.DataToDead(sizeof(uint32_t)); - - FNET_DataBuffer lhs; - lhs.WriteBytes(buf.GetData(), buf.GetDataLen()); - - buf.WriteInt32(0xdeadbeef); // buffers can have extra data at the end. - - bool broken; - uint32_t plen, pcode, chid; - MyPersistentPacketStreamer myStreamer; - EXPECT_TRUE(streamer.GetPacketInfo(&buf, &plen, &pcode, &chid, &broken)); - if ((pcode & ~PCODE_MASK) == 0) { - EXPECT_EQUAL(packet.GetLength(), plen); - } - EXPECT_EQUAL(packet.GetPCODE() & PCODE_MASK, pcode & PCODE_MASK); - EXPECT_EQUAL(myStreamer.getChannelId(pcode, 1u), chid); - - FNET_Packet *ret = streamer.Decode(&buf, plen, pcode, ctx); - assert(ret); - if (ret->GetPCODE() == (pcode & PCODE_MASK)) { - FNET_DataBuffer rhs; - streamer.Encode(ret, 1u, &rhs); - if (!EXPECT_TRUE(lhs.Equals(&rhs))) { - lhs.HexDump(); - rhs.HexDump(); - } - } else { - // Packet was transcoded. - } - return ret; -} - -FNET_Packet * -testEncodeDecode(FNET_Packet &packet) -{ - return testEncodeDecode(FS4PersistentPacketStreamer::Instance, packet); -} - -void fillProperties(FS4Properties &props, const std::string &name, - uint32_t len) { - props.setName(name); - props.allocEntries(len); - for (uint32_t i = 0; i < len; ++i) { - std::string key = vespalib::make_string("key%d", i); - props.setKey(i, key); - - std::string val = vespalib::make_string("val%d", i); - props.setValue(i, val); - } -} - -void testProperties(FS4Properties &props, const std::string &name, - uint32_t len) { - EXPECT_EQUAL(name, props.getName()); - EXPECT_EQUAL(name.size(), props.getNameLen()); - for (uint32_t i = 0; i < len; ++i) { - std::string key = vespalib::make_string("key%d", i); - EXPECT_EQUAL(key, std::string(props.getKey(i), props.getKeyLen(i))); - - std::string val = vespalib::make_string("val%d", i); - EXPECT_EQUAL(val, - std::string(props.getValue(i), props.getValueLen(i))); - } -} - - -// ---------------------------------------------------------------------------- -// -// Tests -// -// ---------------------------------------------------------------------------- - -document::GlobalId gid0("aaaaaaaaaaaa"); -document::GlobalId gid1("bbbbbbbbbbbb"); - -TEST("testPacketArray") { - PacketArray arr; - for (uint32_t i = 0; i < 32; ++i) { - EXPECT_EQUAL(i, arr.Length()); - arr.Add(new FNET_ControlPacket(i)); - EXPECT_EQUAL(i, static_cast<FNET_ControlPacket&>(*arr.Array()[i]).GetCommand()); - } - for (uint32_t i = 0; i < arr.Length(); ++i) { - delete static_cast<FNET_ControlPacket *>(arr.Array()[i]); - } -} - -TEST("testPacketFactory") { - ASSERT_TRUE(FS4PacketFactory::CreateFS4Packet(PCODE_BEGIN - 1) == NULL); - - ASSERT_TRUE(FS4PacketFactory::CreateFS4Packet(PCODE_END) == NULL); - - for (uint32_t pcode = PCODE_BEGIN; pcode < PCODE_END; ++pcode) { - if ((pcode != PCODE_MLD_QUERYRESULT2_NOTUSED) && - (pcode != PCODE_QUERY_NOTUSED) && - (pcode != PCODE_MONITORQUERY_NOTUSED) && - (pcode != PCODE_GETDOCSUMS_NOTUSED) && - (pcode != PCODE_MLD_GETDOCSUMS_NOTUSED) && - (pcode != PCODE_QUERYRESULT_NOTUSED) && - (pcode != PCODE_MLD_QUERYRESULT_NOTUSED) && - (pcode != PCODE_MONITORRESULT_NOTUSED) && - (pcode != PCODE_MLD_MONITORRESULT_NOTUSED) && - (pcode != PCODE_CLEARCACHES_NOTUSED) && - (pcode != PCODE_PARSEDQUERY2_NOTUSED) && - (pcode != PCODE_QUEUELEN_NOTUSED) && - (pcode != PCODE_QUERY2_NOTUSED) && - (pcode != PCODE_MLD_GETDOCSUMS2_NOTUSED)) - { - std::unique_ptr<FNET_Packet> aptr(FS4PacketFactory::CreateFS4Packet(pcode)); - ASSERT_TRUE(aptr.get() != NULL); - EXPECT_EQUAL(pcode, aptr->GetPCODE()); - } - } -} - -TEST("testPersistentPacketStreamer") { - for (uint32_t pcode = PCODE_BEGIN; pcode < PCODE_END; ++pcode) { - if ((pcode == PCODE_QUERYX) || - (pcode != PCODE_MLD_QUERYRESULT2_NOTUSED) || - (pcode != PCODE_MLD_GETDOCSUMS2_NOTUSED)) - { - continue; - } - std::unique_ptr<FNET_Packet> arg(FS4PacketFactory::CreateFS4Packet(pcode)); - std::unique_ptr<FNET_Packet> ret(testEncodeDecode(FS4PersistentPacketStreamer::Instance, *arg)); - EXPECT_TRUE(ret.get() != NULL); - - FNET_Packet *raw = testEncodeDecode(FS4PersistentPacketStreamer::Instance, - *FS4PacketFactory::CreateFS4Packet(pcode)); - EXPECT_TRUE(raw != NULL); - } -} - -TEST("testProperties") { - FS4Properties src; - fillProperties(src, "foo", 32u); - testProperties(src, "foo", 32u); - - FNET_DataBuffer buf; - src.encode(buf); - FNET_DataBuffer lhs; - lhs.WriteBytes(buf.GetData(), buf.GetDataLen()); - - uint32_t len = buf.GetDataLen(); - FS4Properties dst; - dst.decode(buf, len); - EXPECT_EQUAL(src.getLength(), dst.getLength()); - - testProperties(dst, "foo", 32u); - - FNET_DataBuffer rhs; - dst.encode(rhs); - EXPECT_TRUE(lhs.Equals(&rhs)); -} - -TEST("testEol") { - FS4Packet_EOL *src = dynamic_cast<FS4Packet_EOL*>(FS4PacketFactory::CreateFS4Packet(PCODE_EOL)); - ASSERT_TRUE(src != NULL); - - std::vector<FNET_Packet*> lst { src, testEncodeDecode(*src) }; - - for (FNET_Packet * packet : lst) { - FS4Packet_EOL *ptr = dynamic_cast<FS4Packet_EOL*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_EOL, ptr->GetPCODE()); - EXPECT_EQUAL(0u, ptr->GetLength()); - - delete ptr; - } -} - -TEST("testError") { - FS4Packet_ERROR *src = dynamic_cast<FS4Packet_ERROR*>(FS4PacketFactory::CreateFS4Packet(PCODE_ERROR)); - ASSERT_TRUE(src != NULL); - src->_errorCode = 1u; - src->setErrorMessage("foo"); - - std::vector<FNET_Packet*> lst { src, testEncodeDecode(*src) }; - - for (FNET_Packet * packet : lst) { - FS4Packet_ERROR *ptr = dynamic_cast<FS4Packet_ERROR*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_ERROR, ptr->GetPCODE()); - EXPECT_EQUAL(11u, ptr->GetLength()); - EXPECT_EQUAL(1u, ptr->_errorCode); - EXPECT_EQUAL("foo", ptr->_message); - - delete ptr; - } -} - -TEST("testDocsum") { - FS4Packet_DOCSUM *src = dynamic_cast<FS4Packet_DOCSUM*>(FS4PacketFactory::CreateFS4Packet(PCODE_DOCSUM)); - ASSERT_TRUE(src != NULL); - src->setGid(gid0); - src->SetBuf("foo", 3u); - - std::vector<FNET_Packet*> lst { src, testEncodeDecode(*src) }; - - for (FNET_Packet * packet : lst) { - FS4Packet_DOCSUM *ptr = dynamic_cast<FS4Packet_DOCSUM*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_DOCSUM, ptr->GetPCODE()); - EXPECT_EQUAL(3u + 12u, ptr->GetLength()); - EXPECT_EQUAL(gid0, ptr->getGid()); - EXPECT_EQUAL("foo", std::string(ptr->getBuf().c_str(), ptr->getBuf().size())); - - delete ptr; - } -} - -TEST("testMonitorQueryX") { - FS4Packet_MONITORQUERYX *src = dynamic_cast<FS4Packet_MONITORQUERYX*>(FS4PacketFactory::CreateFS4Packet(PCODE_MONITORQUERYX)); - ASSERT_TRUE(src != NULL); - src->_qflags = 1u; - - std::vector<FNET_Packet*> lst; - for (uint32_t i = MQF_QFLAGS, len = (uint32_t)(MQF_QFLAGS << 1); i < len; ++i) { - if (i & ~FNET_MQF_SUPPORTED_MASK) { - continue; // not supported; - } - src->_features = i; - lst.push_back(testEncodeDecode(*src)); - } - src->_features = (uint32_t)-1; - lst.push_back(src); - - for (FNET_Packet * packet : lst) { - FS4Packet_MONITORQUERYX *ptr = dynamic_cast<FS4Packet_MONITORQUERYX*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_MONITORQUERYX, ptr->GetPCODE()); - EXPECT_EQUAL(ptr->_features & MQF_QFLAGS ? 1u : 0u, ptr->_qflags); - - delete ptr; - } -} - -TEST("testMonitorResultX") { - FS4Packet_MONITORRESULTX *src = dynamic_cast<FS4Packet_MONITORRESULTX*>(FS4PacketFactory::CreateFS4Packet(PCODE_MONITORRESULTX)); - ASSERT_TRUE(src != NULL); - src->_partid = 1u; - src->_timestamp = 2u; - src->_totalNodes = 3u; - src->_activeNodes = 4u; - src->_totalParts = 5u; - src->_activeParts = 6u; - src->_rflags = 7u; - - std::vector<FNET_Packet*> lst; - for (uint32_t i = MRF_MLD, len = (uint32_t)(MRF_RFLAGS << 1); i < len; ++i) { - if (i & ~FNET_MRF_SUPPORTED_MASK) { - continue; // not supported; - } - src->_features = i; - lst.push_back(testEncodeDecode(*src)); - } - src->_features = (uint32_t)-1; - lst.push_back(src); - - for (FNET_Packet * packet : lst) { - FS4Packet_MONITORRESULTX *ptr = dynamic_cast<FS4Packet_MONITORRESULTX*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_MONITORRESULTX, ptr->GetPCODE()); - EXPECT_EQUAL(1u, ptr->_partid); - EXPECT_EQUAL(2u, ptr->_timestamp); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 3u : 0u, ptr->_totalNodes); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 4u : 0u, ptr->_activeNodes); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 5u : 0u, ptr->_totalParts); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 6u : 0u, ptr->_activeParts); - EXPECT_EQUAL(ptr->_features & MRF_RFLAGS ? 7u : 0u, ptr->_rflags); - - delete ptr; - } -} - -TEST("testQueryResultX") { - FS4Packet_QUERYRESULTX *src = dynamic_cast<FS4Packet_QUERYRESULTX*>(FS4PacketFactory::CreateFS4Packet(PCODE_QUERYRESULTX)); - ASSERT_TRUE(src != NULL); - src->_offset = 1u; - src->_totNumDocs = 2u; - src->_maxRank = (search::HitRank)3; - src->setDistributionKey(4u); - src->_coverageDocs = 6u; - src->_activeDocs = 7u; - src->_soonActiveDocs = 8; - src->_coverageDegradeReason = 0x17; - src->setNodesQueried(12); - src->setNodesReplied(11); - uint32_t sortIndex[3] = { 0u, 1u, 3u /* size of data */}; // numDocs + 1 - src->SetSortDataRef(2, sortIndex, "foo"); - src->SetGroupDataRef("baz", 3u); - src->AllocateHits(2); - src->_hits[0]._gid = gid0; - src->_hits[0]._metric = (search::HitRank)2; - src->_hits[0]._partid = 3u; - src->_hits[0].setDistributionKey(4u); - src->_hits[1]._gid = gid1; - src->_hits[1]._metric = (search::HitRank)3; - src->_hits[1]._partid = 4u; - src->_hits[1].setDistributionKey(5u); - - std::vector<FNET_Packet*> lst; - for (uint32_t i = QRF_MLD, len = (uint32_t)(QRF_GROUPDATA << 1); i < len; ++i) { - if (i & ~FNET_QRF_SUPPORTED_MASK) { - continue; // not supported; - } - src->_features = i; - lst.push_back(testEncodeDecode(*src)); - } - src->_features = (uint32_t)-1; - lst.push_back(src); - - for (FNET_Packet * packet : lst) { - FS4Packet_QUERYRESULTX *ptr = dynamic_cast<FS4Packet_QUERYRESULTX*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_QUERYRESULTX, ptr->GetPCODE()); - - EXPECT_EQUAL(1u, ptr->_offset); - EXPECT_EQUAL(2u, ptr->_totNumDocs); - EXPECT_EQUAL((search::HitRank)3, ptr->_maxRank); - EXPECT_EQUAL(4u, ptr->getDistributionKey()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 1u, ptr->getNodesQueried()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 1u, ptr->getNodesReplied()); - EXPECT_EQUAL(6u, ptr->_coverageDocs); - EXPECT_EQUAL(7u, ptr->_activeDocs); - EXPECT_EQUAL(8u, ptr->_soonActiveDocs); - EXPECT_EQUAL(0x17u, ptr->_coverageDegradeReason); - if (ptr->_features & QRF_SORTDATA) { - EXPECT_EQUAL(0u, ptr->_sortIndex[0]); - EXPECT_EQUAL(1u, ptr->_sortIndex[1]); - EXPECT_EQUAL(3u, ptr->_sortIndex[2]); - EXPECT_EQUAL("foo", std::string(ptr->_sortData, ptr->_sortIndex[2])); - } else { - EXPECT_EQUAL((void*)NULL, ptr->_sortIndex); - EXPECT_EQUAL((void*)NULL, ptr->_sortData); - } - if (ptr->_features & QRF_GROUPDATA) { - EXPECT_EQUAL("baz", std::string(ptr->_groupData, ptr->_groupDataLen)); - } else { - EXPECT_EQUAL(0u, ptr->_groupDataLen); - EXPECT_EQUAL((void*)NULL, ptr->_groupData); - } - EXPECT_EQUAL(2u, ptr->_numDocs); - for (uint32_t i = 0; i < ptr->_numDocs; ++i) { - EXPECT_EQUAL(i == 0 ? gid0 : gid1, ptr->_hits[i]._gid); - EXPECT_EQUAL((search::HitRank)2 + i, ptr->_hits[i]._metric); - EXPECT_EQUAL(ptr->_features & QRF_MLD ? 3u + i : 0u, ptr->_hits[i]._partid); - EXPECT_EQUAL(ptr->_features & QRF_MLD ? 4u + i : ptr->getDistributionKey(), ptr->_hits[i].getDistributionKey()); - } - - delete ptr; - } -} - -FS4Packet_QUERYX * -createAndFill_QUERYX() -{ - FS4Packet_QUERYX *src = dynamic_cast<FS4Packet_QUERYX*>(FS4PacketFactory::CreateFS4Packet(PCODE_QUERYX)); - ASSERT_TRUE(src != NULL); - src->_offset = 2u; - src->_maxhits = 3u; - src->setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), src->getTimeout()); - src->setTimeout(fastos::TimeStamp(-4*fastos::TimeStamp::MS)); - EXPECT_EQUAL(0l, src->getTimeout()); - src->setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), src->getTimeout()); - src->setQueryFlags(5u); - src->setRanking("seven"); - src->_numStackItems = 14u; - src->_propsVector.resize(2); - fillProperties(src->_propsVector[0], "foo", 8); - fillProperties(src->_propsVector[1], "bar", 16); - src->setSortSpec("sortspec"); - src->setGroupSpec("groupspec"); - src->setLocation("location"); - src->setStackDump("stackdump"); - return src; -} - -void -verifyQueryX(FS4Packet_QUERYX & queryX, uint32_t features) -{ - EXPECT_EQUAL((uint32_t)PCODE_QUERYX, queryX.GetPCODE()); - EXPECT_EQUAL(features, queryX._features); - EXPECT_EQUAL(2u, queryX._offset); - EXPECT_EQUAL(3u, queryX._maxhits); - EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), queryX.getTimeout()); - EXPECT_EQUAL(0x1u, queryX.getQueryFlags()); //Filtered - if (queryX._features & QF_RANKP) { - EXPECT_EQUAL("seven", queryX._ranking); - } else { - EXPECT_EQUAL("", queryX._ranking); - } - EXPECT_EQUAL(queryX._features & QF_PARSEDQUERY ? 14u : 0u, queryX._numStackItems); - if (queryX._features & QF_PROPERTIES) { - EXPECT_EQUAL(2u, queryX._propsVector.size()); - testProperties(queryX._propsVector[0], "foo", 8); - testProperties(queryX._propsVector[1], "bar", 16); - } else { - EXPECT_EQUAL(0u, queryX._propsVector.size()); - } - if (queryX._features & QF_SORTSPEC) { - EXPECT_EQUAL("sortspec", queryX._sortSpec); - } else { - EXPECT_EQUAL(0u, queryX._sortSpec.size()); - } - if (queryX._features & QF_GROUPSPEC) { - EXPECT_EQUAL("groupspec", queryX._groupSpec); - } else { - EXPECT_EQUAL(0u, queryX._groupSpec.size()); - } - if (queryX._features & QF_LOCATION) { - EXPECT_EQUAL("location", queryX._location); - } else { - EXPECT_EQUAL(0u, queryX._location.size()); - } - if (queryX._features & QF_PARSEDQUERY) { - EXPECT_EQUAL("stackdump", queryX._stackDump); - } else { - EXPECT_EQUAL(0u, queryX._stackDump.size()); - } -} - -TEST("testQueryX") { - FS4Packet_QUERYX *src = createAndFill_QUERYX(); - std::vector<std::pair<FNET_Packet*, uint32_t>> lst; - for (uint32_t i = QF_PARSEDQUERY, len = (uint32_t)(QF_GROUPSPEC << 1), skip = 0; i < len; ++i) { - if (!(i & QF_PARSEDQUERY)) { - continue; // skip most - } - if (i & ~FNET_QF_SUPPORTED_MASK) { - continue; // not supported - } - if (++skip % 10) { - continue; // skip most - } - src->_features = i; - lst.emplace_back(testEncodeDecode(*src), i); - } - src->_features = uint32_t(-1); - lst.emplace_back(src, -1); - - for (const auto & pfPair : lst) { - FS4Packet_QUERYX *ptr = dynamic_cast<FS4Packet_QUERYX*>(pfPair.first); - ASSERT_TRUE(ptr != NULL); - verifyQueryX(*ptr, pfPair.second); - - delete ptr; - } -} - -TEST("testSharedPacket") { - FNET_Packet::SP src(createAndFill_QUERYX()); - static_cast<FS4Packet_QUERYX *>(src.get())->_features=FNET_QF_SUPPORTED_MASK; - FNET_Packet::SP decoded(testEncodeDecode(*src)); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded.get()), FNET_QF_SUPPORTED_MASK); - EXPECT_TRUE(decoded.get() != nullptr); - FS4Packet_Shared shared(decoded); - FNET_Packet::UP decoded2(testEncodeDecode(shared)); - EXPECT_TRUE(decoded2.get() != nullptr); - EXPECT_TRUE(nullptr == dynamic_cast<const FS4Packet_Shared *>(decoded2.get())); - EXPECT_TRUE(nullptr != dynamic_cast<const FS4Packet_QUERYX *>(decoded2.get())); - EXPECT_EQUAL(src->GetLength(), decoded2->GetLength()); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded2.get()), FNET_QF_SUPPORTED_MASK); -} - -TEST("test pre serializing packets no compression") { - FNET_Packet::UP src(createAndFill_QUERYX()); - FS4Packet_QUERYX * queryX = static_cast<FS4Packet_QUERYX *>(src.get()); - queryX->_features=FNET_QF_SUPPORTED_MASK; - FNET_Packet::UP decoded(testEncodeDecode(*src)); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded.get()), FNET_QF_SUPPORTED_MASK); - EXPECT_EQUAL(500u, src->GetLength()); - EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); - FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u, serialized.GetPCODE()); - EXPECT_EQUAL(500u, serialized.GetLength()); - FNET_Packet::UP decoded2(testEncodeDecode(serialized)); - EXPECT_EQUAL(500u, decoded2->GetLength()); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded2.get()), FNET_QF_SUPPORTED_MASK); -} - -TEST("test pre serializing packets with compression") { - FNET_Packet::UP src(createAndFill_QUERYX()); - FS4Packet_QUERYX * queryX = static_cast<FS4Packet_QUERYX *>(src.get()); - queryX->_features=FNET_QF_SUPPORTED_MASK; - FNET_Packet::UP decoded(testEncodeDecode(*src)); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded.get()), FNET_QF_SUPPORTED_MASK); - EXPECT_EQUAL(500u, src->GetLength()); - EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); - FS4PersistentPacketStreamer::Instance.SetCompressionLimit(100); - FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u | (CompressionConfig::LZ4 << 24), serialized.GetPCODE()); - EXPECT_GREATER_EQUAL(321u, serialized.GetLength()); - FNET_Packet::UP decoded2(testEncodeDecode(serialized)); - EXPECT_EQUAL(500u, decoded2->GetLength()); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded2.get()), FNET_QF_SUPPORTED_MASK); -} - - -TEST("testGetDocsumsX") { - FS4Packet_GETDOCSUMSX *src = dynamic_cast<FS4Packet_GETDOCSUMSX*>(FS4PacketFactory::CreateFS4Packet(PCODE_GETDOCSUMSX)); - ASSERT_TRUE(src != NULL); - src->setTimeout(fastos::TimeStamp(2*fastos::TimeStamp::MS)); - src->setRanking("four"); - src->_qflags = 5u; - src->_stackItems = 7u; - src->_propsVector.resize(2); - fillProperties(src->_propsVector[0], "foo", 8); - fillProperties(src->_propsVector[1], "bar", 16); - src->setResultClassName("resultclassname"); - src->setStackDump("stackdump"); - src->setLocation("location"); - src->_flags = GDFLAG_IGNORE_ROW; - src->AllocateDocIDs(2); - src->_docid[0]._gid = gid0; - src->_docid[0]._partid = 2u; - src->_docid[1]._gid = gid1; - src->_docid[1]._partid = 3u; - - std::vector<std::pair<FNET_Packet*, uint32_t>> lst; - for (uint32_t i = GDF_MLD, len = (uint32_t)(GDF_FLAGS << 1); i < len; ++i) { - if (i & ~FNET_GDF_SUPPORTED_MASK) { - continue; // not supported - } - src->_features = i; - lst.emplace_back(testEncodeDecode(*src), i); - } - src->_features = uint32_t(-1); - lst.emplace_back(src, uint32_t(-1)); - - for (const auto & pfPair : lst) { - FS4Packet_GETDOCSUMSX *ptr = dynamic_cast<FS4Packet_GETDOCSUMSX*>(pfPair.first); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_GETDOCSUMSX, ptr->GetPCODE()); - EXPECT_EQUAL(pfPair.second, ptr->_features); - EXPECT_EQUAL(fastos::TimeStamp(2*fastos::TimeStamp::MS), ptr->getTimeout()); - if (ptr->_features & GDF_RANKP_QFLAGS) { - EXPECT_EQUAL("four", ptr->_ranking); - } else { - EXPECT_EQUAL("", ptr->_ranking); - } - EXPECT_EQUAL(ptr->_features & GDF_RANKP_QFLAGS ? 5u : 0u, ptr->_qflags); - EXPECT_EQUAL(ptr->_features & GDF_QUERYSTACK ? 7u : 0u, ptr->_stackItems); - if (ptr->_features & GDF_PROPERTIES) { - EXPECT_EQUAL(2u, ptr->_propsVector.size()); - testProperties(ptr->_propsVector[0], "foo", 8); - testProperties(ptr->_propsVector[1], "bar", 16); - } else { - EXPECT_EQUAL(0u, ptr->_propsVector.size()); - } - if (ptr->_features & GDF_RESCLASSNAME) { - EXPECT_EQUAL("resultclassname", ptr->_resultClassName); - } else { - EXPECT_EQUAL(0u, ptr->_resultClassName.size()); - } - if (ptr->_features & GDF_QUERYSTACK) { - EXPECT_EQUAL("stackdump", ptr->_stackDump); - } else { - EXPECT_EQUAL(0u, ptr->_stackDump.size()); - } - if (ptr->_features & GDF_LOCATION) { - EXPECT_EQUAL("location", ptr->_location); - } else { - EXPECT_EQUAL(0u, ptr->_location.size()); - } - if (ptr->_features & GDF_FLAGS) { - EXPECT_EQUAL(static_cast<uint32_t>(GDFLAG_IGNORE_ROW), - ptr->_flags); - } else { - EXPECT_EQUAL(0u, ptr->_flags); - } - EXPECT_EQUAL(2u, ptr->_docid.size()); - for (uint32_t i = 0; i < ptr->_docid.size(); ++i) { - EXPECT_EQUAL(i == 0u ? gid0 : gid1, ptr->_docid[i]._gid); - EXPECT_EQUAL(ptr->_features & GDF_MLD ? 2u + i : 0u, ptr->_docid[i]._partid); - } - - delete ptr; - } -} - -TEST("require that FS4PersistentPacketStreamer can compress packets") { - FS4Packet_ERROR *packet = static_cast<FS4Packet_ERROR*>(FS4PacketFactory::CreateFS4Packet(PCODE_ERROR)); - packet->_errorCode = 1u; - packet->setErrorMessage(string(1000, 'a')); - - FS4PersistentPacketStreamer streamer(FS4PacketFactory::CreateFS4Packet); - - FNET_DataBuffer buf1; - streamer.Encode(packet, 1u, &buf1); - EXPECT_EQUAL(1020u, buf1.GetDataLen()); - - streamer.SetCompressionLimit(100); - FNET_DataBuffer buf2; - streamer.Encode(packet, 1u, &buf2); - EXPECT_EQUAL(38u, buf2.GetDataLen()); - - std::vector<FNET_Packet*> lst{ packet, testEncodeDecode(streamer, *packet) }; - - for (FNET_Packet * fnetPacket : lst) { - FS4Packet_ERROR *ptr = dynamic_cast<FS4Packet_ERROR*>(fnetPacket); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_ERROR, ptr->GetPCODE()); - EXPECT_EQUAL(1008u, ptr->GetLength()); - delete ptr; - } -} - -TEST("require that FS4PersistentPacketStreamer can avoid compressing small packets") { - FS4Packet_ERROR *packet = static_cast<FS4Packet_ERROR*>(FS4PacketFactory::CreateFS4Packet(PCODE_ERROR)); - packet->_errorCode = 1u; - packet->setErrorMessage("a"); - - FS4PersistentPacketStreamer streamer(FS4PacketFactory::CreateFS4Packet); - - FNET_DataBuffer buf1; - streamer.Encode(packet, 1u, &buf1); - EXPECT_EQUAL(21u, buf1.GetDataLen()); - - streamer.SetCompressionLimit(10); - FNET_DataBuffer buf2; - streamer.Encode(packet, 1u, &buf2); - EXPECT_EQUAL(21u, buf2.GetDataLen()); - - delete packet; -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/engine/docsumapi/.gitignore b/searchlib/src/tests/engine/docsumapi/.gitignore deleted file mode 100644 index 1b38a4ff745..00000000000 --- a/searchlib/src/tests/engine/docsumapi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -docsumapi_test -searchlib_docsumapi_test_app diff --git a/searchlib/src/tests/engine/docsumapi/CMakeLists.txt b/searchlib/src/tests/engine/docsumapi/CMakeLists.txt deleted file mode 100644 index d1acc996848..00000000000 --- a/searchlib/src/tests/engine/docsumapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_docsumapi_test_app TEST - SOURCES - docsumapi_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_docsumapi_test_app COMMAND searchlib_docsumapi_test_app) diff --git a/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp b/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp deleted file mode 100644 index 612e4622373..00000000000 --- a/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("docsumapi_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/docsumapi.h> -#include <vespa/searchlib/engine/packetconverter.h> - -using namespace search::engine; -using namespace search::fs4transport; - -namespace { - -// light-weight network hop simulation -template <typename T> void copyPacket(T &src, T &dst) { - FNET_DataBuffer buf; - src.Encode(&buf); - dst.Decode(&buf, buf.GetDataLen()); -} - -} // namespace <unnamed> - -class Test : public vespalib::TestApp -{ -public: - void convertToRequest(); - void convertFromReply(); - int Main() override; -}; - -document::GlobalId gid0("aaaaaaaaaaaa"); -document::GlobalId gid1("bbbbbbbbbbbb"); - -void -Test::convertToRequest() -{ - const string sessionId("qrserver.0.XXXXXXXXXXXXX.0"); - - FS4Packet_GETDOCSUMSX src; - src.setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - src._features |= GDF_RANKP_QFLAGS; - src.setRanking("seven"); - src._qflags = 5u; - src._features |= GDF_RESCLASSNAME; - src.setResultClassName("resclass"); - src._features |= GDF_PROPERTIES; - src._propsVector.resize(3); - src._propsVector[0].allocEntries(2); - src._propsVector[0].setName("feature", strlen("feature")); - src._propsVector[0].setKey(0, "p1k1", strlen("p1k1")); - src._propsVector[0].setValue(0, "p1v1", strlen("p1v1")); - src._propsVector[0].setKey(1, "p1k2", strlen("p1k2")); - src._propsVector[0].setValue(1, "p1v2", strlen("p1v2")); - src._propsVector[1].allocEntries(2); - src._propsVector[1].setName("caches", strlen("caches")); - src._propsVector[1].setKey(0, "p2k1", strlen("p2k1")); - src._propsVector[1].setValue(0, "p2v1", strlen("p2v1")); - src._propsVector[1].setKey(1, "p2k2", strlen("p2k2")); - src._propsVector[1].setValue(1, "p2v2", strlen("p2v2")); - src._propsVector[2].allocEntries(1); - src._propsVector[2].setName("rank", strlen("rank")); - src._propsVector[2].setKey(0, "sessionId", strlen("sessionId")); - src._propsVector[2].setValue(0, sessionId.c_str(), sessionId.size()); - src._features |= GDF_QUERYSTACK; - src._stackItems = 14u; - src.setStackDump("stackdump"); - src._features |= GDF_LOCATION; - src.setLocation("location"); - src._features |= GDF_MLD; - src.AllocateDocIDs(2); - src._docid[0]._gid = gid0; - src._docid[0]._partid = 5; - src._docid[1]._gid = gid1; - src._docid[1]._partid = 6; - - { // full copy - FS4Packet_GETDOCSUMSX cpy; - copyPacket(src, cpy); - - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - EXPECT_EQUAL((dst.getTimeOfDoom() - dst.getStartTime()).ms(), 4u); - EXPECT_EQUAL(dst.ranking, "seven"); - EXPECT_EQUAL(dst.queryFlags, 5u); - EXPECT_EQUAL(dst.resultClassName, "resclass"); - EXPECT_EQUAL(dst.propertiesMap.size(), 3u); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k1").get(), std::string("p1v1")); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k2").get(), std::string("p1v2")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k1").get(), std::string("p2v1")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k2").get(), std::string("p2v2")); - EXPECT_EQUAL(dst.propertiesMap.matchProperties().lookup("p3k1").get(), std::string("")); - EXPECT_EQUAL(std::string(&dst.stackDump[0], dst.stackDump.size()), "stackdump"); - EXPECT_EQUAL(dst.location, "location"); - EXPECT_EQUAL(dst._flags, 0u); - EXPECT_EQUAL(dst.hits.size(), 2u); - EXPECT_EQUAL(dst.hits[0].docid, 0u); - EXPECT_TRUE(dst.hits[0].gid == gid0); - EXPECT_EQUAL(dst.hits[0].path, 5u); - EXPECT_EQUAL(dst.hits[1].docid, 0u); - EXPECT_TRUE(dst.hits[1].gid == gid1); - EXPECT_EQUAL(dst.hits[1].path, 6u); - EXPECT_EQUAL(sessionId, - string(&dst.sessionId[0], dst.sessionId.size())); - } - { // without datetime - FS4Packet_GETDOCSUMSX cpy; - copyPacket(src, cpy); - - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - } - { // without mld - FS4Packet_GETDOCSUMSX cpy; - copyPacket(src, cpy); - cpy._features &= ~GDF_MLD; - - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - EXPECT_EQUAL(dst.useWideHits, false); - EXPECT_EQUAL(dst.hits.size(), 2u); - EXPECT_EQUAL(dst.hits[0].docid, 0u); - EXPECT_TRUE(dst.hits[0].gid == gid0); - EXPECT_EQUAL(dst.hits[1].docid, 0u); - EXPECT_TRUE(dst.hits[1].gid == gid1); - } - { // with ignore row flag - FS4Packet_GETDOCSUMSX tcpy; - copyPacket(src, tcpy); - tcpy._features |= GDF_FLAGS; - tcpy._flags = GDFLAG_IGNORE_ROW; - FS4Packet_GETDOCSUMSX cpy; - copyPacket(tcpy, cpy); - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - EXPECT_EQUAL(dst._flags, static_cast<uint32_t>(GDFLAG_IGNORE_ROW)); - } -} - -void -Test::convertFromReply() -{ - DocsumReply src; - src.docsums.resize(2); - src.docsums[0].docid = 1; - src.docsums[0].gid = gid0; - src.docsums[0].data.resize(2); - src.docsums[0].data.str()[0] = 5; - src.docsums[0].data.str()[1] = 6; - src.docsums[1].docid = 2; - src.docsums[1].gid = gid1; - src.docsums[1].data.resize(3); - src.docsums[1].data.str()[0] = 7; - src.docsums[1].data.str()[1] = 8; - src.docsums[1].data.str()[2] = 9; - - { // test first - FS4Packet_DOCSUM dst; - PacketConverter::fromDocsumReplyElement(src.docsums[0], dst); - EXPECT_EQUAL(dst.getGid(), gid0); - EXPECT_EQUAL(dst.getBuf().size(), 2u); - EXPECT_EQUAL(dst.getBuf().c_str()[0], 5); - EXPECT_EQUAL(dst.getBuf().c_str()[1], 6); - } - { // test second - FS4Packet_DOCSUM dst; - PacketConverter::fromDocsumReplyElement(src.docsums[1], dst); - EXPECT_EQUAL(dst.getGid(), gid1); - EXPECT_EQUAL(dst.getBuf().size(), 3u); - EXPECT_EQUAL(dst.getBuf().c_str()[0], 7); - EXPECT_EQUAL(dst.getBuf().c_str()[1], 8); - EXPECT_EQUAL(dst.getBuf().c_str()[2], 9); - } -} - -int -Test::Main() -{ - TEST_INIT("docsumapi_test"); - convertToRequest(); - convertFromReply(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/engine/monitorapi/.gitignore b/searchlib/src/tests/engine/monitorapi/.gitignore deleted file mode 100644 index 66fc005087f..00000000000 --- a/searchlib/src/tests/engine/monitorapi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -monitorapi_test -searchlib_monitorapi_test_app diff --git a/searchlib/src/tests/engine/monitorapi/CMakeLists.txt b/searchlib/src/tests/engine/monitorapi/CMakeLists.txt deleted file mode 100644 index 3fea05f4841..00000000000 --- a/searchlib/src/tests/engine/monitorapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_monitorapi_test_app TEST - SOURCES - monitorapi_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_monitorapi_test_app COMMAND searchlib_monitorapi_test_app) diff --git a/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp b/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp deleted file mode 100644 index 27b00b8a074..00000000000 --- a/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("monitorapi_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/monitorapi.h> -#include <vespa/searchlib/engine/packetconverter.h> - -using namespace search::engine; -using namespace search::fs4transport; - -namespace { - -bool checkFeature(uint32_t features, uint32_t mask) { - return ((features & mask) != 0); -} - -bool checkNotFeature(uint32_t features, uint32_t mask) { - return !checkFeature(features, mask); -} - -// light-weight network hop simulation -template <typename T> void copyPacket(T &src, T &dst) { - FNET_DataBuffer buf; - src.Encode(&buf); - dst.Decode(&buf, buf.GetDataLen()); -} - -} // namespace <unnamed> - -class Test : public vespalib::TestApp -{ -public: - void convertToRequest(); - void convertFromReply(); - int Main() override; -}; - -void -Test::convertToRequest() -{ - FS4Packet_MONITORQUERYX src; - src._features |= MQF_QFLAGS; - src._qflags = 1u; - - { // copy all - FS4Packet_MONITORQUERYX cpy; - copyPacket(src, cpy); - - MonitorRequest dst; - PacketConverter::toMonitorRequest(cpy, dst); - EXPECT_EQUAL(dst.flags, 1u); - } -} - -void -Test::convertFromReply() -{ - MonitorReply src; - src.mld = true; - src.partid = 1u; - src.timestamp = 2u; - src.totalNodes = 3u; - src.activeNodes = 4u; - src.totalParts = 5u; - src.activeParts = 6u; - src.flags = 7u; - src.activeDocs = 8u; - src.activeDocsRequested = true; - - { // full copy - MonitorReply cpy = src; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_EQUAL(dst._partid, 1u); - EXPECT_EQUAL(dst._timestamp, 2u); - EXPECT_TRUE(checkFeature(dst._features, MRF_MLD)); - EXPECT_EQUAL(dst._totalNodes, 3u); - EXPECT_EQUAL(dst._activeNodes, 4u); - EXPECT_EQUAL(dst._totalParts, 5u); - EXPECT_EQUAL(dst._activeParts, 6u); - EXPECT_TRUE(checkFeature(dst._features, MRF_RFLAGS)); - EXPECT_EQUAL(dst._rflags, 7u); - EXPECT_EQUAL(dst._activeDocs, 8u); - EXPECT_TRUE(checkFeature(dst._features, MRF_ACTIVEDOCS)); - } - { // non-mld - MonitorReply cpy = src; - cpy.mld = false; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_TRUE(checkNotFeature(dst._features, MRF_MLD)); - } - { // without flags - MonitorReply cpy = src; - cpy.flags = 0; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_TRUE(checkNotFeature(dst._features, MRF_RFLAGS)); - EXPECT_EQUAL(dst._rflags, 0u); - } - { // without activedocs - MonitorReply cpy = src; - cpy.activeDocsRequested = false; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_TRUE(checkNotFeature(dst._features, MRF_ACTIVEDOCS)); - EXPECT_EQUAL(dst._activeDocs, 0u); - } -} - -int -Test::Main() -{ - TEST_INIT("monitorapi_test"); - convertToRequest(); - convertFromReply(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/engine/searchapi/.gitignore b/searchlib/src/tests/engine/searchapi/.gitignore deleted file mode 100644 index 92089e63cdd..00000000000 --- a/searchlib/src/tests/engine/searchapi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -searchapi_test -searchlib_searchapi_test_app diff --git a/searchlib/src/tests/engine/searchapi/CMakeLists.txt b/searchlib/src/tests/engine/searchapi/CMakeLists.txt deleted file mode 100644 index c4e3de1280a..00000000000 --- a/searchlib/src/tests/engine/searchapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_searchapi_test_app TEST - SOURCES - searchapi_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_searchapi_test_app COMMAND searchlib_searchapi_test_app) diff --git a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp deleted file mode 100644 index 99d1967b0df..00000000000 --- a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/searchapi.h> -#include <vespa/searchlib/engine/packetconverter.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/log/log.h> -LOG_SETUP("searchapi_test"); - -using namespace search::engine; -using namespace search::fs4transport; - -namespace { - -bool checkFeature(uint32_t features, uint32_t mask) { - return ((features & mask) != 0); -} - -bool checkNotFeature(uint32_t features, uint32_t mask) { - return !checkFeature(features, mask); -} - -// light-weight network hop simulation -template <typename T> void copyPacket(T &src, T &dst) { - FNET_DataBuffer buf; - src.Encode(&buf); - dst.Decode(&buf, buf.GetDataLen()); -} - -} // namespace <unnamed> - - -TEST("propertyNames") { - EXPECT_EQUAL(search::MapNames::RANK, "rank"); - EXPECT_EQUAL(search::MapNames::FEATURE, "feature"); - EXPECT_EQUAL(search::MapNames::HIGHLIGHTTERMS, "highlightterms"); - EXPECT_EQUAL(search::MapNames::MATCH, "match"); - EXPECT_EQUAL(search::MapNames::CACHES, "caches"); - EXPECT_EQUAL(search::MapNames::TRACE, "trace"); -} - -TEST("convertToReques") { - FS4Packet_QUERYX src; - src._offset = 2u; - src._maxhits = 3u; - src.setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - src.setQueryFlags(5u); - src._features |= QF_RANKP; - src.setRanking("seven"); - src._features |= QF_PROPERTIES; - src._propsVector.resize(2); - src._propsVector[0].allocEntries(2); - src._propsVector[0].setName("feature", strlen("feature")); - src._propsVector[0].setKey(0, "p1k1", strlen("p1k1")); - src._propsVector[0].setValue(0, "p1v1", strlen("p1v1")); - src._propsVector[0].setKey(1, "p1k2", strlen("p1k2")); - src._propsVector[0].setValue(1, "p1v2", strlen("p1v2")); - src._propsVector[1].allocEntries(2); - src._propsVector[1].setName("caches", strlen("caches")); - src._propsVector[1].setKey(0, "p2k1", strlen("p2k1")); - src._propsVector[1].setValue(0, "p2v1", strlen("p2v1")); - src._propsVector[1].setKey(1, "p2k2", strlen("p2k2")); - src._propsVector[1].setValue(1, "p2v2", strlen("p2v2")); - src._features |= QF_SORTSPEC; - src.setSortSpec("sortspec"); - src._features |= QF_GROUPSPEC; - src.setGroupSpec("groupspec"); - src._features |= QF_SESSIONID; - src.setSessionId("sessionid"); - src._features |= QF_LOCATION; - src.setLocation("location"); - src._features |= QF_PARSEDQUERY; - src._numStackItems = 14u; - src.setStackDump("stackdump"); - - { // full copy - FS4Packet_QUERYX cpy; - copyPacket(src, cpy); - - SearchRequest dst; - PacketConverter::toSearchRequest(cpy, dst); - EXPECT_EQUAL(dst.offset, 2u); - EXPECT_EQUAL(dst.maxhits, 3u); - EXPECT_EQUAL((dst.getTimeOfDoom() - dst.getStartTime()).ms(), 4u); - EXPECT_EQUAL(dst.queryFlags, 1u); //Filtered - EXPECT_EQUAL(vespalib::string("seven"), dst.ranking); - EXPECT_EQUAL(dst.propertiesMap.size(), 2u); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k1").get(), std::string("p1v1")); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k2").get(), std::string("p1v2")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k1").get(), std::string("p2v1")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k2").get(), std::string("p2v2")); - EXPECT_EQUAL(dst.propertiesMap.matchProperties().lookup("p3k1").get(), std::string("")); - EXPECT_EQUAL(dst.sortSpec, "sortspec"); - EXPECT_EQUAL(std::string(&dst.groupSpec[0], dst.groupSpec.size()), "groupspec"); - EXPECT_EQUAL(std::string(&dst.sessionId[0], dst.sessionId.size()), "sessionid"); - EXPECT_EQUAL(dst.location, "location"); - EXPECT_EQUAL(dst.stackItems, 14u); - EXPECT_EQUAL(std::string(&dst.stackDump[0], dst.stackDump.size()), "stackdump"); - } - { // without datetime - FS4Packet_QUERYX cpy; - copyPacket(src, cpy); - - SearchRequest dst; - PacketConverter::toSearchRequest(cpy, dst); - } -} - -TEST("convertFromReply") { - SearchReply src; - src.offset = 1u; - src.totalHitCount = 2u; - src.maxRank = 3; - src.setDistributionKey(4u); - src.sortIndex.push_back(0); - src.sortIndex.push_back(1); - src.sortIndex.push_back(2); - src.sortData.push_back(11); - src.sortData.push_back(22); - src.groupResult.push_back(2); - src.coverage = SearchReply::Coverage(5, 3); - src.useWideHits = true; - src.hits.resize(2); - document::GlobalId gid0("aaaaaaaaaaaa"); - document::GlobalId gid1("bbbbbbbbbbbb"); - src.hits[0].gid = gid0; - src.hits[0].metric = 5; - src.hits[0].path = 11; - src.hits[0].setDistributionKey(100); - src.hits[1].gid = gid1; - src.hits[1].metric = 4; - src.hits[1].path = 10; - src.hits[1].setDistributionKey(105); - - { // full copy - SearchReply cpy = src; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_EQUAL(dst._offset, 1u); - EXPECT_EQUAL(dst._numDocs, 2u); - EXPECT_EQUAL(dst._totNumDocs, 2u); - EXPECT_EQUAL(dst._maxRank, 3); - EXPECT_EQUAL(4u, dst.getDistributionKey()); - EXPECT_TRUE(checkFeature(dst._features, QRF_SORTDATA)); - EXPECT_EQUAL(dst._sortIndex[0], 0u); - EXPECT_EQUAL(dst._sortIndex[1], 1u); - EXPECT_EQUAL(dst._sortIndex[2], 2u); - EXPECT_EQUAL(dst._sortData[0], 11); - EXPECT_EQUAL(dst._sortData[1], 22); - EXPECT_TRUE(checkFeature(dst._features, QRF_GROUPDATA)); - EXPECT_EQUAL(dst._groupDataLen, 1u); - EXPECT_EQUAL(dst._groupData[0], 2); - EXPECT_TRUE(checkFeature(dst._features, QRF_COVERAGE)); - EXPECT_EQUAL(dst._coverageDocs, 3u); - EXPECT_EQUAL(dst._activeDocs, 5u); - EXPECT_TRUE(checkFeature(dst._features, QRF_MLD)); - EXPECT_TRUE(dst._hits[0]._gid == gid0); - EXPECT_EQUAL(dst._hits[0]._metric, 5); - EXPECT_EQUAL(dst._hits[0]._partid, 11u); - EXPECT_EQUAL(dst._hits[0].getDistributionKey(), 100u); - EXPECT_TRUE(dst._hits[1]._gid == gid1); - EXPECT_EQUAL(dst._hits[1]._metric, 4); - EXPECT_EQUAL(dst._hits[1]._partid, 10u); - EXPECT_EQUAL(dst._hits[1].getDistributionKey(), 105u); - } - { // not sortdata - SearchReply cpy = src; - cpy.sortIndex.clear(); - cpy.sortData.clear(); - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_SORTDATA)); - } - { // not groupdata - SearchReply cpy = src; - cpy.groupResult.clear(); - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_GROUPDATA)); - } - { // non-full coverage - SearchReply cpy = src; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkFeature(dst._features, QRF_COVERAGE)); - EXPECT_EQUAL(dst._coverageDocs, 3u); - EXPECT_EQUAL(dst._activeDocs, 5u); - } - { // non-mld - SearchReply cpy = src; - cpy.useWideHits = false; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_MLD)); - EXPECT_TRUE(dst._hits[0]._gid == gid0); - EXPECT_EQUAL(dst._hits[0]._metric, 5); - EXPECT_TRUE(dst._hits[1]._gid == gid1); - EXPECT_EQUAL(dst._hits[1]._metric, 4); - } - { // non-mld not siteid - SearchReply cpy = src; - cpy.useWideHits = false; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_MLD)); - EXPECT_TRUE(dst._hits[0]._gid == gid0); - EXPECT_EQUAL(dst._hits[0]._metric, 5); - EXPECT_TRUE(dst._hits[1]._gid == gid1); - EXPECT_EQUAL(dst._hits[1]._metric, 4); - } -} - -void verify(vespalib::stringref expected, const vespalib::Slime & slime) { - vespalib::Slime expectedSlime; - vespalib::slime::JsonFormat::decode(expected, expectedSlime); - EXPECT_EQUAL(expectedSlime, slime); -} - -TEST("verify trace") { - RelativeTime clock(std::make_unique<CountingClock>(fastos::TimeStamp::fromSec(1500000000), 1700000L)); - Trace t(clock); - EXPECT_FALSE(t.hasTrace()); - t.start(0); - EXPECT_TRUE(t.hasTrace()); - t.createCursor("tag_a"); - verify("{" - " start_time_utc: '2017-07-14 02:40:00.000 UTC'," - " traces: [" - " {" - " tag: 'tag_a'," - " timestamp_ms: 1.7" - " }" - " ]" - "}", - t.getSlime()); - Trace::Cursor & tagB = t.createCursor("tag_b"); - tagB.setLong("long", 19); - t.done(); - verify("{" - " start_time_utc: '2017-07-14 02:40:00.000 UTC'," - " traces: [" - " {" - " tag: 'tag_a'," - " timestamp_ms: 1.7" - " }," - " {" - " tag: 'tag_b'," - " timestamp_ms: 3.4," - " long: 19" - " }" - " ]," - " duration_ms: 5.1" - "}", - t.getSlime()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/engine/transportserver/.gitignore b/searchlib/src/tests/engine/transportserver/.gitignore deleted file mode 100644 index 09d836e0004..00000000000 --- a/searchlib/src/tests/engine/transportserver/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -transportserver_test -vlog.txt -searchlib_transportserver_test_app diff --git a/searchlib/src/tests/engine/transportserver/CMakeLists.txt b/searchlib/src/tests/engine/transportserver/CMakeLists.txt deleted file mode 100644 index b4f94884ce4..00000000000 --- a/searchlib/src/tests/engine/transportserver/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_transportserver_test_app TEST - SOURCES - transportserver_test.cpp - DEPENDS - searchlib -) -vespa_add_test( - NAME searchlib_transportserver_test_app - COMMAND searchlib_transportserver_test_app - ENVIRONMENT "VESPA_LOG_TARGET=file:vlog.txt;VESPA_LOG_LEVEL=\"all -spam\"" -) diff --git a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp deleted file mode 100644 index baa581c65f9..00000000000 --- a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/base/documentid.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/transportserver.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/fnet.h> -#include <vespa/searchlib/engine/errorcodes.h> -#include <thread> -#include <chrono> -#include <vespa/log/log.h> -LOG_SETUP("transportserver_test"); - -using namespace document; -using namespace vespalib; -using namespace search::engine; -using namespace search::fs4transport; - -class SyncServer : public search::engine::SearchServer, - public search::engine::DocsumServer, - public search::engine::MonitorServer -{ -private: - virtual SearchReply::UP search(SearchRequest::Source request, SearchClient &client) override; - virtual DocsumReply::UP getDocsums(DocsumRequest::Source request, DocsumClient &client) override; - virtual MonitorReply::UP ping(MonitorRequest::UP request, MonitorClient &client) override; - - SyncServer(const SyncServer &); - SyncServer &operator=(const SyncServer &); -public: - SyncServer() {} - virtual ~SyncServer() {} -}; - -SearchReply::UP -SyncServer::search(SearchRequest::Source request, SearchClient &) -{ - // fastos should use steady clock (this may fail) - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - fastos::TimeStamp my_time = fastos::ClockSystem::now(); - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - EXPECT_GREATER_EQUAL(my_time.val(), request->getStartTime()); - const SearchRequest &req = *request.get(); - SearchReply::UP reply(new SearchReply()); - SearchReply &ret = *reply; - ret.request = request.release(); - LOG(info, "responding to search request..."); - ret.offset = req.offset; - return reply; -} - -DocsumReply::UP -SyncServer::getDocsums(DocsumRequest::Source request, DocsumClient &) -{ - // fastos should use steady clock (this may fail) - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - fastos::TimeStamp my_time = fastos::ClockSystem::now(); - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - EXPECT_GREATER_EQUAL(my_time.val(), request->getStartTime()); - DocsumReply::UP reply(new DocsumReply()); - DocsumReply &ret = *reply; - ret.request = request.release(); - LOG(info, "responding to docsum request..."); - ret.docsums.resize(1); - ret.docsums[0].setData("data", strlen("data")); - ret.docsums[0].gid = DocumentId(vespalib::make_string("id:ns:type::100")).getGlobalId(); - return reply; -} - -MonitorReply::UP -SyncServer::ping(MonitorRequest::UP request, MonitorClient &) -{ - MonitorRequest &req = *request; - MonitorReply::UP reply(new MonitorReply()); - MonitorReply &ret = *reply; - LOG(info, "responding to monitor request..."); - ret.timestamp = req.flags; - return reply; -} - -TEST("transportserver") { - { - SyncServer server; - TransportServer transport(server, server, server, 0, - TransportServer::DEBUG_ALL); - ASSERT_TRUE(transport.start()); - int port = transport.getListenPort(); - ASSERT_TRUE(port > 0); - { - FNET_Context ctx; - FastOS_ThreadPool pool(128 * 1024); - FNET_Transport client; - ASSERT_TRUE(client.Start(&pool)); - - FNET_PacketQueue adminQ; - FNET_Connection *conn = client.Connect(make_string("tcp/localhost:%d", port).c_str(), - &FS4PersistentPacketStreamer::Instance, &adminQ); - ASSERT_TRUE(conn != 0); - { - FS4Packet_MONITORQUERYX *mq = new FS4Packet_MONITORQUERYX(); - mq->_qflags = 30; - mq->_features |= MQF_QFLAGS; - conn->PostPacket(mq, FNET_NOID); - FNET_Packet *p = adminQ.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_MONITORRESULTX); - FS4Packet_MONITORRESULTX *r = (FS4Packet_MONITORRESULTX*)p; - EXPECT_EQUAL(r->_timestamp, 30u); - p->Free(); - } - { - FNET_PacketQueue q; - FNET_Channel *ch = conn->OpenChannel(&q, FNET_Context()); - FS4Packet_QUERYX *qx = new FS4Packet_QUERYX(); - qx->_features |= QF_PARSEDQUERY; - qx->_offset = 100; - ch->Send(qx); - FNET_Packet *p = q.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_QUERYRESULTX); - FS4Packet_QUERYRESULTX *r = (FS4Packet_QUERYRESULTX*)p; - EXPECT_EQUAL(r->_offset, 100u); - p->Free(); - ch->CloseAndFree(); - } - { - FS4Packet_MONITORQUERYX *mq = new FS4Packet_MONITORQUERYX(); - mq->_qflags = 40; - mq->_features |= MQF_QFLAGS; - conn->PostPacket(mq, FNET_NOID); - FNET_Packet *p = adminQ.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_MONITORRESULTX); - FS4Packet_MONITORRESULTX *r = (FS4Packet_MONITORRESULTX*)p; - EXPECT_EQUAL(r->_timestamp, 40u); - p->Free(); - } - { - FNET_PacketQueue q; - FNET_Channel *ch = conn->OpenChannel(&q, FNET_Context()); - FS4Packet_GETDOCSUMSX *qdx = new FS4Packet_GETDOCSUMSX(); - ch->Send(qdx); - FNET_Packet *p = q.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_DOCSUM); - FS4Packet_DOCSUM *r = (FS4Packet_DOCSUM*)p; - EXPECT_EQUAL(r->getGid(), DocumentId("id:ns:type::100").getGlobalId()); - p->Free(); - p = q.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_EOL); - p->Free(); - ch->CloseAndFree(); - } - { - FS4Packet_MONITORQUERYX *mq = new FS4Packet_MONITORQUERYX(); - mq->_qflags = 50; - mq->_features |= MQF_QFLAGS; - conn->PostPacket(mq, FNET_NOID); - FNET_Packet *p = adminQ.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_MONITORRESULTX); - FS4Packet_MONITORRESULTX *r = (FS4Packet_MONITORRESULTX*)p; - EXPECT_EQUAL(r->_timestamp, 50u); - p->Free(); - } - // shut down client - conn->CloseAdminChannel(); - client.Close(conn); - conn->SubRef(); - client.sync(); - client.ShutDown(true); - pool.Close(); - } - - } -} - -void printError(ErrorCode ecode) { - fprintf(stderr, "error code %u: '%s'\n", ecode, getStringFromErrorCode(ecode)); -} - -TEST("print errors") { - printError(ECODE_NO_ERROR); - printError(ECODE_GENERAL_ERROR); - printError(ECODE_QUERY_PARSE_ERROR); - printError(ECODE_ALL_PARTITIONS_DOWN); - printError(ECODE_ILLEGAL_DATASET); - printError(ECODE_OVERLOADED); - printError(ECODE_NOT_IMPLEMENTED); - printError(ECODE_QUERY_NOT_ALLOWED); - printError(ECODE_TIMEOUT); -} - -TEST("test SearchReply::Coverage") { - SearchReply::Coverage c; - EXPECT_EQUAL(0u, c.getActive()); - EXPECT_EQUAL(0u, c.getSoonActive()); - EXPECT_EQUAL(0u, c.getCovered()); - EXPECT_EQUAL(0u, c.getDegradeReason()); -} - -TEST("test SearchReply::Coverage(7)") { - SearchReply::Coverage c(7); - EXPECT_EQUAL(7u, c.getActive()); - EXPECT_EQUAL(7u, c.getSoonActive()); - EXPECT_EQUAL(7u, c.getCovered()); - EXPECT_EQUAL(0u, c.getDegradeReason()); -} - -TEST("test SearchReply::Coverage(7, 19)") { - SearchReply::Coverage c(19, 7); - EXPECT_EQUAL(19u, c.getActive()); - EXPECT_EQUAL(19u, c.getSoonActive()); - EXPECT_EQUAL(7u, c.getCovered()); - EXPECT_EQUAL(0u, c.getDegradeReason()); -} - -TEST("test SearchReply::Coverage set and get") { - SearchReply::Coverage c; - EXPECT_EQUAL(7u, c.setActive(7).getActive()); - EXPECT_EQUAL(9u, c.setSoonActive(9).getSoonActive()); - EXPECT_EQUAL(19u, c.setCovered(19).getCovered()); - EXPECT_EQUAL(5u, c.setDegradeReason(5).getDegradeReason()); - EXPECT_EQUAL(1u, SearchReply::Coverage().degradeMatchPhase().getDegradeReason()); - EXPECT_EQUAL(2u, SearchReply::Coverage().degradeTimeout().getDegradeReason()); - EXPECT_EQUAL(4u, SearchReply::Coverage().degradeAdaptiveTimeout().getDegradeReason()); - EXPECT_EQUAL(7u, SearchReply::Coverage().degradeAdaptiveTimeout().degradeTimeout().degradeMatchPhase().getDegradeReason()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 9862e221610..4b20513c66d 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -2,17 +2,12 @@ #include "mapnames.h" #include "packets.h" -#include "sortdata.h" -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/util/compressor.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/databuffer.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); -using vespalib::ConstBufferRef; using vespalib::make_string; using vespalib::stringref; @@ -21,192 +16,17 @@ namespace search::fs4transport { /** * Persistent packet streamer. **/ -FS4PersistentPacketStreamer FS4PersistentPacketStreamer:: -Instance(FS4PacketFactory::CreateFS4Packet); +FS4PersistentPacketStreamer FS4PersistentPacketStreamer::Instance; //============================================================ -bool -FS4PersistentPacketStreamer::HasChannelID(uint32_t pcode) -{ - switch(pcode & PCODE_MASK) { - case PCODE_EOL: - case PCODE_ERROR: - case PCODE_DOCSUM: - case PCODE_QUERYRESULTX: - case PCODE_QUERYX: - case PCODE_GETDOCSUMSX: - case PCODE_TRACEREPLY: - return true; - default: - return false; - } -} - FS4PersistentPacketStreamer:: -FS4PersistentPacketStreamer(FS4PacketFactory::CreatePacket_t cp) +FS4PersistentPacketStreamer() : _compressionLimit(0), _compressionLevel(9), - _compressionType(CompressionConfig::LZ4), - _conservative(false), - _createPacket(cp) + _compressionType(CompressionConfig::LZ4) { } -bool -FS4PersistentPacketStreamer::GetPacketInfo(FNET_DataBuffer *src, - uint32_t *plen, uint32_t *pcode, - uint32_t *chid, bool *broken) -{ - uint32_t tmpVal; - bool hasCHID; - - if (src->GetDataLen() < 2 * sizeof(uint32_t) || - ((hasCHID = HasChannelID(src->PeekInt32(sizeof(uint32_t)))) && - src->GetDataLen() < 3 * sizeof(uint32_t))) - return false; - - if (hasCHID) { - tmpVal = src->ReadInt32(); - if (tmpVal < 2 * sizeof(uint32_t)) { - // This is not a valid packet length. We might - // be out of sync. - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid packet length %u\n", tmpVal); - } - return false; - } else { - *plen = tmpVal - 2 * sizeof(uint32_t); - } - tmpVal = src->ReadInt32(); - if (!ValidPCode(tmpVal)) { - // Out of sync? - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid pcode %u (%u)\n", tmpVal, *plen); - } - return false; - } else { - *pcode = tmpVal; - } - *chid = src->ReadInt32(); - } else { - tmpVal = src->ReadInt32(); - if (tmpVal < sizeof(uint32_t)) { - // This is not a valid packet length. We might - // be out of sync. - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid length (noch) %u\n", tmpVal); - } - return false; - } else { - *plen = tmpVal - sizeof(uint32_t); - } - tmpVal = src->ReadInt32(); - if (!ValidPCode(tmpVal)) { - // Out of sync? - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid pcode (noch) %u (%u)\n", tmpVal, *plen); - } - return false; - } else { - *pcode = tmpVal; - } - *chid = FNET_NOID; - } - return true; -} - -namespace { - -void decodePacket(FNET_Packet *&packet, FNET_DataBuffer &buf, uint32_t size, uint32_t pcode) { - try { - if (!packet->Decode(&buf, size)) { - LOG(error, "could not decode packet (pcode=%u); " - "this could be caused by a protocol and/or " - "version incompatibility\n", pcode); - packet->Free(); - packet = NULL; - } - } catch (const vespalib::Exception & e) { - packet->Free(); - packet = NULL; - LOG(error, "%s", e.toString().c_str()); - } -} - -} // namespace - -FNET_Packet* -FS4PersistentPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, uint32_t pcode, FNET_Context) -{ - FNET_Packet *packet(_createPacket(pcode & PCODE_MASK)); - if (packet != NULL) { - uint32_t compressionByte = (pcode & ~PCODE_MASK) >> 24; - CompressionConfig::Type compressionType(CompressionConfig::toType(compressionByte)); - if (compressionType != 0) { - uint32_t uncompressed_size = src->ReadInt32(); - ConstBufferRef org(src->GetData(), plen - sizeof(uint32_t)); - vespalib::DataBuffer uncompressed(uncompressed_size); - vespalib::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); - FNET_DataBuffer buf(uncompressed.getData(), uncompressed.getDataLen()); - decodePacket(packet, buf, uncompressed_size, pcode); - src->DataToDead(plen - sizeof(uint32_t)); - } else { - decodePacket(packet, *src, plen, pcode); - } - } else { - src->DataToDead(plen); - } - return packet; -} - - -void -FS4PersistentPacketStreamer::Encode(FNET_Packet *packet, uint32_t chid, FNET_DataBuffer *dst) -{ - uint32_t len = packet->GetLength(); - uint32_t pcode = packet->GetPCODE(); - - uint32_t packet_start = dst->GetDataLen(); - if (HasChannelID(pcode)) { - dst->EnsureFree(len + 3 * sizeof(uint32_t)); - dst->WriteInt32Fast(len + 2 * sizeof(uint32_t)); - dst->WriteInt32Fast(pcode); - dst->WriteInt32Fast(chid); - } else { - dst->EnsureFree(len + 2 * sizeof(uint32_t)); - dst->WriteInt32Fast(len + sizeof(uint32_t)); - dst->WriteInt32Fast(pcode); - } - uint32_t header_len = dst->GetDataLen() - packet_start; - packet->Encode(dst); - dst->AssertValid(); - uint32_t body_len = dst->GetDataLen() - packet_start - header_len; - bool isCompressable((pcode & ~PCODE_MASK) == 0); - - if (isCompressable && _compressionLimit && (body_len > _compressionLimit)) { - CompressionConfig config(_compressionType, _compressionLevel, 90); - ConstBufferRef org(dst->GetData() + packet_start + header_len, body_len); - vespalib::DataBuffer compressed(org.size()); - CompressionConfig::Type r = vespalib::compression::compress(config, org, compressed, false); - if (r != CompressionConfig::NONE) { - dst->DataToFree(body_len + header_len); - // sizeof(data + header + uncompressed_size) - sizeof(uint32_t) - dst->WriteInt32Fast(compressed.getDataLen() + header_len); - dst->WriteInt32Fast(pcode | (_compressionType << 24)); - if (HasChannelID(pcode)) { - dst->FreeToData(sizeof(uint32_t)); // channel - } - dst->WriteInt32Fast(body_len); - dst->WriteBytes(compressed.getData(), compressed.getDataLen()); - dst->AssertValid(); - } - } -} - //============================================================ FS4Properties::FS4Properties() @@ -230,14 +50,7 @@ FS4Properties::operator=(FS4Properties && rhs) return *this; } -FS4Properties::~FS4Properties() { } - -void -FS4Properties::allocEntries(uint32_t cnt) -{ - _entries.resize(cnt); - _backing.reserve(cnt*2*40); // Assume strings are average 40 bytes -} +FS4Properties::~FS4Properties() = default; void FS4Properties::set(StringRef & e, vespalib::stringref s) { @@ -267,26 +80,29 @@ FS4Properties::getLength() return len; } -void -FS4Properties::encode(FNET_DataBuffer &dst) +vespalib::string +FS4Properties::toString(uint32_t indent) const { - dst.WriteInt32Fast(_name.size()); - dst.WriteBytesFast(_name.c_str(), _name.size()); - dst.WriteInt32Fast(size()); + vespalib::string s; + s += make_string("%*sProperties {\n", indent, ""); + s += make_string("%*s name: ", indent, ""); + s += _name; + s += "\n"; for (uint32_t i = 0; i < size(); ++i) { - dst.WriteInt32Fast(getKeyLen(i)); - dst.WriteBytesFast(getKey(i), getKeyLen(i)); - dst.WriteInt32Fast(getValueLen(i)); - dst.WriteBytesFast(getValue(i), getValueLen(i)); + s += make_string("%*s Entry[%d] {\n", indent, "", i); + s += make_string("%*s key : %s\n", indent, "", vespalib::string(getKey(i), getKeyLen(i)).c_str()); + s += make_string("%*s value: %s\n", indent, "", vespalib::string(getValue(i), getValueLen(i)).c_str()); + s += make_string("%*s }\n", indent, ""); } + s += make_string("%*s}\n", indent, ""); + return s; } bool FS4Properties::decode(FNET_DataBuffer &src, uint32_t &len) { - uint32_t strLen; if (len < sizeof(uint32_t)) return false; - strLen = src.ReadInt32(); + uint32_t strLen = src.ReadInt32(); len -= sizeof(uint32_t); if (len < strLen) return false; setName(src.GetData(), strLen); @@ -315,1467 +131,11 @@ FS4Properties::decode(FNET_DataBuffer &src, uint32_t &len) return true; } -vespalib::string -FS4Properties::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sProperties {\n", indent, ""); - s += make_string("%*s name: ", indent, ""); - s += _name; - s += "\n"; - for (uint32_t i = 0; i < size(); ++i) { - s += make_string("%*s Entry[%d] {\n", indent, "", i); - s += make_string("%*s key : %s\n", indent, "", vespalib::string(getKey(i), getKeyLen(i)).c_str()); - s += make_string("%*s value: %s\n", indent, "", vespalib::string(getValue(i), getValueLen(i)).c_str()); - s += make_string("%*s }\n", indent, ""); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -/** - * Write a string in usual format to a buffer. Usual format is first - * a 32-bit integer holding the string length, then the bytes that the - * string contained. Skip checking for free space. - * - * @param buf buffer to write to - * @param str string to write, of any type that has c_str() and size() - **/ -template<typename STR> -void -writeLenString(FNET_DataBuffer *buf, const STR &str) -{ - buf->WriteInt32Fast(str.size()); - buf->WriteBytesFast(str.c_str(), str.size()); -} - -//============================================================ - -FS4Packet::FS4Packet() - : FNET_Packet() -{ } - -FS4Packet::~FS4Packet() { } - -void -FS4Packet::Free() { - delete this; -} - -vespalib::string -FS4Packet::Print(uint32_t indent) { - return toString(indent); -} - -//============================================================ - -FS4Packet_EOL::FS4Packet_EOL() - : FS4Packet() -{ } - -FS4Packet_EOL::~FS4Packet_EOL() { } - -uint32_t -FS4Packet_EOL::GetLength() { - return 0; -} - -void -FS4Packet_EOL::Encode(FNET_DataBuffer *dst) { - (void) dst; -} - -bool -FS4Packet_EOL::Decode(FNET_DataBuffer *src, uint32_t len) -{ - src->DataToDead(len); - return (len == 0); -} - -vespalib::string -FS4Packet_EOL::toString(uint32_t indent) const -{ - return make_string("%*sFS4Packet_EOL {}\n", indent, ""); -} - -//============================================================ - -FS4Packet_Shared::FS4Packet_Shared(FNET_Packet::SP packet) - : FS4Packet(), - _packet(std::move(packet)) -{ } - -FS4Packet_Shared::~FS4Packet_Shared() { } - -uint32_t -FS4Packet_Shared::GetPCODE() { - return _packet->GetPCODE(); -} - -uint32_t -FS4Packet_Shared::GetLength() { - return _packet->GetLength(); -} - -void -FS4Packet_Shared::Encode(FNET_DataBuffer *dst) { - _packet->Encode(dst); -} - -bool -FS4Packet_Shared::Decode(FNET_DataBuffer *, uint32_t ) { - LOG_ABORT("should not be reached"); -} - -vespalib::string -FS4Packet_Shared::toString(uint32_t indent) const -{ - return _packet->Print(indent); -} - -//============================================================ - -FS4Packet_PreSerialized::FS4Packet_PreSerialized(FNET_Packet & packet) - : FS4Packet(), - _pcode(packet.GetPCODE()), - _compressionType(CompressionConfig::NONE), - _data(packet.GetLength() + 1*sizeof(uint32_t)) -{ - const uint32_t body_len(packet.GetLength()); - const uint32_t compressionLimit=FS4PersistentPacketStreamer::Instance.getCompressionLimit(); - if (compressionLimit && (body_len > compressionLimit)) { - FNET_DataBuffer tmp(packet.GetLength()); - packet.Encode(&tmp); - tmp.AssertValid(); - CompressionConfig config(FS4PersistentPacketStreamer::Instance.getCompressionType(), - FS4PersistentPacketStreamer::Instance.getCompressionLevel(), - 90); - ConstBufferRef org(tmp.GetData(), tmp.GetDataLen()); - vespalib::DataBuffer compressed(org.size()); - _compressionType = vespalib::compression::compress(config, org, compressed, false); - if (_compressionType != CompressionConfig::NONE) { - _data.WriteInt32Fast(body_len); - _data.WriteBytes(compressed.getData(), compressed.getDataLen()); - _data.AssertValid(); - } else { - packet.Encode(&_data); - } - } else { - packet.Encode(&_data); - } -} - -FS4Packet_PreSerialized::~FS4Packet_PreSerialized() { } - -uint32_t -FS4Packet_PreSerialized::GetPCODE() -{ - return ((_compressionType == CompressionConfig::NONE) - ? _pcode - : (_pcode | (_compressionType << 24))); -} - -uint32_t -FS4Packet_PreSerialized::GetLength() -{ - return _data.GetDataLen(); -} - -void -FS4Packet_PreSerialized::Encode(FNET_DataBuffer *dst) -{ - dst->WriteBytes(_data.GetData(), _data.GetDataLen()); -} - -bool -FS4Packet_PreSerialized::Decode(FNET_DataBuffer *, uint32_t) -{ - LOG_ABORT("should not be reached"); -} - -vespalib::string -FS4Packet_PreSerialized::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_PreSerialized {\n", indent, ""); - s += make_string("%*s length : %d\n", indent, "", _data.GetDataLen()); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -FS4Packet_ERROR::FS4Packet_ERROR() - : FS4Packet(), - _errorCode(0), - _message() -{ } - -FS4Packet_ERROR::~FS4Packet_ERROR() { } - -uint32_t -FS4Packet_ERROR::GetLength() -{ - return 2 * sizeof(uint32_t) + _message.size(); -} - -void -FS4Packet_ERROR::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_errorCode); - writeLenString(dst, _message); -} - -bool -FS4Packet_ERROR::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < sizeof(uint32_t) * 2) { - src->DataToDead(len); - return false; - } - _errorCode = src->ReadInt32(); - uint32_t messageLen = src->ReadInt32(); - len -= 2 * sizeof(uint32_t); - if (len != messageLen) { - src->DataToDead(len); - return false; - } - setErrorMessage(stringref(src->GetData(), messageLen)); - src->DataToDead(messageLen); - return true; -} - - -vespalib::string -FS4Packet_ERROR::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_ERROR {\n", indent, ""); - s += make_string("%*s errorCode : %d\n", indent, "", _errorCode); - s += make_string("%*s message : %s\n", indent, "", _message.c_str()); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -void -FS4Packet_DOCSUM::SetBuf(const char *buf, uint32_t len) -{ - _buf.resize(len); - memcpy(_buf.str(), buf, len); -} - -FS4Packet_DOCSUM::FS4Packet_DOCSUM() - : FS4Packet(), - _gid(), - _buf() -{ } - -FS4Packet_DOCSUM::~FS4Packet_DOCSUM() { } - void -FS4Packet_DOCSUM::Encode(FNET_DataBuffer *dst) -{ - dst->WriteBytesFast(_gid.get(), document::GlobalId::LENGTH); - dst->WriteBytesFast(_buf.c_str(), _buf.size()); -} - -bool -FS4Packet_DOCSUM::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < document::GlobalId::LENGTH) { - src->DataToDead(len); - return false; - } - unsigned char rawGid[document::GlobalId::LENGTH]; - src->ReadBytes(rawGid, document::GlobalId::LENGTH); - _gid.set(rawGid); - len -= document::GlobalId::LENGTH; - SetBuf(src->GetData(), len); - src->DataToDead(len); - return true; -} - -vespalib::string -FS4Packet_DOCSUM::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_DOCSUM {\n", indent, ""); - s += make_string("%*s gid : %s\n", indent, "", _gid.toString().c_str()); - - uint32_t magic = SLIME_MAGIC_ID; - if (_buf.size() >= sizeof(magic) && - memcmp(_buf.c_str(), &magic, sizeof(magic)) == 0) { - vespalib::Slime slime; - vespalib::Memory input(_buf.c_str() + sizeof(magic), - _buf.size() - sizeof(magic)); - vespalib::SimpleBuffer buf; - vespalib::slime::BinaryFormat::decode(input, slime); - vespalib::slime::JsonFormat::encode(slime, buf, false); - s += make_string("%*s json dump : ", indent, ""); - s += buf.get().make_string(); - } else { - s += make_string("%*s data dump :\n", indent, ""); - const char *pt = _buf.c_str(); - uint32_t i = 0; - if ( ! _buf.empty()) - s += make_string("%*s ", indent, ""); - while (i < _buf.size()) { - s += make_string("%x ", (unsigned char) pt[i]); - if ((++i % 16) == 0) - s += make_string("\n%*s ", indent, ""); - } - if ((i % 16) != 0) - s += make_string("\n"); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -FS4Packet_MONITORQUERYX::FS4Packet_MONITORQUERYX() - : FS4Packet(), - _features(0), - _qflags(0u) -{ -} - - -FS4Packet_MONITORQUERYX::~FS4Packet_MONITORQUERYX() -{ -} - - -uint32_t -FS4Packet_MONITORQUERYX::GetLength() -{ - uint32_t plen = 0; - - plen += sizeof(uint32_t); - if (_features & MQF_QFLAGS) - plen += sizeof(uint32_t); - return plen; -} - - -void -FS4Packet_MONITORQUERYX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - if ((_features & MQF_QFLAGS) != 0) - dst->WriteInt32Fast(_qflags); -} - - -bool -FS4Packet_MONITORQUERYX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < sizeof(uint32_t)) - goto error; - _features = src->ReadInt32(); - len -= sizeof(uint32_t); - if ((_features & ~FNET_MQF_SUPPORTED_MASK) != 0) - goto error; - - if ((_features & MQF_QFLAGS) != 0) { - if (len < sizeof(uint32_t)) - goto error; - _qflags = src->ReadInt32(); - len -= sizeof(uint32_t); - } - - if (len != 0) - goto error; - - return true; - error: - src->DataToDead(len); - return false; // FAIL -} - - -vespalib::string -FS4Packet_MONITORQUERYX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_MONITORQUERYX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s qflags : %d\n", indent, "", _qflags); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -FS4Packet_MONITORRESULTX::FS4Packet_MONITORRESULTX() - : FS4Packet(), - _features(0), - _partid(0), - _timestamp(0), - _totalNodes(0), - _activeNodes(0), - _totalParts(0), - _activeParts(0), - _rflags(0u), - _activeDocs(0) -{ } - -FS4Packet_MONITORRESULTX::~FS4Packet_MONITORRESULTX() { } - -uint32_t -FS4Packet_MONITORRESULTX::GetLength() -{ - uint32_t plen = 2 * sizeof(uint32_t); - - plen += sizeof(uint32_t); - if ((_features & MRF_MLD) != 0) - plen += 4 * sizeof(uint32_t); - if ((_features & MRF_RFLAGS) != 0) - plen += sizeof(uint32_t); - if ((_features & MRF_ACTIVEDOCS) != 0) - plen += sizeof(uint64_t); - - return plen; -} - - -void -FS4Packet_MONITORRESULTX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - dst->WriteInt32Fast(_partid); - dst->WriteInt32Fast(_timestamp); - if ((_features & MRF_MLD) != 0) { - dst->WriteInt32Fast(_totalNodes); - dst->WriteInt32Fast(_activeNodes); - dst->WriteInt32Fast(_totalParts); - dst->WriteInt32Fast(_activeParts); - } - if ((_features & MRF_RFLAGS) != 0) { - dst->WriteInt32Fast(_rflags); - } - if ((_features & MRF_ACTIVEDOCS) != 0) { - dst->WriteInt64Fast(_activeDocs); - } -} - - -bool -FS4Packet_MONITORRESULTX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < sizeof(uint32_t)) goto error; - _features = src->ReadInt32(); - len -= sizeof(uint32_t); - if ((_features & ~FNET_MRF_SUPPORTED_MASK) != 0) - goto error; - - if (len < 2 * sizeof(uint32_t)) - goto error; - _partid = src->ReadInt32(); - _timestamp = src->ReadInt32(); - len -= 2 * sizeof(uint32_t); - - if ((_features & MRF_MLD) != 0) { - if (len < 4 * sizeof(uint32_t)) - goto error; - _totalNodes = src->ReadInt32(); - _activeNodes = src->ReadInt32(); - _totalParts = src->ReadInt32(); - _activeParts = src->ReadInt32(); - len -= 4 * sizeof(uint32_t); - } - - if ((_features & MRF_RFLAGS) != 0) { - if (len < sizeof(uint32_t)) - goto error; - _rflags = src->ReadInt32(); - len -= sizeof(uint32_t); - } - - if ((_features & MRF_ACTIVEDOCS) != 0) { - if (len < sizeof(uint64_t)) - goto error; - _activeDocs = src->ReadInt64(); - len -= sizeof(uint64_t); - } - - if (len != 0) - goto error; - - return true; // OK - error: - src->DataToDead(len); - return false; // FAIL -} - - -vespalib::string -FS4Packet_MONITORRESULTX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_MONITORRESULTX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s partid : %d\n", indent, "", _partid); - s += make_string("%*s timestamp : %d\n", indent, "", _timestamp); - s += make_string("%*s totalnodes : %d\n", indent, "", _totalNodes); - s += make_string("%*s activenodes : %d\n", indent, "", _activeNodes); - s += make_string("%*s totalparts : %d\n", indent, "", _totalParts); - s += make_string("%*s activeparts : %d\n", indent, "", _activeParts); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -void -FS4Packet_QUERYRESULTX::AllocateSortIndex(uint32_t cnt) -{ - if (cnt == 0) - return; - - cnt++; // end of data index entry - _sortIndex = new uint32_t[cnt]; -} - - -void -FS4Packet_QUERYRESULTX::AllocateSortData(uint32_t len) -{ - if (len == 0) - return; - - _sortData = (char *) malloc(len); -} - - -void -FS4Packet_QUERYRESULTX::SetSortDataRef(uint32_t cnt, - uint32_t *sortIndex, - const char *sortData) -{ - if (cnt == 0) - return; - - AllocateSortIndex(cnt); - AllocateSortData(sortIndex[cnt] - sortIndex[0]); - _sortIndex[0] = 0; - search::common::SortData::Copy(cnt, _sortIndex, _sortData, sortIndex, sortData); -} - - -void -FS4Packet_QUERYRESULTX::AllocateGroupData(uint32_t len) -{ - if (len == 0) - return; - - _groupData = (char *) malloc(len); - _groupDataLen = len; -} - - -void -FS4Packet_QUERYRESULTX::SetGroupDataRef(const char *groupData, - uint32_t len) -{ - if (len == 0) - return; - - AllocateGroupData(len); - memcpy(_groupData, groupData, len); -} - - -void -FS4Packet_QUERYRESULTX::AllocateHits(uint32_t cnt) -{ - if (cnt == 0) - return; - - _hits = new FS4_hit[cnt]; - _numDocs = cnt; -} - - -FS4Packet_QUERYRESULTX::FS4Packet_QUERYRESULTX() - : FS4Packet(), - _distributionKey(0), - _nodesQueried(1), - _nodesReplied(1), - _features(QRF_COVERAGE | QRF_EXTENDED_COVERAGE), - _offset(0), - _numDocs(0), - _totNumDocs(0), - _maxRank(0), - _sortIndex(NULL), - _sortData(NULL), - _groupDataLen(0), - _groupData(NULL), - _coverageDocs(0), - _activeDocs(0), - _soonActiveDocs(0), - _coverageDegradeReason(0), - _hits(NULL), - _propsVector() -{ } - - -FS4Packet_QUERYRESULTX::~FS4Packet_QUERYRESULTX() -{ - if (_sortIndex) { delete [] _sortIndex; } - if (_sortData) { free(_sortData); } - if (_groupData) { free(_groupData); } - if (_hits) { delete [] _hits; } -} - - -uint32_t -FS4Packet_QUERYRESULTX::GetLength() -{ - uint32_t plen = 3 * sizeof(uint32_t) + - sizeof(uint64_t) + // hit count is now 64-bit - sizeof(search::HitRank) + - _numDocs * (sizeof(document::GlobalId) + sizeof(search::HitRank)); - - plen += sizeof(uint32_t); - plen += (_features & QRF_COVERAGE_NODES) ? (2 * sizeof(uint16_t)) : 0; - plen += (_features & QRF_MLD) ? (_numDocs * 2 * sizeof(uint32_t)) : 0; - plen += (_features & QRF_GROUPDATA) ? (sizeof(uint32_t) + _groupDataLen) : 0; - plen += 3 * sizeof(uint64_t) + sizeof(uint32_t); - - if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) - plen += _numDocs * sizeof(uint32_t) + (_sortIndex[_numDocs] - _sortIndex[0]); - - if ((_features & QRF_PROPERTIES) != 0) { - plen += sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - } - - return plen; -} - - -void -FS4Packet_QUERYRESULTX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - dst->WriteInt32Fast(_offset); - dst->WriteInt32Fast(_numDocs); - dst->WriteInt64Fast(_totNumDocs); - union { uint64_t INT64; double DOUBLE; } mrval; - mrval.DOUBLE = _maxRank; - dst->WriteInt64Fast(mrval.INT64); - dst->WriteInt32Fast(_distributionKey); - - if (_features & QRF_COVERAGE_NODES) { - dst->WriteInt16Fast(_nodesQueried); - dst->WriteInt16Fast(_nodesReplied); - } - - if (((_features & QRF_SORTDATA) != 0) && - (_numDocs > 0)) - { - uint32_t idx0 = _sortIndex[0]; - // implicit: first index entry always 0 - for (uint32_t i = 1; i <= _numDocs; i++) { - dst->WriteInt32Fast(_sortIndex[i] - idx0); - } - dst->WriteBytesFast(_sortData + idx0, _sortIndex[_numDocs] - idx0); - } - - if ((_features & QRF_GROUPDATA) != 0) { - dst->WriteInt32Fast(_groupDataLen); - dst->WriteBytesFast(_groupData, _groupDataLen); - } - - dst->WriteInt64Fast(_coverageDocs); - dst->WriteInt64Fast(_activeDocs); - dst->WriteInt64Fast(_soonActiveDocs); - dst->WriteInt32Fast(_coverageDegradeReason); - - - for (uint32_t i = 0; i < _numDocs; i++) { - dst->WriteBytesFast(_hits[i]._gid.get(), document::GlobalId::LENGTH); - union { uint64_t INT64; double DOUBLE; } val; - val.DOUBLE = _hits[i]._metric; - dst->WriteInt64Fast(val.INT64); - if ((_features & QRF_MLD) != 0) { - dst->WriteInt32Fast(_hits[i]._partid); - dst->WriteInt32Fast(_hits[i].getDistributionKey()); - } - } - - if ((_features & QRF_PROPERTIES) != 0) { - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } - } -} - - -bool -FS4Packet_QUERYRESULTX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - uint32_t hitSize = sizeof(document::GlobalId); - - if (len < sizeof(uint32_t)) goto error; - _features = src->ReadInt32(); - len -= sizeof(uint32_t); - - if ((_features & ~FNET_QRF_SUPPORTED_MASK) != 0) { - throwUnsupportedFeatures(_features, FNET_QRF_SUPPORTED_MASK); - } - hitSize += sizeof(uint64_t); - - if (len < 3 * sizeof(uint32_t) + sizeof(uint64_t) + sizeof(search::HitRank)) goto error; - _offset = src->ReadInt32(); - _numDocs = src->ReadInt32(); - _totNumDocs = src->ReadInt64(); - union { uint64_t INT64; double DOUBLE; } mrval; - mrval.INT64 = src->ReadInt64(); - _maxRank = mrval.DOUBLE; - _distributionKey = src->ReadInt32(); - len -= 3 * sizeof(uint32_t) + sizeof(uint64_t) + sizeof(search::HitRank); - - if (_features & QRF_COVERAGE_NODES) { - if (len < 2* sizeof(uint16_t)) goto error; - _nodesQueried = src->ReadInt16(); - _nodesReplied = src->ReadInt16(); - len -= 2*sizeof(uint16_t); - } - if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) { - if (len < _numDocs * sizeof(uint32_t)) goto error; - AllocateSortIndex(_numDocs); - _sortIndex[0] = 0; // implicit - for (uint32_t i = 1; i <= _numDocs; i++) { - _sortIndex[i] = src->ReadInt32(); - } - len -= _numDocs * sizeof(uint32_t); - uint32_t sortDataLen = _sortIndex[_numDocs]; - - if (len < sortDataLen) goto error; - AllocateSortData(sortDataLen); - src->ReadBytes(_sortData, sortDataLen); - len -= sortDataLen; - } - - if ((_features & QRF_GROUPDATA) != 0) { - if (len < sizeof(uint32_t)) goto error; - _groupDataLen = src->ReadInt32(); - len -= sizeof(uint32_t); - - if (len < _groupDataLen) goto error; - AllocateGroupData(_groupDataLen); - src->ReadBytes(_groupData, _groupDataLen); - len -= _groupDataLen; - } - - if (len < 3 * sizeof(uint64_t) + sizeof(uint32_t)) goto error; - _coverageDocs = src->ReadInt64(); - _activeDocs = src->ReadInt64(); - _soonActiveDocs = src->ReadInt64(); - _coverageDegradeReason = src->ReadInt32(); - - len -= 3*sizeof(uint64_t) + sizeof(uint32_t); - - if ((_features & QRF_MLD) != 0) { - hitSize += 2 * sizeof(uint32_t); - } - - if (len < _numDocs * hitSize) goto error; - AllocateHits(_numDocs); - unsigned char rawGid[document::GlobalId::LENGTH]; - for (uint32_t i = 0; i < _numDocs; i++) { - src->ReadBytes(rawGid, document::GlobalId::LENGTH); - _hits[i]._gid.set(rawGid); - union { uint64_t INT64; double DOUBLE; } val; - val.INT64 = src->ReadInt64(); - _hits[i]._metric = val.DOUBLE; - if ((_features & QRF_MLD) != 0) { - _hits[i]._partid = src->ReadInt32(); - _hits[i].setDistributionKey(src->ReadInt32()); - } else { - _hits[i]._partid = 0; // partid not available - _hits[i].setDistributionKey(getDistributionKey()); - } - } - len -= _numDocs * hitSize; - - if ((_features & QRF_PROPERTIES) != 0) { - uint32_t sz = src->ReadInt32(); - _propsVector.resize(sz); - len -= sizeof(uint32_t); - for (uint32_t i = 0; i < sz; ++i) { - if (! _propsVector[i].decode(*src, len)) goto error; - } - } - - if (len != 0) goto error; - - return true; // OK - - error: - src->DataToDead(len); - return false; // FAIL -} - - -vespalib::string -FS4Packet_QUERYRESULTX::toString(uint32_t indent) const -{ - vespalib::string s; - uint32_t i; - - s += make_string("%*sFS4Packet_QUERYRESULTX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s offset : %d\n", indent, "", _offset); - s += make_string("%*s numDocs : %d\n", indent, "", _numDocs); - s += make_string("%*s totNumDocs : %" PRIu64 "\n", indent, "", _totNumDocs); - s += make_string("%*s maxRank : %f\n", indent, "", _maxRank); - s += make_string("%*s distrib key : %d\n", indent, "", getDistributionKey()); - if (_numDocs > 0 && _sortIndex != NULL) { - uint32_t offset = _sortIndex[0]; - for (i = 0; i < _numDocs; i++) { - uint32_t end = _sortIndex[i + 1]; - s += make_string("%*s sort[%d] = { 0x", indent, "", i); - for (; offset < end; offset++) - s += make_string("%02x", (unsigned char)*(_sortData + offset)); - s += make_string(" }\n"); - } - } - s += make_string("%*s groupData : %d bytes\n", indent, "", _groupDataLen); - s += make_string("%*s coverageDocs : %" PRIu64 "\n", indent, "", _coverageDocs); - s += make_string("%*s activeDocs : %" PRIu64 "\n", indent, "", _activeDocs); - for (i = 0; i < _numDocs; i++) { - s += make_string("%*s hit {", indent, ""); - s += make_string("gid=%s, ", _hits[i]._gid.toString().c_str()); - s += make_string("metric=%f, ", _hits[i]._metric); - s += make_string("partid=%d, ", _hits[i]._partid); - s += make_string("distribkey=%d, ", _hits[i].getDistributionKey()); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - - -FS4Packet_QUERYX::FS4Packet_QUERYX() - : FS4Packet(), - _timeout(0), - _qflags(0), - _features(0), - _offset(0), - _maxhits(0), - _ranking(), - _propsVector(), - _sortSpec(), - _groupSpec(), - _sessionId(), - _location(), - _numStackItems(0), - _stackDump() -{ } - -FS4Packet_QUERYX::~FS4Packet_QUERYX() { } - -uint32_t -FS4Packet_QUERYX::GetLength() -{ - uint32_t plen = 2 * sizeof(uint32_t); - plen += FNET_DataBuffer::getCompressedPositiveLength(_offset); - plen += FNET_DataBuffer::getCompressedPositiveLength(_maxhits); - plen += sizeof(uint32_t); - - if ((_features & QF_PARSEDQUERY) != 0) { - plen += sizeof(uint32_t)*2; - plen += _stackDump.size(); - } - if ((_features & QF_RANKP) != 0) { - plen += FNET_DataBuffer::getCompressedPositiveLength(_ranking.size()); - plen += _ranking.size(); - } - if ((_features & QF_PROPERTIES) != 0) { - plen += sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - } - - if ((_features & QF_SORTSPEC) != 0) - plen += sizeof(uint32_t) + _sortSpec.size(); - - if ((_features & QF_GROUPSPEC) != 0) - plen += sizeof(uint32_t) + _groupSpec.size(); - - if ((_features & QF_SESSIONID) != 0) - plen += sizeof(uint32_t) + _sessionId.size(); - - if ((_features & QF_LOCATION) != 0) - plen += sizeof(uint32_t) + _location.size(); - - return plen; -} - - -void -FS4Packet_QUERYX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - dst->writeCompressedPositive(_offset); - dst->writeCompressedPositive(_maxhits); - dst->WriteInt32Fast(_timeout); - dst->WriteInt32Fast(_qflags); - - if ((_features & QF_RANKP) != 0) { - dst->writeCompressedPositive(_ranking.size()); - dst->WriteBytesFast(_ranking.c_str(), _ranking.size()); - } - - if ((_features & QF_PROPERTIES) != 0) { - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } - } - - if ((_features & QF_SORTSPEC) != 0) { - dst->WriteInt32Fast(_sortSpec.size()); - dst->WriteBytesFast(_sortSpec.c_str(), _sortSpec.size()); - } - - if ((_features & QF_GROUPSPEC) != 0) { - dst->WriteInt32Fast(_groupSpec.size()); - dst->WriteBytesFast(_groupSpec.c_str(), _groupSpec.size()); - } - - if ((_features & QF_SESSIONID) != 0) { - dst->WriteInt32Fast(_sessionId.size()); - dst->WriteBytesFast(_sessionId.c_str(), _sessionId.size()); - } - - if ((_features & QF_LOCATION) != 0) { - dst->WriteInt32Fast(_location.size()); - dst->WriteBytesFast(_location.c_str(), _location.size()); - } - - if ((_features & QF_PARSEDQUERY) != 0) { - dst->WriteInt32Fast(_numStackItems); - dst->WriteInt32Fast(_stackDump.size()); - dst->WriteBytesFast(_stackDump.c_str(), _stackDump.size()); - } -} - -void FS4Packet::throwPropertieDecodeError(size_t i) -{ - throw vespalib::IllegalArgumentException(make_string("Failed decoding properties[%ld]", i)); -} - -void FS4Packet::throwUnsupportedFeatures(uint32_t features, uint32_t set) -{ - throw vespalib::UnderflowException(make_string("Unsupported features(%x), supported set(%x)", features, set)); -} - -void FS4Packet::throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) -{ - (void) buf; - throw vespalib::UnderflowException(make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); -} - -#define VERIFY_LEN(needed, text) \ - { \ - if (len < needed) { \ - throwNotEnoughData(*src, len, needed, text); \ - } \ - len -= needed; \ - } - -uint32_t FS4Packet::readUInt32(FNET_DataBuffer & buf, uint32_t & len, const char *text) -{ - if (len < sizeof(uint32_t)) { - throwNotEnoughData(buf, len, sizeof(uint32_t), text); \ - } - len -= sizeof(uint32_t); - return buf.ReadInt32(); -} - -void -FS4Packet_GETDOCSUMSX::setTimeout(const fastos::TimeStamp & timeout) -{ - _timeout = std::max(INT64_C(0), timeout.ms()); -} - -fastos::TimeStamp -FS4Packet_GETDOCSUMSX::getTimeout() const -{ - return fastos::TimeStamp(_timeout*fastos::TimeStamp::MS); -} - -void -FS4Packet_QUERYX::setTimeout(const fastos::TimeStamp & timeout) -{ - _timeout = std::max(INT64_C(0), timeout.ms()); -} - -fastos::TimeStamp -FS4Packet_QUERYX::getTimeout() const -{ - return fastos::TimeStamp(_timeout*fastos::TimeStamp::MS); -} - -bool -FS4Packet_QUERYX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - _features = readUInt32(*src, len, "features"); - - if (((_features & ~FNET_QF_SUPPORTED_MASK) != 0)) { - throwUnsupportedFeatures(_features, FNET_QF_SUPPORTED_MASK); - } - _offset = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(_offset); - _maxhits = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(_maxhits); - VERIFY_LEN(2 * sizeof(uint32_t), "offset, maxhits, timeout and qflags"); - _timeout = src->ReadInt32(); - _qflags = src->ReadInt32(); - - if ((_features & QF_RANKP) != 0) { - uint32_t rankingLen = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(rankingLen); - VERIFY_LEN(rankingLen, "ranking blob"); - setRanking(stringref(src->GetData(), rankingLen)); - src->DataToDead(rankingLen); - } - - if ((_features & QF_PROPERTIES) != 0) { - uint32_t cnt = readUInt32(*src, len, "#properties"); - _propsVector.resize(cnt); - for (uint32_t i = 0; i < cnt; ++i) { - if (!_propsVector[i].decode(*src, len)) { - throwPropertieDecodeError(i); - } - } - } - - if ((_features & QF_SORTSPEC) != 0) { - uint32_t sortSpecLen = readUInt32(*src, len, "sortspec length"); - - VERIFY_LEN(sortSpecLen, "sortspec string"); - setSortSpec(stringref(src->GetData(), sortSpecLen)); - src->DataToDead(sortSpecLen); - } - - if ((_features & QF_GROUPSPEC) != 0) { - uint32_t groupSpecLen = readUInt32(*src, len, "groupspec length"); - - VERIFY_LEN(groupSpecLen, "groupspec string"); - setGroupSpec(stringref(src->GetData(), groupSpecLen)); - src->DataToDead(groupSpecLen); - } - - if ((_features & QF_SESSIONID) != 0) { - uint32_t sessionIdLen = readUInt32(*src, len, "sessionid length"); - VERIFY_LEN(sessionIdLen, "sessionid string"); - setSessionId(stringref(src->GetData(), sessionIdLen)); - src->DataToDead(sessionIdLen); - } - - if ((_features & QF_LOCATION) != 0) { - uint32_t locationLen = readUInt32(*src, len, "location length"); - - VERIFY_LEN(locationLen, "location string"); - setLocation(stringref(src->GetData(), locationLen)); - src->DataToDead(locationLen); - } - - if ((_features & QF_PARSEDQUERY) != 0) { - _numStackItems = readUInt32(*src, len, "# querystack items"); - - uint32_t stackDumpLen = readUInt32(*src, len, "stackdump length"); - VERIFY_LEN(stackDumpLen, "stackdump"); - setStackDump(stringref(src->GetData(), stackDumpLen)); - src->DataToDead(stackDumpLen); - } - if (len != 0) { - throwNotEnoughData(*src, len, 0, "eof"); - } - - return true; -} - - -vespalib::string -FS4Packet_QUERYX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_QUERYX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s offset : %d\n", indent, "", _offset); - s += make_string("%*s maxhits : %d\n", indent, "", _maxhits); - s += make_string("%*s qflags : %x\n", indent, "", _qflags); - s += make_string("%*s ranking : %s\n", indent, "", _ranking.c_str()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - s += _propsVector[i].toString(indent + 2); - } - s += make_string("%*s sortspec : %s\n", indent, "", _sortSpec.c_str()); - s += make_string("%*s groupspec : (%d bytes)\n", indent, "", (int)_groupSpec.size()); - s += make_string("%*s sessionId : (%d bytes) %s\n", indent, "", (int)_sessionId.size(), _sessionId.c_str()); - s += make_string("%*s location : %s\n", indent, "", _location.c_str()); - s += make_string("%*s timeout : %d\n", indent, "", _timeout); - s += make_string("%*s stackitems : %d\n", indent, "", _numStackItems); - s += make_string("%*s stack dump :\n", indent, ""); - if (_stackDump.size() > 0) { - const char *pt = _stackDump.c_str(); - s += make_string("%*s ", indent, ""); - uint32_t i = 0; - while (i < _stackDump.size()) { - s += make_string("%x ", (unsigned char) pt[i]); - if ((++i % 16) == 0 && i < _stackDump.size()) { - s += make_string("\n%*s ", indent, ""); - } - } - if ((i % 16) != 0) s += make_string("\n"); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - - -void -FS4Packet_GETDOCSUMSX::AllocateDocIDs(uint32_t cnt) -{ - if (cnt == 0) - return; - - _docid.resize(cnt); -} - - -FS4Packet_GETDOCSUMSX::FS4Packet_GETDOCSUMSX() - : FS4Packet(), - _timeout(0), - _features(0), - _ranking(), - _qflags(0), - _resultClassName(), - _propsVector(), - _stackItems(0), - _stackDump(), - _location(), - _flags(0u), - _docid() -{ } - - -FS4Packet_GETDOCSUMSX::~FS4Packet_GETDOCSUMSX() { } - -uint32_t -FS4Packet_GETDOCSUMSX::GetLength() -{ - uint32_t plen = 2 * sizeof(uint32_t) + - + _docid.size() * (sizeof(document::GlobalId)); - - plen += sizeof(uint32_t); - - if ((_features & GDF_MLD) != 0) - plen += 2 * _docid.size() * sizeof(uint32_t); - - if ((_features & GDF_QUERYSTACK) != 0) - plen += 2 * sizeof(uint32_t) + _stackDump.size(); - - if ((_features & GDF_RESCLASSNAME) != 0) - plen += sizeof(uint32_t) + _resultClassName.size(); - - if ((_features & GDF_PROPERTIES) != 0) { - plen += sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - } - - if ((_features & GDF_RANKP_QFLAGS) != 0) { - plen += FNET_DataBuffer::getCompressedPositiveLength(_ranking.size()); - plen += _ranking.size(); - plen += sizeof(uint32_t); - } - - if ((_features & GDF_LOCATION) != 0) - plen += sizeof(uint32_t) + _location.size(); - - if ((_features & GDF_FLAGS) != 0) - plen += sizeof(uint32_t); - - return plen; -} - - -void -FS4Packet_GETDOCSUMSX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - dst->WriteInt32Fast(0); - dst->WriteInt32Fast(_timeout); - - if ((_features & GDF_RANKP_QFLAGS) != 0) { - dst->writeCompressedPositive(_ranking.size()); - dst->WriteBytesFast(_ranking.c_str(), _ranking.size()); - dst->WriteInt32Fast(_qflags); - } - - if ((_features & GDF_RESCLASSNAME) != 0) { - writeLenString(dst, _resultClassName); - } - - if ((_features & GDF_PROPERTIES) != 0) { - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } - } - - if ((_features & GDF_QUERYSTACK) != 0) { - dst->WriteInt32Fast(_stackItems); - writeLenString(dst, _stackDump); - } - - if ((_features & GDF_LOCATION) != 0) { - writeLenString(dst, _location); - } - - if ((_features & GDF_FLAGS) != 0) { - dst->WriteInt32Fast(_flags); - } - - for (const auto & docid : _docid) { - dst->WriteBytesFast(docid._gid.get(), document::GlobalId::LENGTH); - - if ((_features & GDF_MLD) != 0) { - dst->WriteInt32Fast(docid._partid); - dst->WriteInt32Fast(0); - } - } -} - - -bool -FS4Packet_GETDOCSUMSX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - uint32_t docidSize = sizeof(document::GlobalId); - - _features = readUInt32(*src, len, "features"); - - if ((_features & ~FNET_GDF_SUPPORTED_MASK) != 0) { - throwUnsupportedFeatures(_features, FNET_GDF_SUPPORTED_MASK); - } - - VERIFY_LEN(2*sizeof(uint32_t), "unused and timeout"); - src->ReadInt32(); // unused - _timeout = src->ReadInt32(); - - if ((_features & GDF_RANKP_QFLAGS) != 0) { - uint32_t rankingLen = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(rankingLen); - - VERIFY_LEN(rankingLen, "ranking blob"); - setRanking(vespalib::stringref(src->GetData(), rankingLen)); - src->DataToDead(rankingLen); - - _qflags = readUInt32(*src, len, "qflags"); - } - - if ((_features & GDF_RESCLASSNAME) != 0) { - uint32_t resultClassNameLen = readUInt32(*src, len, "result class name length"); - - VERIFY_LEN(resultClassNameLen, "result class"); - setResultClassName(stringref(src->GetData(), resultClassNameLen)); - src->DataToDead(resultClassNameLen); - } - - if ((_features & GDF_PROPERTIES) != 0) { - uint32_t cnt = readUInt32(*src, len, "#properties"); - _propsVector.resize(cnt); - for (uint32_t i = 0; i < cnt; ++i) { - if (!_propsVector[i].decode(*src, len)) { - throwPropertieDecodeError(i); - } - } - } - - if ((_features & GDF_QUERYSTACK) != 0) { - _stackItems = readUInt32(*src, len, "num stack items"); - uint32_t stackDumpLen = readUInt32(*src, len, "stackdump length"); - VERIFY_LEN(stackDumpLen, "stackdump"); - setStackDump(stringref(src->GetData(), stackDumpLen)); - src->DataToDead(stackDumpLen); - } - - if ((_features & GDF_LOCATION) != 0) { - uint32_t locationLen = readUInt32(*src, len, "location length"); - VERIFY_LEN(locationLen, "location string"); - setLocation(stringref(src->GetData(), locationLen)); - src->DataToDead(locationLen); - } - - if ((_features & GDF_FLAGS) != 0) { - _flags = readUInt32(*src, len, "flags"); - } - - if ((_features & GDF_MLD) != 0) - docidSize += 2 * sizeof(uint32_t); - - AllocateDocIDs(len / docidSize); - - unsigned char rawGid[document::GlobalId::LENGTH]; - for (auto & docid : _docid) { - src->ReadBytes(rawGid, document::GlobalId::LENGTH); - docid._gid.set(rawGid); - - if ((_features & GDF_MLD) != 0) { - docid._partid = src->ReadInt32(); - src->ReadInt32(); // unused - } else { - docid._partid = 0; // partid not available - } - } - len -= _docid.size() * docidSize; - - if (len != 0) { - throwNotEnoughData(*src, len, 0, "eof"); - } - - return true; // OK -} - - -vespalib::string -FS4Packet_GETDOCSUMSX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_GETDOCSUMSX {\n", indent, ""); - s += make_string("%*s features : %d\n", indent, "", _features); - s += make_string("%*s ranking : %s\n", indent, "", _ranking.c_str()); - s += make_string("%*s qflags : %x\n", indent, "", _qflags); - s += make_string("%*s resClassName: %s\n", indent, "", _resultClassName.c_str()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - s += _propsVector[i].toString(indent + 2); - } - s += make_string("%*s stackItems : %d\n", indent, "", _stackItems); - s += make_string("%*s stackDumpLen : %d\n", indent, "", (int)_stackDump.size()); - s += make_string("%*s stackDump :\n", indent, ""); - - uint32_t i = 0; - if (_stackDump.size() > 0) { - const char *pt = _stackDump.c_str(); - s += make_string("%*s ", indent, ""); - while (i < _stackDump.size()) { - s += make_string("%x ", (unsigned char) pt[i]); - if ((++i % 16) == 0) - s += make_string("\n%*s ", indent, ""); - } - if ((i % 16) != 0) s += make_string("\n"); - } - for (const auto & docId : _docid) { - s += make_string("%*s gid=%s, partid=%d\n", indent, "", - docId._gid.toString().c_str(), docId._partid); - } - s += make_string("%*s location : %s\n", indent, "", _location.c_str()); - s += make_string("%*s timeout : %d\n", indent, "", _timeout); - s += make_string("%*s flags : %d\n", indent, "", _flags); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -uint32_t -FS4Packet_TRACEREPLY::GetLength() -{ - uint32_t plen = sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - return plen; -} - -void -FS4Packet_TRACEREPLY::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } -} - -bool -FS4Packet_TRACEREPLY::Decode(FNET_DataBuffer *src, uint32_t len) -{ - uint32_t cnt = readUInt32(*src, len, "#properties"); - _propsVector.resize(cnt); - for (uint32_t i = 0; i < cnt; ++i) { - if (!_propsVector[i].decode(*src, len)) { - throwPropertieDecodeError(i); - } - } - if (len != 0) goto error; - return true; // OK - error: - src->DataToDead(len); - return false; // FAIL -} - -vespalib::string -FS4Packet_TRACEREPLY::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_TRACEREPLY {\n", indent, ""); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - s += _propsVector[i].toString(indent + 2); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - - -//============================================================ - -FNET_Packet* -FS4PacketFactory::CreateFS4Packet(uint32_t pcode) +FS4Properties::allocEntries(uint32_t cnt) { - switch(pcode) { - case PCODE_EOL: - return new FS4Packet_EOL; - case PCODE_ERROR: - return new FS4Packet_ERROR; - case PCODE_DOCSUM: - return new FS4Packet_DOCSUM; - case PCODE_QUERYRESULTX: - return new FS4Packet_QUERYRESULTX; - case PCODE_QUERYX: - return new FS4Packet_QUERYX; - case PCODE_GETDOCSUMSX: - return new FS4Packet_GETDOCSUMSX; - case PCODE_MONITORQUERYX: - return new FS4Packet_MONITORQUERYX; - case PCODE_MONITORRESULTX: - return new FS4Packet_MONITORRESULTX; - case PCODE_TRACEREPLY: - return new FS4Packet_TRACEREPLY; - default: - return NULL; - } + _entries.resize(cnt); + _backing.reserve(cnt*2*40); // Assume strings are average 40 bytes } } diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index 3fd87ccfa25..5e0072c27c0 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -2,148 +2,28 @@ #pragma once -#include "transport.h" -#include "hitrank.h" -#include <vespa/fnet/context.h> -#include <vespa/fnet/ipacketstreamer.h> -#include <vespa/fnet/packet.h> -#include <vespa/fnet/databuffer.h> -#include <vespa/document/base/globalid.h> #include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/memory.h> -#include <vespa/fastos/timestamp.h> #include <vector> +class FNET_DataBuffer; + namespace search::fs4transport { using vespalib::string; -enum fnet_feature_masks { - FNET_QRF_SUPPORTED_MASK = (QRF_MLD | - QRF_SORTDATA | - QRF_COVERAGE_NODES | - QRF_EXTENDED_COVERAGE | - QRF_COVERAGE | - QRF_GROUPDATA | - QRF_PROPERTIES), - - FNET_QF_SUPPORTED_MASK = (QF_PARSEDQUERY | - QF_RANKP | - QF_SORTSPEC | - QF_LOCATION | - QF_PROPERTIES | - QF_GROUPSPEC | - QF_SESSIONID), - - FNET_GDF_SUPPORTED_MASK = (GDF_MLD | - GDF_QUERYSTACK | - GDF_RANKP_QFLAGS | - GDF_LOCATION | - GDF_RESCLASSNAME | - GDF_PROPERTIES | - GDF_FLAGS), - ACTIVE_QUERY_FLAGS = (QFLAG_EXTENDED_COVERAGE | - QFLAG_COVERAGE_NODES | - QFLAG_ESTIMATE | - QFLAG_DROP_SORTDATA | - QFLAG_NO_RESULTCACHE | - QFLAG_DUMP_FEATURES), - - FNET_MQF_SUPPORTED_MASK = (MQF_QFLAGS), - - FNET_MRF_SUPPORTED_MASK = (MRF_MLD | MRF_RFLAGS | MRF_ACTIVEDOCS) -}; - -enum pcode_mask { - PCODE_MASK = 0x00ffffff -}; - -//========================================================================== - -class PacketArray -{ -private: - PacketArray(const PacketArray &); - PacketArray& operator=(const PacketArray &); - - FNET_Packet **_extArray; - FNET_Packet **_array; - uint32_t _size; - uint32_t _used; - -public: - PacketArray(FNET_Packet **arr = nullptr, uint32_t size = 0) - : _extArray(arr), - _array(arr), - _size(size), - _used(0) {} - ~PacketArray() - { - if (_array != _extArray) - delete [] _array; - } - void Add(FNET_Packet *packet) - { - if (_used == _size) { - _size *= 2; - if (_size < 16) - _size = 16; - FNET_Packet **newArray = new FNET_Packet*[_size]; - for (uint32_t i = 0; i < _used; i++) - newArray[i] = _array[i]; - if (_array != _extArray) - delete [] _array; - _array = newArray; - } - _array[_used++] = packet; - } - FNET_Packet **Array() const { return _array; } - uint32_t Length() const { return _used; } -}; - -//========================================================================== - -class FS4PacketFactory -{ -public: - typedef FNET_Packet *(* CreatePacket_t)(uint32_t pcode); - - static FNET_Packet *CreateFS4Packet(uint32_t pcode); -}; - -//========================================================================== - -class FS4PersistentPacketStreamer : public FNET_IPacketStreamer { - FS4PersistentPacketStreamer(const FS4PersistentPacketStreamer &); - FS4PersistentPacketStreamer& operator=(const FS4PersistentPacketStreamer &); +class FS4PersistentPacketStreamer { using CompressionConfig = vespalib::compression::CompressionConfig; unsigned int _compressionLimit; unsigned int _compressionLevel; CompressionConfig::Type _compressionType; -protected: - bool _conservative; // Set to true if out of sync should mark the - // stream as broken. - FS4PacketFactory::CreatePacket_t _createPacket; - - bool HasChannelID(uint32_t pcode); - bool ValidPCode(uint32_t pcode) const { - return ((pcode & PCODE_MASK) >= PCODE_EOL) - && ((pcode & PCODE_MASK) < PCODE_LastCode); - } public: static FS4PersistentPacketStreamer Instance; - FS4PersistentPacketStreamer(FS4PacketFactory::CreatePacket_t cp); + FS4PersistentPacketStreamer(); - bool GetPacketInfo(FNET_DataBuffer *src, uint32_t *plen, - uint32_t *pcode, uint32_t *chid, bool *broken) override; - FNET_Packet *Decode(FNET_DataBuffer *src, uint32_t plen, - uint32_t pcode, FNET_Context context) override; - void Encode(FNET_Packet *packet, uint32_t chid, FNET_DataBuffer *dst) override; - - void SetConservativeMode(bool cons) { _conservative = cons; } void SetCompressionLimit(unsigned int limit) { _compressionLimit = limit; } void SetCompressionLevel(unsigned int level) { _compressionLevel = level; } void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType = compressionType; } @@ -166,13 +46,13 @@ private: vespalib::string _backing; const char * c_str(size_t sz) const { return _backing.c_str() + sz; } void set(StringRef & e, vespalib::stringref s); + void allocEntries(uint32_t cnt); public: FS4Properties(FS4Properties &&); FS4Properties &operator=(FS4Properties &&); FS4Properties(); ~FS4Properties(); - void allocEntries(uint32_t cnt); void setName(const char *name, uint32_t nameSize) { _name.assign(name, nameSize); } void setName(vespalib::stringref val) { setName(val.data(), val.size()); @@ -195,355 +75,10 @@ public: // sub-packet methods below uint32_t getLength(); + void encode(FNET_DataBuffer &dst); bool decode(FNET_DataBuffer &src, uint32_t &len); vespalib::string toString(uint32_t indent = 0) const; }; -//========================================================================== - -typedef std::vector<FS4Properties> PropsVector; - -//========================================================================== - -class FS4Packet : public FNET_Packet -{ -private: - FS4Packet(const FS4Packet &); - FS4Packet& operator=(const FS4Packet &); - -public: - FS4Packet(); - ~FS4Packet(); - vespalib::string Print(uint32_t indent) override; - void Free() override; - virtual vespalib::string toString(uint32_t indent) const = 0; -protected: - uint32_t readUInt32(FNET_DataBuffer & buf, uint32_t & len, const char *text) __attribute__((noinline)); - void throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) __attribute__((noinline)); - void throwUnsupportedFeatures(uint32_t features, uint32_t set) __attribute__((noinline)); - void throwPropertieDecodeError(size_t i) __attribute__((noinline)); -}; - -//========================================================================== - -class FS4Packet_EOL : public FS4Packet -{ -public: - FS4Packet_EOL(); - ~FS4Packet_EOL(); - uint32_t GetPCODE() override { return PCODE_EOL; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -class FS4Packet_PreSerialized : public FS4Packet -{ -public: - FS4Packet_PreSerialized(FNET_Packet & packet); - ~FS4Packet_PreSerialized(); - uint32_t GetPCODE() override; - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -private: - using CompressionConfig = vespalib::compression::CompressionConfig; - uint32_t _pcode; - CompressionConfig::Type _compressionType; - FNET_DataBuffer _data; -}; - -class FS4Packet_Shared : public FS4Packet -{ -public: - FS4Packet_Shared(FNET_Packet::SP packet); - ~FS4Packet_Shared(); - uint32_t GetPCODE() override; - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *, uint32_t ) override; - vespalib::string toString(uint32_t indent) const override; -private: - FNET_Packet::SP _packet; -}; - -//========================================================================== - -class FS4Packet_ERROR : public FS4Packet -{ -private: - FS4Packet_ERROR(const FS4Packet_ERROR &); - FS4Packet_ERROR& operator=(const FS4Packet_ERROR &); - -public: - uint32_t _errorCode; - string _message; - - void setErrorMessage(vespalib::stringref msg) { _message = msg; } - - FS4Packet_ERROR(); - ~FS4Packet_ERROR(); - uint32_t GetPCODE() override { return PCODE_ERROR; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_DOCSUM : public FS4Packet -{ -public: - typedef vespalib::MallocPtr Buf; -private: - FS4Packet_DOCSUM(const FS4Packet_DOCSUM &); - FS4Packet_DOCSUM& operator=(const FS4Packet_DOCSUM &); - - document::GlobalId _gid; - Buf _buf; -public: - FS4Packet_DOCSUM(); - ~FS4Packet_DOCSUM(); - const Buf & getBuf() const { return _buf; } - void swapBuf(Buf & other) { _buf.swap(other); } - void setGid(const document::GlobalId & gid) { _gid = gid; } - const document::GlobalId & getGid() const { return _gid; } - bool empty() const { return _buf.empty(); } - void SetBuf(const char *buf, uint32_t len); - uint32_t GetPCODE() override { return PCODE_DOCSUM; } - uint32_t GetLength() override { return sizeof(_gid) + _buf.size(); } - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_MONITORQUERYX : public FS4Packet -{ - FS4Packet_MONITORQUERYX(const FS4Packet_MONITORQUERYX &); - FS4Packet_MONITORQUERYX& operator=(const FS4Packet_MONITORQUERYX &); - -public: - uint32_t _features; // see monitorquery_features - uint32_t _qflags; // if MQF_QFLAGS - - FS4Packet_MONITORQUERYX(); - ~FS4Packet_MONITORQUERYX(); - uint32_t GetPCODE() override { return PCODE_MONITORQUERYX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_MONITORRESULTX : public FS4Packet -{ -private: - FS4Packet_MONITORRESULTX(const FS4Packet_MONITORRESULTX &); - FS4Packet_MONITORRESULTX& operator=(const FS4Packet_MONITORRESULTX &); - -public: - uint32_t _features; // see monitor - uint32_t _partid; - uint32_t _timestamp; - - uint32_t _totalNodes; // if MRF_MLD - uint32_t _activeNodes; // if MRF_MLD - uint32_t _totalParts; // if MRF_MLD - uint32_t _activeParts; // if MRF_MLD - - uint32_t _rflags; // if MRF_RFLAGS - uint64_t _activeDocs; // if MRF_ACTIVEDOCS - - FS4Packet_MONITORRESULTX(); - ~FS4Packet_MONITORRESULTX(); - uint32_t GetPCODE() override { return PCODE_MONITORRESULTX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_QUERYRESULTX : public FS4Packet -{ -private: - FS4Packet_QUERYRESULTX(const FS4Packet_QUERYRESULTX &); - FS4Packet_QUERYRESULTX& operator=(const FS4Packet_QUERYRESULTX &); - - uint32_t _distributionKey; - uint16_t _nodesQueried; - uint16_t _nodesReplied; - -public: - uint32_t _features; // see queryresult_features - uint32_t _offset; - uint32_t _numDocs; - uint64_t _totNumDocs; - search::HitRank _maxRank; - uint32_t *_sortIndex; // if QRF_SORTDATA - char *_sortData; // if QRF_SORTDATA - uint32_t _groupDataLen; // if QRF_GROUPDATA - char *_groupData; // if QRF_GROUPDATA - uint64_t _coverageDocs; // if QRF_COVERAGE - uint64_t _activeDocs; // if QRF_COVERAGE - uint64_t _soonActiveDocs; // if QRF_EXTENDED_COVERAGE - uint32_t _coverageDegradeReason; // if QRF_EXTENDED_COVERAGE - class FS4_hit { - public: - FS4_hit() : _gid(), _metric(0), _partid(0), _distributionKey(0) { } - uint32_t getDistributionKey() const { return _distributionKey; } - void setDistributionKey(uint32_t key) { _distributionKey = key; } - const document::GlobalId & HT_GetGlobalID() const { return _gid; } - search::HitRank HT_GetMetric() const { return _metric; } - uint32_t HT_GetPartID() const { return _partid; } - - void HT_SetGlobalID(const document::GlobalId & val) { _gid = val; } - void HT_SetMetric(search::HitRank val) { _metric = val; } - void HT_SetPartID(uint32_t val) { _partid = val; } - document::GlobalId _gid; - search::HitRank _metric; - uint32_t _partid; // if QRF_MLD - private: - uint32_t _distributionKey; // if QRF_MLD - } *_hits; - PropsVector _propsVector; // if QRF_PROPERTIES - - void AllocateSortIndex(uint32_t cnt); - void AllocateSortData(uint32_t len); - void SetSortDataRef(uint32_t cnt, uint32_t *sortIndex, const char *sortData); - void AllocateGroupData(uint32_t len); - void SetGroupDataRef(const char *groupData, uint32_t len); - void AllocateHits(uint32_t cnt); - - FS4Packet_QUERYRESULTX(); - ~FS4Packet_QUERYRESULTX(); - uint32_t GetPCODE() override { return PCODE_QUERYRESULTX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override ; - vespalib::string toString(uint32_t indent) const override ; - uint32_t getDistributionKey() const { return _distributionKey; } - void setDistributionKey(uint32_t key) { _distributionKey = key; } - uint16_t getNodesQueried() const { return _nodesQueried; } - void setNodesQueried(uint16_t key) { _nodesQueried = key; } - uint16_t getNodesReplied() const { return _nodesReplied; } - void setNodesReplied(uint16_t key) { _nodesReplied = key; } -}; -//========================================================================== - -class FS4Packet_QUERYX : public FS4Packet -{ -private: - FS4Packet_QUERYX(const FS4Packet_QUERYX &); - FS4Packet_QUERYX& operator=(const FS4Packet_QUERYX &); - - uint32_t _timeout; - uint32_t _qflags; - -public: - uint32_t _features; // see query_features - uint32_t _offset; - uint32_t _maxhits; - string _ranking; // if QF_RANKP - PropsVector _propsVector; // if QF_PROPERTIES - string _sortSpec; // if QF_SORTSPEC - string _groupSpec; // if QF_GROUPSPEC - string _sessionId; // if QF_SESSIONID - string _location; // if QF_LOCATION - - uint32_t _numStackItems; // if QF_PARSEDQUERY - string _stackDump; // if QF_PARSEDQUERY - - void setQueryFlags(uint32_t qflags) { _qflags = ACTIVE_QUERY_FLAGS & qflags; } - void setRanking(vespalib::stringref ranking) { _ranking = ranking; } - void setSortSpec(vespalib::stringref spec) { _sortSpec = spec; } - void setGroupSpec(vespalib::stringref spec) { _groupSpec = spec; } - void setSessionId(vespalib::stringref sid) { _sessionId = sid; } - void setLocation(vespalib::stringref loc) { _location = loc; } - void setStackDump(vespalib::stringref buf) { _stackDump = buf; } - void setTimeout(const fastos::TimeStamp & timeout); - fastos::TimeStamp getTimeout() const; - uint32_t getQueryFlags() const { return _qflags; } - - explicit FS4Packet_QUERYX(); - ~FS4Packet_QUERYX(); - uint32_t GetPCODE() override { return PCODE_QUERYX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_GETDOCSUMSX : public FS4Packet -{ -private: - FS4Packet_GETDOCSUMSX(const FS4Packet_GETDOCSUMSX &); - FS4Packet_GETDOCSUMSX& operator=(const FS4Packet_GETDOCSUMSX &); - - uint32_t _timeout; -public: - uint32_t _features; // see getdocsums_features - string _ranking; // if GDF_RANKP_QFLAGS - uint32_t _qflags; // if GDF_RANKP_QFLAGS - string _resultClassName; // if GDF_RESCLASSNAME - PropsVector _propsVector; // if GDF_PROPERTIES - uint32_t _stackItems; // if GDF_QUERYSTACK - string _stackDump; // if GDF_QUERYSTACK - string _location; // if GDF_LOCATION - uint32_t _flags; // if GDF_FLAGS - class FS4_docid { - public: - FS4_docid() : _gid(), _partid(0) { } - document::GlobalId _gid; - uint32_t _partid; // if GDF_MLD - }; - std::vector<FS4_docid> _docid; - - void AllocateDocIDs(uint32_t cnt); - - void setResultClassName(vespalib::stringref name) { _resultClassName = name; } - void setStackDump(vespalib::stringref buf) { _stackDump = buf; } - void setRanking(vespalib::stringref ranking) { _ranking = ranking; } - void setLocation(vespalib::stringref loc) { _location = loc; } - void setTimeout(const fastos::TimeStamp & timeout); - fastos::TimeStamp getTimeout() const; - - FS4Packet_GETDOCSUMSX(); - ~FS4Packet_GETDOCSUMSX(); - uint32_t GetPCODE() override { return PCODE_GETDOCSUMSX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_TRACEREPLY : public FS4Packet -{ -public: - FS4Packet_TRACEREPLY() {} - ~FS4Packet_TRACEREPLY() {} - uint32_t GetPCODE() override { return PCODE_TRACEREPLY; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; - - PropsVector _propsVector; -}; - -//========================================================================== - } diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h index 51098112476..301acde7bd6 100644 --- a/searchlib/src/vespa/searchlib/common/transport.h +++ b/searchlib/src/vespa/searchlib/common/transport.h @@ -25,262 +25,11 @@ namespace search::fs4transport { * </ul> **/ enum queryflags { - QFLAG_EXTENDED_COVERAGE = 0x00000001, - QFLAG_COVERAGE_NODES = 0x00000002, - QFLAG_ESTIMATE = 0x00000080, QFLAG_DROP_SORTDATA = 0x00004000, - QFLAG_NO_RESULTCACHE = 0x00010000, QFLAG_DUMP_FEATURES = 0x00040000 }; - -/** - * The new PCODE_QUERYRESULTX packet contains a 32-bit field called - * 'featureflags'. Each bit in that field denotes a separate feature - * that may be present in the query result packet or not. The comment - * describing the packet format indicates what data fields depend on - * what features. The features present in the 'old' query result packets - * are defined in this enum along with the Query Result Features - * themselves. The value called QRF_SUPPORTED_MASK denotes which - * features are supported by the current version. If a packet with - * unknown features is received on the network is is discarded (as it - * would be if it had an illegal PCODE). - **/ -enum queryresult_features { - QRF_MLD = 0x00000001, - QRF_COVERAGE_NODES = 0x00000002, - QRF_SORTDATA = 0x00000010, - QRF_EXTENDED_COVERAGE = 0x00000020, - QRF_COVERAGE = 0x00000040, - QRF_GROUPDATA = 0x00000200, - QRF_PROPERTIES = 0x00000400 -}; - - -/** - * The new PCODE_QUERYX packet contains a 32-bit field called - * 'featureflags'. Each bit in that field denotes a separate feature - * that may be present in the query packet or not. The comment - * describing the packet format indicates what data fields depend on - * what features. The features present in the - * 'old' query packets are defined in this enum along with the Query - * Features themselves. The values called - * QF_SUPPORTED_[FSEARCH/FDISPATCH]_MASK denotes which features are - * supported by the current version. If a packet with unknown features - * is received on the network is is discarded (as it would be if it - * had an illegal PCODE). - **/ -enum query_features { - QF_PARSEDQUERY = 0x00000002, - QF_RANKP = 0x00000004, - QF_SORTSPEC = 0x00000080, - QF_LOCATION = 0x00000800, - QF_PROPERTIES = 0x00100000, - QF_GROUPSPEC = 0x00400000, - QF_SESSIONID = 0x00800000 -}; - - -/** - * The new PCODE_GETDOCSUMSX packet contains a 32-bit field called - * 'featureflags'. Each bit in that field denotes a separate feature - * that may be present in the getdocsums packet or not. The comment - * describing the packet format indicates what data fields depend on - * what features. The features present in the 'old' getdocsums packets are - * defined in this enum along with the GetDocsums Features - * themselves. The values called - * GDF_SUPPORTED_[FSEARCH/FDISPATCH]_MASK denotes which features are - * supported by the current version. If a packet with unknown features - * is received on the network is is discarded (as it would be if it - * had an illegal PCODE). - **/ -enum getdocsums_features { - GDF_MLD = 0x00000001, - GDF_QUERYSTACK = 0x00000004, - GDF_RANKP_QFLAGS = 0x00000010, - GDF_LOCATION = 0x00000080, - GDF_RESCLASSNAME = 0x00000800, - GDF_PROPERTIES = 0x00001000, - GDF_FLAGS = 0x00002000 -}; - - -enum getdocsums_flags { - GDFLAG_IGNORE_ROW = 0x00000001 -}; - // docsum class for slime tunneling const uint32_t SLIME_MAGIC_ID = 0x55555555; -enum monitorquery_features { - MQF_QFLAGS = 0x00000002, -}; - -enum monitorquery_flags { - MQFLAG_REPORT_ACTIVEDOCS = 0x00000020 -}; - -enum monitorresult_features { - MRF_MLD = 0x00000001, - MRF_RFLAGS = 0x00000008, - MRF_ACTIVEDOCS = 0x00000010, -}; - -/** - * Codes for packets between dispatch nodes and search nodes. - * general packet (i.e. message) format: - * uint32_t packetLength- length in bytes, EXCLUDING this length field - * packetcode pCode - see the enum below; same length as uint32_t - * packetData - variable length - */ -enum packetcode { - PCODE_EOL = 200, /* ..fdispatch <-> ..fsearch. PacketData: - *0 {uint32_t queryId,} - only in new format!*/ - PCODE_QUERY_NOTUSED = 201, - PCODE_QUERYRESULT_NOTUSED = 202, - PCODE_ERROR = 203, /* ..fdispatch <- ..fsearch/..fdispatch - * {uint32_t queryId,} - only in new format! - * uint32_t error_code [see common/errorcodes.h] - * uint32_t message_len - * char[] message (UTF-8) */ - PCODE_GETDOCSUMS_NOTUSED = 204, - PCODE_DOCSUM = 205, /* ..fdispatch <- ..fsearch. - *0 {uint32_t queryId,} - only in new format! - *1 uint32_t location - *2 char[] <title, incipit, URL, ...> - */ - PCODE_MONITORQUERY_NOTUSED = 206, - PCODE_MONITORRESULT_NOTUSED = 207, - PCODE_MLD_QUERYRESULT_NOTUSED = 208, - PCODE_MLD_GETDOCSUMS_NOTUSED = 209, - PCODE_MLD_MONITORRESULT_NOTUSED = 210, - PCODE_CLEARCACHES_NOTUSED = 211, - PCODE_QUERY2_NOTUSED = 212, - PCODE_PARSEDQUERY2_NOTUSED = 213, - PCODE_MLD_QUERYRESULT2_NOTUSED = 214, - PCODE_MLD_GETDOCSUMS2_NOTUSED = 215, - PCODE_QUEUELEN_NOTUSED = 216, - - PCODE_QUERYRESULTX = 217, /* - * {uint32_t queryId,} - only if persistent - * uint32_t featureflags, - see 'queryresult_features' - * uint32_t offset, - * uint32_t numDocs, - * uint32_t totNumDocs, - * search::HitRank maxRank, - * uint32_t docstamp, - * uint32_t[numDocs] sortIndex - if QRF_SORTDATA - * char[sidx[n - 1]] sortData - if QRF_SORTDATA - * uint32_t groupDataLen - if QRF_GROUPDATA - * char[groupDataLen] groupData - if QRF_GROUPDATA - * uint64_t coverageDocs - if QRF_COVERAGE - * uint32_t coverageNodes - if QRF_COVERAGE - * uint32_t coverageFull - if QRF_COVERAGE - * numDocs * hit { - * uint32_t docid, - * search::HitRank metric, - * uint32_t partid, - if QRF_MLD - * uint32_t docstamp, - if QRF_MLD - * } */ - PCODE_QUERYX = 218, /* - * {uint32_t queryId,} - only if persistent - * uint32_t featureflags, - see 'query_features' - * uint32_t querytype - * uint32_t offset, - * uint32_t maxhits, - * uint32_t qflags, - * uint32_t minhits, - if QF_MINHITS - * uint32_t numProperties - if QF_PROPERTIES - * numProperties * props { - if QF_PROPERTIES - * uint32_t nameLen - * char[nameLen] name - * uint32_t numEntries - * numentries * entry { - * uint32_t keyLen - * char[keyLen] key - * uint32_t valueLen - * char[valueLen] value - * } - * } - * uint32_t sortSpecLen - if QF_SORTSPEC - * char[sortSpecLen] sortSpec - if QF_SORTSPEC - * uint32_t groupSpecLen - if QF_GROUPSPEC - * char[groupSpecLen] groupSpec - if QF_GROUPSPEC - * uint32_t locationLen - if QF_LOCATION - * char[locationLen] location - if QF_LOCATION - * uint32_t numStackItems, - if QF_PARSEDQUERY - * multiple encoded stackitems: - if QF_PARSEDQUERY - - uint32_t OR|AND|NOT|RANK - uint32_t arity - - uint32_t PHRASE - uint32_t arity - uint32_t indexNameLen - char[] indexName - - uint32_t TERM - uint32_t indexNameLen - char[] indexName - uint32_t termLen - char[] term - */ - PCODE_GETDOCSUMSX = 219, /* - * {uint32_t queryId,} - only if persistent - * uint32_t featureflags, - see 'getdocsums_features' - * uint32_t docstamp, - * uint32_t rankprofile, - if GDF_RANKP_QFLAGS - * uint32_t qflags, - if GDF_RANKP_QFLAGS - * uint32_t resClassNameLen - if GDF_RESCLASSNAME - * char [] resClassName - if GDF_RESCLASSNAME - * uint32_t numProperties - if GDF_PROPERTIES - * numProperties * props { - if GDF_PROPERTIES - * uint32_t nameLen - * char[nameLen] name - * uint32_t numEntries - * numentries * entry { - * uint32_t keyLen - * char[keyLen] key - * uint32_t valueLen - * char[valueLen] value - * } - * } - * uint32_t stackItems, - if GDF_STACKDUMP - * uint32_t stackDumpLen, - if GDF_STACKDUMP - * char[stackDumpLen] stackDump, - if GDF_STACKDUMP - * uint32_t locationLen - if GDF_LOCATION - * char[locationLen] location - if GDF_LOCATION - * N * doc { - * uint32_t docid, - * uint32_t partid, - if GDF_MLD - * uint32_t docstamp, - if GDF_MLD - * } - */ - PCODE_MONITORQUERYX = 220, /* - * uint32_t featureFlags; - * - see monitorquery_features - */ - PCODE_MONITORRESULTX = 221, /* - * uint32_t featureFlags; - * - see monitorresult_features - * uint32_t partitionId; - * uint32_t timestamp; - * uint32_t totalNodes; - if MRF_MLD - * uint32_t activeNodes; - if MRF_MLD - * uint32_t totalParts; - if MRF_MLD - * uint32_t activeParts; - if MRF_MLD - */ - PCODE_TRACEREPLY = 222, /* - * numProperties * props { - * uint32_t nameLen - * char[nameLen] name - * uint32_t numEntries - * numentries * entry { - * uint32_t keyLen - * char[keyLen] key - * uint32_t valueLen - * char[valueLen] value - * } - * } - */ - PCODE_LastCode = 223 // Used for consistency checking only, must be last. -}; - } diff --git a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt index 36ff4e12108..21a5b232ae0 100644 --- a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt @@ -14,11 +14,9 @@ vespa_add_library(searchlib_engine OBJECT docsumapi.cpp docsumreply.cpp docsumrequest.cpp - errorcodes.cpp lazy_source.cpp monitorreply.cpp monitorrequest.cpp - packetconverter.cpp propertiesmap.cpp proto_converter.cpp proto_rpc_adapter.cpp @@ -27,8 +25,6 @@ vespa_add_library(searchlib_engine OBJECT searchreply.cpp searchrequest.cpp trace.cpp - transport_metrics.cpp - transportserver.cpp ${searchlib_engine_PROTOBUF_SRCS} DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp b/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp index e2dffb54e56..33a2c437a80 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp +++ b/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsumrequest.h" -#include "packetconverter.h" namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/engine/errorcodes.cpp b/searchlib/src/vespa/searchlib/engine/errorcodes.cpp deleted file mode 100644 index 0182d85177a..00000000000 --- a/searchlib/src/vespa/searchlib/engine/errorcodes.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "errorcodes.h" - -namespace search::engine { - -const char * -getStringFromErrorCode(ErrorCode ecode) -{ - switch (ecode) { - case ECODE_NO_ERROR: - return "No error has occurred"; - case ECODE_GENERAL_ERROR: - return "General error"; - case ECODE_QUERY_PARSE_ERROR: - return "Error parsing query"; - case ECODE_ALL_PARTITIONS_DOWN: - return "All searchnodes are down. This might indicate that no index is available yet."; - case ECODE_ILLEGAL_DATASET: - return "No such dataset"; - case ECODE_OVERLOADED: - return "System is overloaded"; - case ECODE_NOT_IMPLEMENTED: - return "The requested functionality is not implemented"; - case ECODE_QUERY_NOT_ALLOWED: - return "Query not allowed to run"; - case ECODE_TIMEOUT: - return "Query timed out"; - } - return "Unknown error"; -} - -} - diff --git a/searchlib/src/vespa/searchlib/engine/errorcodes.h b/searchlib/src/vespa/searchlib/engine/errorcodes.h deleted file mode 100644 index d0812ad9d6c..00000000000 --- a/searchlib/src/vespa/searchlib/engine/errorcodes.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -namespace search::engine { - -/** - * Enum defining global error codes. - * Used in error_code field in search::fs4transport::PCODE_ERROR packets. - **/ -enum ErrorCode { - ECODE_NO_ERROR = 0, - ECODE_GENERAL_ERROR = 1, - ECODE_QUERY_PARSE_ERROR = 2, - ECODE_ALL_PARTITIONS_DOWN = 3, - ECODE_ILLEGAL_DATASET = 4, - ECODE_OVERLOADED = 5, - ECODE_NOT_IMPLEMENTED = 6, - ECODE_QUERY_NOT_ALLOWED = 7, - ECODE_TIMEOUT = 8 -}; - -/** - * Normally error codes should be accompanied by an error message - * describing the error. If no such message is present, this method - * may be used to obtain the default description of an error code. - * - * @param error the error code we want info about. - * @return the default error message for the given error code. - **/ -const char* getStringFromErrorCode(ErrorCode error); - -} - diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp deleted file mode 100644 index 863c204f26c..00000000000 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packetconverter.h" - -#include <vespa/log/log.h> -LOG_SETUP(".engine.packetconverter"); - -using search::fef::Property; -using search::fef::Properties; - -namespace { - -bool checkFeature(uint32_t features, uint32_t mask) { - return ((features & mask) != 0); -} - -struct FS4PropertiesBuilder : public search::fef::IPropertiesVisitor { - uint32_t idx; - search::fs4transport::FS4Properties &props; - FS4PropertiesBuilder(search::fs4transport::FS4Properties &p) : idx(0), props(p) {} - void visitProperty(const Property::Value &key, const Property &values) override { - for (uint32_t i = 0; i < values.size(); ++i) { - props.setKey(idx, key.data(), key.size()); - props.setValue(idx, values.getAt(i).data(), values.getAt(i).size()); - ++idx; - } - } -}; - -} // namespace <unnamed> - -namespace search::engine { - -using namespace search::fs4transport; - -void -PacketConverter::fillPacketProperties(const PropertiesMap &source, PropsVector& target) -{ - target.resize(source.size()); - PropertiesMap::ITR itr = source.begin(); - PropertiesMap::ITR end = source.end(); - for (uint32_t i = 0; itr != end; ++itr, ++i) { - const vespalib::string &name = itr->first; - const Properties &values = itr->second; - target[i].setName(name.c_str(), name.size()); - target[i].allocEntries(values.numValues()); - FS4PropertiesBuilder builder(target[i]); - values.visitProperties(builder); - LOG_ASSERT(builder.idx == target[i].size()); - LOG_ASSERT(builder.idx == values.numValues()); - } -} - -void -PacketConverter::toSearchRequest(const QUERYX &packet, SearchRequest &request) -{ - request.offset = packet._offset; - request.maxhits = packet._maxhits; - request.setTimeout(packet.getTimeout()); - request.queryFlags = packet.getQueryFlags(); - request.ranking = packet._ranking; - - for (const FS4Properties &src : packet._propsVector) { - Properties &dst = request.propertiesMap.lookupCreate(src.getName()); - for (uint32_t e = 0; e < src.size(); ++e) { - dst.add(vespalib::stringref(src.getKey(e), src.getKeyLen(e)), - vespalib::stringref(src.getValue(e), src.getValueLen(e))); - } - } - request.sortSpec = packet._sortSpec; - request.groupSpec.assign( packet._groupSpec.begin(), packet._groupSpec.end()); - request.sessionId.assign( packet._sessionId.begin(), packet._sessionId.end()); - request.location = packet._location; - request.stackItems = packet._numStackItems; - request.stackDump.assign( packet._stackDump.begin(), packet._stackDump.end()); -} - -void -PacketConverter::fromSearchRequest(const SearchRequest &request, QUERYX &packet) -{ - // not needed yet - (void) packet; - (void) request; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::toSearchReply(const QUERYRESULTX &packet, SearchReply &reply) -{ - // not needed yet - (void) packet; - (void) reply; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::fromSearchReply(const SearchReply &reply, QUERYRESULTX &packet) -{ - packet._offset = reply.offset; - packet._numDocs = reply.hits.size(); - packet._totNumDocs = reply.totalHitCount; - packet._maxRank = reply.maxRank; - packet.setDistributionKey(reply.getDistributionKey()); - if ( ! reply.sortIndex.empty()) { - packet._features |= QRF_SORTDATA; - uint32_t idxCnt = reply.sortIndex.size(); - LOG_ASSERT(reply.sortIndex.size() == reply.hits.size()+1); - // allocate for N hits (will make space for N+1 indexes) - packet.AllocateSortIndex(reply.hits.size()); - packet.AllocateSortData(reply.sortData.size()); - for (uint32_t i = 0; i < idxCnt; ++i) { - packet._sortIndex[i] = reply.sortIndex[i]; - } - memcpy(packet._sortData, &(reply.sortData[0]), reply.sortData.size()); - } - if ( ! reply.groupResult.empty()) { - packet._features |= QRF_GROUPDATA; - packet.AllocateGroupData(reply.groupResult.size()); - memcpy(packet._groupData, &(reply.groupResult[0]), reply.groupResult.size()); - } - packet._coverageDocs = reply.coverage.getCovered(); - packet._activeDocs = reply.coverage.getActive(); - packet._soonActiveDocs = reply.coverage.getSoonActive(); - packet._coverageDegradeReason = reply.coverage.getDegradeReason(); - packet.setNodesQueried(reply.coverage.getNodesQueried()); - packet.setNodesReplied(reply.coverage.getNodesReplied()); - if (reply.request) { - if (reply.request->queryFlags & QFLAG_COVERAGE_NODES) { - packet._features |= QRF_COVERAGE_NODES; - } - } - if (reply.useWideHits) { - packet._features |= QRF_MLD; - } - if (reply.propertiesMap.size() > 0) { - fillPacketProperties(reply.propertiesMap, packet._propsVector); - packet._features |= QRF_PROPERTIES; - } - uint32_t hitCnt = reply.hits.size(); - packet.AllocateHits(hitCnt); - for (uint32_t i = 0; i < hitCnt; ++i) { - packet._hits[i]._gid = reply.hits[i].gid; - packet._hits[i]._metric = reply.hits[i].metric; - packet._hits[i]._partid = reply.hits[i].path; - packet._hits[i].setDistributionKey(reply.hits[i].getDistributionKey()); - } -} - -void -PacketConverter::toDocsumRequest(const GETDOCSUMSX &packet, DocsumRequest &request) -{ - request.setTimeout(packet.getTimeout()); - request.ranking = packet._ranking; - request.queryFlags = packet._qflags; - request.resultClassName = packet._resultClassName; - for (const FS4Properties &src : packet._propsVector) { - Properties &dst = request.propertiesMap.lookupCreate(src.getName()); - for (uint32_t e = 0; e < src.size(); ++e) { - dst.add(vespalib::stringref(src.getKey(e), src.getKeyLen(e)), - vespalib::stringref(src.getValue(e), src.getValueLen(e))); - } - } - request.stackItems = packet._stackItems; - request.stackDump.assign(packet._stackDump.begin(), packet._stackDump.end()); - request.location = packet._location; - request._flags = packet._flags; - request.useWideHits = checkFeature(packet._features, GDF_MLD); - uint32_t hitCnt = packet._docid.size(); - request.hits.resize(hitCnt); - for (uint32_t i = 0; i < hitCnt; ++i) { - request.hits[i].gid = packet._docid[i]._gid; - request.hits[i].path = packet._docid[i]._partid; - } - Property sessionId = request.propertiesMap.rankProperties().lookup("sessionId"); - if (sessionId.found()) { - vespalib::string id = sessionId.get(); - request.sessionId.assign(id.begin(), id.end()); - } -} - -void -PacketConverter::fromDocsumRequest(const DocsumRequest &request, GETDOCSUMSX &packet) -{ - // not needed yet - (void) packet; - (void) request; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::toDocsumReplyElement(const DOCSUM &packet, DocsumReply::Docsum &docsum) -{ - // not needed yet - (void) packet; - (void) docsum; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::fromDocsumReplyElement(const DocsumReply::Docsum &docsum, DOCSUM &packet) -{ - if (docsum.data.get() != nullptr) { - packet.SetBuf(docsum.data.c_str(), docsum.data.size()); - } - packet.setGid(docsum.gid); -} - -void -PacketConverter::toMonitorRequest(const MONITORQUERYX &packet, MonitorRequest &request) -{ - request.flags = packet._qflags; - if ((packet._qflags & MQFLAG_REPORT_ACTIVEDOCS) != 0) { - request.reportActiveDocs = true; - } -} - -void -PacketConverter::fromMonitorRequest(const MonitorRequest &request, MONITORQUERYX &packet) -{ - // not needed yet - (void) packet; - (void) request; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::toMonitorReply(const MONITORRESULTX &packet, MonitorReply &reply) -{ - // not needed yet - (void) packet; - (void) reply; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::fromMonitorReply(const MonitorReply &reply, MONITORRESULTX &packet) -{ - if (reply.mld) { - packet._features |= MRF_MLD; - } - if (reply.activeDocsRequested) { - packet._features |= MRF_ACTIVEDOCS; - packet._activeDocs = reply.activeDocs; - } - packet._partid = reply.partid; - packet._timestamp = reply.timestamp; - packet._totalNodes = reply.totalNodes; - packet._activeNodes = reply.activeNodes; - packet._totalParts = reply.totalParts; - packet._activeParts = reply.activeParts; - packet._rflags = reply.flags; - if (packet._rflags != 0) { - packet._features |= MRF_RFLAGS; - } -} - -void -PacketConverter::fromTraceReply(const TraceReply &reply, TRACEREPLY &packet) -{ - fillPacketProperties(reply.propertiesMap, packet._propsVector); -} - -} - diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.h b/searchlib/src/vespa/searchlib/engine/packetconverter.h deleted file mode 100644 index 90fc27c8c97..00000000000 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.h +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "searchrequest.h" -#include "searchreply.h" -#include "docsumrequest.h" -#include "docsumreply.h" -#include "monitorrequest.h" -#include "monitorreply.h" -#include "tracereply.h" -#include <vespa/searchlib/common/packets.h> - -namespace search::engine { - - -/** - * This class helps convert data back and forth between transport - * packets and engine api request/reply objects. All converting - * methods expect the const object to be fully filled out and the - * non-const object to be newly created and thus empty. Half of the - * methods are left unimplemented for now as they would only be needed - * if we also were to use the api to wrap remote engines. However, if - * such a time comes, we will probably not be using the packet - * protocol anymore anyways. - **/ -class PacketConverter -{ -private: - PacketConverter(); // can not be instantiated - PacketConverter(const PacketConverter &); - PacketConverter &operator=(const PacketConverter &); - -public: - typedef search::fs4transport::FS4Packet_QUERYX QUERYX; - typedef search::fs4transport::FS4Packet_QUERYRESULTX QUERYRESULTX; - typedef search::fs4transport::FS4Packet_ERROR ERROR; - typedef search::fs4transport::FS4Packet_GETDOCSUMSX GETDOCSUMSX; - typedef search::fs4transport::FS4Packet_DOCSUM DOCSUM; - typedef search::fs4transport::FS4Packet_EOL EOL; - typedef search::fs4transport::FS4Packet_MONITORQUERYX MONITORQUERYX; - typedef search::fs4transport::FS4Packet_MONITORRESULTX MONITORRESULTX; - typedef search::fs4transport::FS4Packet_TRACEREPLY TRACEREPLY; - - /** - * Utility conversion from a "fef" set of propertymaps to an array of FS4Properties. - * @return false if no properties were converted. - **/ - static void - fillPacketProperties(const PropertiesMap &source, search::fs4transport::PropsVector& target); - - /** - * Convert from a QUERYX packet to a SearchRequest object. - * - * @param packet transport packet - * @param request api request object - **/ - static void toSearchRequest(const QUERYX &packet, SearchRequest &request); - - /** - * Convert from a SearchRequest object to a QUERYX packet. - * - * (NOT YET IMPLEMENTED) - * - * @param request api request object - * @param packet transport packet - **/ - static void fromSearchRequest(const SearchRequest &request, QUERYX &packet); - - /** - * Convert from a QUERYRESULTX packet to a SearchReply object. - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param reply api reply object - **/ - static void toSearchReply(const QUERYRESULTX &packet, SearchReply &reply); - - /** - * Convert from a SearchReply object to a QUERYRESULTX - * packet. Note that this method only handles the query result - * aspect of the reply, errors and queue length reporting still - * needs to be handled separately by the code using this utility - * method. - * - * @param reply api reply object - * @param packet transport packet - **/ - static void fromSearchReply(const SearchReply &reply, QUERYRESULTX &packet); - - /** - * Convert from a GETDOCSUMSX packet to a DocsumRequest object. - * - * @param packet transport packet - * @param request api request object - **/ - static void toDocsumRequest(const GETDOCSUMSX &packet, DocsumRequest &request); - - /** - * Convert from a DocsumRequest object to a GETDOCSUMSX packet. - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param request api request object - **/ - static void fromDocsumRequest(const DocsumRequest &request, GETDOCSUMSX &packet); - - /** - * Convert from a DOCSUM packet to an entry in a DocsumReply object - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param docsum api reply object element - **/ - static void toDocsumReplyElement(const DOCSUM &packet, DocsumReply::Docsum &docsum); - - /** - * Convert from an entry in a DocsumReply object to a DOCSUM packet. - * - * @param docsum api reply object element - * @param packet transport packet - **/ - static void fromDocsumReplyElement(const DocsumReply::Docsum &docsum, DOCSUM &packet); - - /** - * Convert a MONITORQUERYX packet to a MonitorRequest object. - * - * @param packet transport packet - * @param request api request object - **/ - static void toMonitorRequest(const MONITORQUERYX &packet, MonitorRequest &request); - - /** - * Convert from a MonitorRequest object to a MONITORQUERYX packet - * - * (NOT YET IMPLEMENTED) - * - * @param request api request object - * @param packet transport packet - **/ - static void fromMonitorRequest(const MonitorRequest &request, MONITORQUERYX &packet); - - /** - * Convert from a MONITORRESULTX packet to a MonitorReply object. - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param reply api reply object - **/ - static void toMonitorReply(const MONITORRESULTX &packet, MonitorReply &reply); - - /** - * Convert from a MonitorReply object to a MONITORRESULTX packet. - * - * @param reply api reply object - * @param packet transport packet - **/ - static void fromMonitorReply(const MonitorReply &reply, MONITORRESULTX &packet); - - /** - * Convert from a TraceReply object to a TRACE packet. - * - * @param reply api reply object - * @param packet transport packet - **/ - static void fromTraceReply(const TraceReply &reply, TRACEREPLY &packet); -}; - -} - diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.cpp b/searchlib/src/vespa/searchlib/engine/searchreply.cpp index ef39e4da027..7fc0ac81289 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.cpp +++ b/searchlib/src/vespa/searchlib/engine/searchreply.cpp @@ -16,8 +16,6 @@ SearchReply::SearchReply() coverage(), useWideHits(false), hits(), - errorCode(0), - errorMessage(), request() { } @@ -35,8 +33,6 @@ SearchReply::SearchReply(const SearchReply &rhs) : coverage (rhs.coverage), useWideHits (rhs.useWideHits), hits (rhs.hits), - errorCode (rhs.errorCode), - errorMessage (rhs.errorMessage), request() // NB not copied { } diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index d2645bbb2f6..d9468216a90 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -83,10 +83,6 @@ public: std::vector<Hit> hits; PropertiesMap propertiesMap; - // in case of error - uint32_t errorCode; - vespalib::string errorMessage; - SearchRequest::UP request; SearchReply(); diff --git a/searchlib/src/vespa/searchlib/engine/searchrequest.cpp b/searchlib/src/vespa/searchlib/engine/searchrequest.cpp index 53467e5c56c..9ca89a8636f 100644 --- a/searchlib/src/vespa/searchlib/engine/searchrequest.cpp +++ b/searchlib/src/vespa/searchlib/engine/searchrequest.cpp @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchrequest.h" -#include "packetconverter.h" -#include "proto_converter.h" -#include <vespa/fnet/frt/rpcrequest.h> namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/engine/transport_metrics.cpp b/searchlib/src/vespa/searchlib/engine/transport_metrics.cpp deleted file mode 100644 index 62d2b9d5489..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transport_metrics.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "transport_metrics.h" - -namespace search::engine { - -TransportMetrics::QueryMetrics::QueryMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("query", {}, "Query metrics", parent), - count("count", {{"logdefault"}}, "Query requests handled", this), - latency("latency", {{"logdefault"}}, "Query request latency", this) -{ -} - -TransportMetrics::QueryMetrics::~QueryMetrics() = default; - -TransportMetrics::DocsumMetrics::DocsumMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("docsum", {}, "Docsum metrics", parent), - count("count", {{"logdefault"}}, "Docsum requests handled", this), - docs("docs", {{"logdefault"}}, "Total docsums returned", this), - latency("latency", {{"logdefault"}}, "Docsum request latency", this) -{ -} - -TransportMetrics::DocsumMetrics::~DocsumMetrics() = default; - -TransportMetrics::TransportMetrics() - : metrics::MetricSet("transport", {}, "Transport server metrics", nullptr), - updateLock(), - query(this), - docsum(this) -{ -} - -TransportMetrics::~TransportMetrics() = default; - -} - diff --git a/searchlib/src/vespa/searchlib/engine/transport_metrics.h b/searchlib/src/vespa/searchlib/engine/transport_metrics.h deleted file mode 100644 index 8b85438f688..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transport_metrics.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metrics.h> -#include <vespa/vespalib/util/sync.h> - -namespace search::engine { - -struct TransportMetrics : metrics::MetricSet -{ - struct QueryMetrics : metrics::MetricSet { - metrics::LongCountMetric count; - metrics::DoubleAverageMetric latency; - - QueryMetrics(metrics::MetricSet *parent); - ~QueryMetrics(); - }; - - struct DocsumMetrics : metrics::MetricSet { - metrics::LongCountMetric count; - metrics::LongCountMetric docs; - metrics::DoubleAverageMetric latency; - - DocsumMetrics(metrics::MetricSet *parent); - ~DocsumMetrics(); - }; - - vespalib::Lock updateLock; - QueryMetrics query; - DocsumMetrics docsum; - - TransportMetrics(); - ~TransportMetrics() override; -}; - -} - diff --git a/searchlib/src/vespa/searchlib/engine/transportserver.cpp b/searchlib/src/vespa/searchlib/engine/transportserver.cpp deleted file mode 100644 index d14735b7770..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transportserver.cpp +++ /dev/null @@ -1,464 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packetconverter.h" -#include "transportserver.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/channel.h> -#include <vespa/fnet/connection.h> -#include <vespa/fnet/connector.h> -#include <vespa/fnet/iexecutable.h> -#include <vespa/vespalib/net/crypto_engine.h> - -#include <vespa/log/log.h> -LOG_SETUP(".engine.transportserver"); - -namespace search::engine { - -namespace { - -struct SearchRequestDecoder : SearchRequest::Source::Decoder { - PacketConverter::QUERYX *packet; - RelativeTime relative_time; - SearchRequestDecoder(PacketConverter::QUERYX *qx) - : packet(qx), relative_time(std::make_unique<FastosClock>()) {} - std::unique_ptr<SearchRequest> decode() override { - auto req = std::make_unique<SearchRequest>(std::move(relative_time)); - PacketConverter::toSearchRequest(*packet, *req); - return req; - } - ~SearchRequestDecoder() override { packet->Free(); } -}; - -std::unique_ptr<SearchRequest::Source::Decoder> search_request_decoder(PacketConverter::QUERYX *qx) { - return std::make_unique<SearchRequestDecoder>(qx); -} - -struct DocsumRequestDecoder : DocsumRequest::Source::Decoder { - PacketConverter::GETDOCSUMSX *packet; - RelativeTime relative_time; - DocsumRequestDecoder(PacketConverter::GETDOCSUMSX *gdx) - : packet(gdx), relative_time(std::make_unique<FastosClock>()) {} - std::unique_ptr<DocsumRequest> decode() override { - auto req = std::make_unique<DocsumRequest>(std::move(relative_time), false); - PacketConverter::toDocsumRequest(*packet, *req); - return req; - } - ~DocsumRequestDecoder() override { packet->Free(); } -}; - -std::unique_ptr<DocsumRequest::Source::Decoder> docsum_request_decoder(PacketConverter::GETDOCSUMSX *gdx) { - return std::make_unique<DocsumRequestDecoder>(gdx); -} - -} - -//----------------------------------------------------------------------------- - -typedef search::fs4transport::FS4PersistentPacketStreamer PacketStreamer; - -//----------------------------------------------------------------------------- - -constexpr uint32_t TransportServer::DEBUG_NONE; -constexpr uint32_t TransportServer::DEBUG_CONNECTION; -constexpr uint32_t TransportServer::DEBUG_CHANNEL; -constexpr uint32_t TransportServer::DEBUG_SEARCH; -constexpr uint32_t TransportServer::DEBUG_DOCSUM; -constexpr uint32_t TransportServer::DEBUG_MONITOR; -constexpr uint32_t TransportServer::DEBUG_UNHANDLED; -constexpr uint32_t TransportServer::DEBUG_ALL; - -void -TransportServer::SearchHandler::start() -{ - SearchReply::UP reply = parent._searchServer.search(std::move(request), *this); - if (reply) { - searchDone(std::move(reply)); - } -} - -void -TransportServer::SearchHandler::searchDone(SearchReply::UP reply) -{ - if (reply) { - const SearchReply &r = *reply; - if (r.valid) { - if (r.errorCode == 0) { - PacketConverter::QUERYRESULTX *p = new PacketConverter::QUERYRESULTX(); - PacketConverter::fromSearchReply(r, *p); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } else { - PacketConverter::ERROR *p = new PacketConverter::ERROR(); - p->_errorCode = r.errorCode; - p->setErrorMessage(r.errorMessage); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } - if (r.request) { - parent.updateQueryMetrics(r.request->getTimeUsed().sec()); // possible thread issue - } - } else { - PacketConverter::EOL *p = new PacketConverter::EOL(); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } - } else { - LOG(warning, "got <null> search reply from back-end"); - } - delete this; // we are done -} - -TransportServer::SearchHandler::~SearchHandler() -{ - channel->Free(); -} - -//----------------------------------------------------------------------------- - -void -TransportServer::DocsumHandler::start() -{ - DocsumReply::UP reply = parent._docsumServer.getDocsums(std::move(request), *this); - if (reply) { - getDocsumsDone(std::move(reply)); - } -} - -void -TransportServer::DocsumHandler::getDocsumsDone(DocsumReply::UP reply) -{ - if (reply) { - const DocsumReply &r = *reply; - for (uint32_t i = 0; i < r.docsums.size(); ++i) { - PacketConverter::DOCSUM *p = new PacketConverter::DOCSUM(); - PacketConverter::fromDocsumReplyElement(r.docsums[i], *p); - if (shouldLog(DEBUG_DOCSUM)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } - PacketConverter::EOL *p = new PacketConverter::EOL(); - if (shouldLog(DEBUG_DOCSUM)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - if (r.request) { - parent.updateDocsumMetrics(r.request->getTimeUsed().sec(), r.docsums.size()); - } - } else { - LOG(warning, "got <null> docsum reply from back-end"); - } - delete this; // we are done -} - -TransportServer::DocsumHandler::~DocsumHandler() -{ - channel->Free(); -} - -//----------------------------------------------------------------------------- - -void -TransportServer::MonitorHandler::start() -{ - MonitorReply::UP reply = parent._monitorServer.ping(std::move(request), *this); - if (reply) { - pingDone(std::move(reply)); - } -} - -void -TransportServer::MonitorHandler::pingDone(MonitorReply::UP reply) -{ - if (reply) { - const MonitorReply &r = *reply; - PacketConverter::MONITORRESULTX *p = new PacketConverter::MONITORRESULTX(); - PacketConverter::fromMonitorReply(r, *p); - if (shouldLog(DEBUG_MONITOR)) { - logPacket("outgoing packet", p, 0, connection); - } - connection->PostPacket(p, FNET_NOID); - } else { - LOG(warning, "got <null> monitor reply from back-end"); - } - delete this; // we are done -} - -TransportServer::MonitorHandler::~MonitorHandler() -{ - connection->SubRef(); -} - -//----------------------------------------------------------------------------- - -FNET_IPacketHandler::HP_RetCode -TransportServer::HandlePacket(FNET_Packet *packet, FNET_Context context) -{ - uint32_t pcode = packet->GetPCODE(); - FNET_Channel *channel = context._value.CHANNEL; - HP_RetCode rc = FNET_FREE_CHANNEL; - - if (channel->GetID() == FNET_NOID) { // admin packet - if (packet->IsChannelLostCMD()) { - _clients.erase(channel); - if (shouldLog(DEBUG_CONNECTION)) { - LOG(debug, "connection closed: tag=%u", channel->GetConnection()->GetContext()._value.INT); - } - } else if (pcode == search::fs4transport::PCODE_MONITORQUERYX) { - const PacketConverter::MONITORQUERYX &mqx = static_cast<PacketConverter::MONITORQUERYX&>(*packet); - if (shouldLog(DEBUG_MONITOR)) { - logPacket("incoming packet", packet, channel, 0); - } - MonitorRequest::UP req(new MonitorRequest()); - PacketConverter::toMonitorRequest(mqx, *req); - channel->GetConnection()->AddRef(); - _pending.push(new MonitorHandler(*this, std::move(req), channel->GetConnection())); - rc = FNET_KEEP_CHANNEL; - } else if (shouldLog(DEBUG_UNHANDLED)) { - logPacket("unhandled packet", packet, channel, 0); - } - } else { // search/docsum request - if (pcode == search::fs4transport::PCODE_QUERYX) { - PacketConverter::QUERYX * qx = static_cast<PacketConverter::QUERYX *>(packet); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("incoming packet", packet, channel, 0); - } - SearchRequest::Source req(search_request_decoder(qx)); - packet = nullptr; - _pending.push(new SearchHandler(*this, std::move(req), channel, _clients.size())); - rc = FNET_CLOSE_CHANNEL; - } else if (pcode == search::fs4transport::PCODE_GETDOCSUMSX) { - PacketConverter::GETDOCSUMSX * gdx = static_cast<PacketConverter::GETDOCSUMSX *>(packet); - if (shouldLog(DEBUG_DOCSUM)) { - logPacket("incoming packet", packet, channel, 0); - } - DocsumRequest::Source req(docsum_request_decoder(gdx)); - packet = nullptr; - _pending.push(new DocsumHandler(*this, std::move(req), channel)); - rc = FNET_CLOSE_CHANNEL; - } else if (shouldLog(DEBUG_UNHANDLED)) { - logPacket("unhandled packet", packet, channel, 0); - } - } - if (packet != nullptr) { - packet->Free(); - } - return rc; -} - -bool -TransportServer::InitAdminChannel(FNET_Channel *channel) -{ - if (_listener == nullptr) { - // handle race where we get an incoming connection and - // disables listening at the 'same time'. Note that sync close - // is only allowed in the InitAdminChannel method - channel->GetConnection()->Close(); // sync close - return false; - } - channel->SetContext(channel); - channel->SetHandler(this); - assert(_clients.count(channel) == 0); - _clients.insert(channel); - channel->GetConnection()->SetContext(FNET_Context(++_connTag)); - if (shouldLog(DEBUG_CONNECTION)) { - LOG(debug, "connection established: tag=%u", _connTag); - } - return true; -} - -bool -TransportServer::InitChannel(FNET_Channel *channel, uint32_t pcode) -{ - channel->SetContext(channel); - channel->SetHandler(this); - if (shouldLog(DEBUG_CHANNEL)) { - LOG(debug, "new channel: id=%u, first pcode=%u", channel->GetID(), pcode); - } - return true; -} - -void -TransportServer::Run(FastOS_ThreadInterface *, void *) -{ - _dispatchTask.ScheduleNow(); - _ready = true; - _transport.Main(); // <- transport event loop - _dispatchTask.Kill(); - _listenTask.Kill(); - discardRequests(); -} - -bool -TransportServer::updateListen() -{ - bool doListen = _doListen; - if (doListen) { - if (_listener == nullptr) { // start listening - _listener = _transport.Listen(_listenSpec.c_str(), &PacketStreamer::Instance, this); - if (_listener == nullptr) { - LOG(error, "Could not bind fnet transport socket to %s", _listenSpec.c_str()); - _failed = true; - return false; - } - } - } else { - if (_listener != nullptr) { // stop listening - _transport.Close(_listener); // async close - _listener->SubRef(); - _listener = nullptr; - // also close client connections - std::set<FNET_Channel*>::iterator it = _clients.begin(); - for (; it != _clients.end(); ++it) { - _transport.Close((*it)->GetConnection()); // async close - } - } - } - return true; -} - -void -TransportServer::dispatchRequests() -{ - while (!_pending.empty()) { - Handler *h = _pending.front(); - _pending.pop(); - h->start(); - } -} - -void -TransportServer::discardRequests() -{ - while (!_pending.empty()) { - Handler *h = _pending.front(); - _pending.pop(); - delete h; - } -} - -void -TransportServer::logPacket(vespalib::stringref msg, FNET_Packet *p, FNET_Channel *ch, FNET_Connection *conn) -{ - uint32_t chid = -1; - uint32_t conntag = -1; - vespalib::string str; - if (ch != 0) { - chid = ch->GetID(); - conntag = ch->GetConnection()->GetContext()._value.INT; - } else if (conn != 0) { - conntag = conn->GetContext()._value.INT; - } - search::fs4transport::FS4Packet *fs4p = dynamic_cast<search::fs4transport::FS4Packet*>(p); - if (fs4p != 0) { - str = fs4p->toString(0); - } else { - str = vespalib::make_string("packet { pcode=%u }", p->GetPCODE()); - } - LOG(debug, "%s (chid=%u, conn=%u):\n%s", msg.data(), chid, conntag, str.c_str()); -} - -void -TransportServer::updateQueryMetrics(double latency_s) -{ - vespalib::LockGuard guard(_metrics.updateLock); - _metrics.query.count.inc(); - _metrics.query.latency.set(latency_s); -} - -void -TransportServer::updateDocsumMetrics(double latency_s, uint32_t numDocs) -{ - vespalib::LockGuard guard(_metrics.updateLock); - _metrics.docsum.count.inc(); - _metrics.docsum.docs.inc(numDocs); - _metrics.docsum.latency.set(latency_s); -} - -//----------------------------------------------------------------------------- - -bool -TransportServer::shouldLog(uint32_t msgType) { - return (((msgType & _debugMask) != 0) - && ((msgType != DEBUG_MONITOR && LOG_WOULD_LOG(debug)) || - (msgType == DEBUG_MONITOR && LOG_WOULD_LOG(spam)))); -} - -TransportServer::TransportServer(SearchServer &searchServer, - DocsumServer &docsumServer, - MonitorServer &monitorServer, - int port, uint32_t debugMask) - : _searchServer(searchServer), - _docsumServer(docsumServer), - _monitorServer(monitorServer), - _transport(std::make_shared<vespalib::NullCryptoEngine>(), 1), // disable encryption - _ready(false), - _failed(false), - _doListen(true), - _threadPool(256 * 1024), - _listenSpec(), - _listener(0), - _clients(), - _pending(), - _dispatchTask(*this), - _listenTask(*this), - _connTag(0), - _debugMask(debugMask), - _metrics() -{ - _listenSpec = vespalib::make_string("tcp/%d", port); -} - -bool -TransportServer::start() -{ - if (!updateListen()) { - return false; - } - if (_threadPool.NewThread(this) == 0) { - LOG(error, "Could not start internal transport thread"); - _failed = true; - return false; - } - return true; -} - -int -TransportServer::getListenPort() -{ - struct Cmd : public FNET_IExecutable { - TransportServer &server; - vespalib::Gate done; - int port; - Cmd(TransportServer &s) : server(s), done(), port(-1) {} - void execute() override { - if (server._listener != 0) { - port = server._listener->GetPortNumber(); - } - done.countDown(); - } - }; - Cmd cmd(*this); - if (_transport.execute(&cmd)) { - cmd.done.await(); - } - return cmd.port; -}; - -TransportServer::~TransportServer() -{ - shutDown(); // ensure shutdown - if (_listener != 0) { - _listener->SubRef(); - _listener = 0; - } -} - -} - diff --git a/searchlib/src/vespa/searchlib/engine/transportserver.h b/searchlib/src/vespa/searchlib/engine/transportserver.h deleted file mode 100644 index 60bcc79a7fe..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transportserver.h +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "transport_metrics.h" -#include "searchapi.h" -#include "docsumapi.h" -#include "monitorapi.h" -#include <vespa/searchlib/common/packets.h> -#include <vespa/fnet/iserveradapter.h> -#include <vespa/fnet/ipackethandler.h> -#include <vespa/fnet/task.h> -#include <vespa/fnet/transport.h> -#include <vespa/vespalib/util/sync.h> -#include <vespa/fastos/thread.h> -#include <set> -#include <queue> - -namespace search::engine { - -/** - * Common transport server implementation interacting with the - * underlying search engine using the common search api. This - * implementation has less optimization tricks compared to the - * previous ones being integrated into specific applications. - **/ -class TransportServer : public FastOS_Runnable, - public FNET_IServerAdapter, - public FNET_IPacketHandler -{ -private: - TransportServer(const TransportServer &); - TransportServer &operator=(const TransportServer &); - - /** - * Task used to update listen status - **/ - struct ListenTask : public FNET_Task - { - TransportServer &parent; - ListenTask(TransportServer &p) : FNET_Task(p._transport.GetScheduler()), parent(p) {} - void PerformTask() override { parent.updateListen(); } - }; - - /** - * Task used to dispatch incoming requests in an untangled way - * (aka not in the packet callback). - **/ - struct DispatchTask : public FNET_Task - { - TransportServer &parent; - DispatchTask(TransportServer &p) : FNET_Task(p._transport.GetScheduler()), parent(p) {} - void PerformTask() override { - parent.dispatchRequests(); - ScheduleNow(); // run each tick - } - }; - - struct Handler; - - SearchServer &_searchServer; - DocsumServer &_docsumServer; - MonitorServer &_monitorServer; - FNET_Transport _transport; - bool _ready; // flag indicating initial readyness - bool _failed; // flag indicating a critical failure - bool _doListen; // flag telling us to accept requests or not - FastOS_ThreadPool _threadPool; // thread pool owning transport thread - vespalib::string _listenSpec; // where to listen; FNET connect spec - FNET_Connector *_listener; // object accepting incoming connections - std::set<FNET_Channel*> _clients; // the admin channel of all client connections - std::queue<Handler*> _pending; // queue of incoming requests not yet started - DispatchTask _dispatchTask; // task used to dispatch incoming requests - ListenTask _listenTask; // task used to update listen status - uint32_t _connTag; // sequential number used to tag connections - uint32_t _debugMask; // enable more debug logging with this - TransportMetrics _metrics; // metrics for this transport server - - /** - * Toplevel class used to wrap incoming requests. Actual objects - * are used both to delay starting the request until we are not in - * the packet delivery callback and also as the callback target - * used by the underlying api objects to notify completion of - * individual requests. - **/ - struct Handler - { - TransportServer &parent; - uint32_t _debugMask; - Handler(TransportServer &p) : parent(p), _debugMask(p._debugMask) {} - bool shouldLog(uint32_t msgType) { return parent.shouldLog(msgType); } // possible thread issue - virtual void start() = 0; - virtual ~Handler() {} - private: - Handler(const Handler &rhs); - Handler &operator=(const Handler &rhs); - }; - - /** - * Wrapper for search requests - **/ - struct SearchHandler : public Handler, - public SearchClient - { - SearchRequest::Source request; - FNET_Channel *channel; - uint32_t clientCnt; - - SearchHandler(TransportServer &p, SearchRequest::Source req, FNET_Channel *ch, uint32_t cnt) - : Handler(p), request(std::move(req)), channel(ch), clientCnt(cnt) {} - void start() override; - void searchDone(SearchReply::UP reply) override; - ~SearchHandler(); - }; - - /** - * Wrapper for docsum requests - **/ - struct DocsumHandler : public Handler, - public DocsumClient - { - DocsumRequest::Source request; - FNET_Channel *channel; - - DocsumHandler(TransportServer &p, DocsumRequest::Source req, FNET_Channel *ch) - : Handler(p), request(std::move(req)), channel(ch) {} - void start() override; - void getDocsumsDone(DocsumReply::UP reply) override; - ~DocsumHandler(); - }; - - /** - * Wrapper for monitor requests - **/ - struct MonitorHandler : public Handler, - public MonitorClient - { - MonitorRequest::UP request; - FNET_Connection *connection; - - MonitorHandler(TransportServer &p, MonitorRequest::UP req, FNET_Connection *conn) - : Handler(p), request(std::move(req)), connection(conn) {} - void start() override; - void pingDone(MonitorReply::UP reply) override; - ~MonitorHandler(); - }; - - // handle incoming network packets - HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context context) override; - - // set up admin channel for new clients - bool InitAdminChannel(FNET_Channel *channel) override; - - // set up channel for individual request - bool InitChannel(FNET_Channel *channel, uint32_t pcode) override; - - // entry point for thread running transport thread - void Run(FastOS_ThreadInterface *thisThread, void *arg) override; - - // update listen status - bool updateListen(); - - // dispatch incoming requests - void dispatchRequests(); - - // discard any pending requests during shutdown - void discardRequests(); - - // convenience method used to log packets - static void logPacket(vespalib::stringref msg, FNET_Packet *p, FNET_Channel *ch, FNET_Connection *conn); - - void updateQueryMetrics(double latency_s); - void updateDocsumMetrics(double latency_s, uint32_t numDocs); - -public: - /** - * Convenience typedes. - */ - typedef std::unique_ptr<TransportServer> UP; - typedef std::shared_ptr<TransportServer> SP; - - /** no debug logging flags set **/ - static constexpr uint32_t DEBUG_NONE = 0x00000000; - - /** log connect disconnect from clients **/ - static constexpr uint32_t DEBUG_CONNECTION = 0x00000001; - - /** log channel open events **/ - static constexpr uint32_t DEBUG_CHANNEL = 0x00000002; - - /** log search related packets **/ - static constexpr uint32_t DEBUG_SEARCH = 0x00000004; - - /** log docsum related packets **/ - static constexpr uint32_t DEBUG_DOCSUM = 0x00000008; - - /** log monitor related packets **/ - static constexpr uint32_t DEBUG_MONITOR = 0x00000010; - - /** log unhandled packets **/ - static constexpr uint32_t DEBUG_UNHANDLED = 0x00000020; - - /** all debug logging flags set **/ - static constexpr uint32_t DEBUG_ALL = 0x0000003f; - - /** - * Check if we should log a debug message - * - * @return true if we should log a message for this event - * @param msgType the event we might want to log - **/ - bool shouldLog(uint32_t msgType); - - /** - * Create a transport server based on the given underlying api - * objects. An appropriate debug mask can be made by or'ing - * together the appropriate DEBUG_ constants defined in this - * class. - * - * @param searchServer search api - * @param docsumServer docsum api - * @param monitorServer monitor api - * @param port listen port. - * @param debugMask mask indicating what information should be logged as debug messages. - **/ - TransportServer(SearchServer &searchServer, - DocsumServer &docsumServer, - MonitorServer &monitorServer, - int port, uint32_t debugMask = DEBUG_NONE); - - /** - * Obtain the metrics used by this transport server. - * - * @return internal metrics - **/ - TransportMetrics &getMetrics() { return _metrics; } - - /** - * Obtain the listen spec used by this transport server - * - * @return listen spec - **/ - const vespalib::string &getListenSpec() const { return _listenSpec; } - - /** - * Start this server. - * - * @return success(true)/failure(false) - **/ - bool start(); - - /** - * Check for initial readyness. - * - * @return true if we are ready. - **/ - bool isReady() const { return _ready; } - - /** - * Check if a critical error has occurred. - * - * @return true if something bad has happened. - **/ - bool isFailed() const { return _failed; } - - /** - * Get a reference to the internal fnet scheduler. - * - * @return fnet scheduler - **/ - FNET_Scheduler &getScheduler() { return *(_transport.GetScheduler()); } - - /** - * Set a flag indicating whether we should accept incoming - * requests or not. Setting the flag to false will make this - * server unavailable to any client application. - * - * @param listen flag indicating if we should listen - **/ - void setListen(bool listen) { - _doListen = listen; - _listenTask.ScheduleNow(); - } - - /** - * Check which port this server is currently listening to. This - * method is useful when using automatically allocated port - * numbers (listening to port 0). - * - * @return current listening port number, -1 if not listening. - **/ - int getListenPort(); - - /** - * Enable or disable nagles algorithm. - * - * @param noDelay set to true to disable nagles algorithm - **/ - void setTCPNoDelay(bool noDelay) { _transport.SetTCPNoDelay(noDelay); } - - /** - * Set a limit on how long a connection may be idle before closing it. - * - * @param millisecs max idle time in milliseconds - **/ - void setIdleTimeout(double millisecs) { _transport.SetIOCTimeOut((uint32_t) millisecs); } - - /** - * Shut down this component. This method will block until the - * transport server has been shut down. After this method returns, - * no new requests will be generated by this component. - **/ - void shutDown() { - _transport.ShutDown(false); - _threadPool.Close(); - } - - /** - * Destructor will perform shutdown if needed. - **/ - ~TransportServer(); -}; - -} - |